| Index: branches/new-upload/phase3/includes/GlobalFunctions.php |
| — | — | @@ -2189,7 +2189,13 @@ |
| 2190 | 2190 | * |
| 2191 | 2191 | * @param string $cmd |
| 2192 | 2192 | */ |
| 2193 | | -function wfShellBackgroundExec(&$cmd){ |
| | 2193 | +function wfShellBackgroundExec( $cmd ){ |
| | 2194 | + wfDebug( "wfShellBackgroundExec: $cmd\n" ); |
| | 2195 | + |
| | 2196 | + if ( ! wfShellExecEnabled() ) { |
| | 2197 | + return "Unable to run external programs"; |
| | 2198 | + } |
| | 2199 | + |
| 2194 | 2200 | $pid = shell_exec( "nohup $cmd > /dev/null & echo $!" ); |
| 2195 | 2201 | return $pid; |
| 2196 | 2202 | } |
| Index: branches/new-upload/phase3/includes/api/ApiMain.php |
| — | — | @@ -375,7 +375,7 @@ |
| 376 | 376 | /** |
| 377 | 377 | * Execute the actual module, without any error handling |
| 378 | 378 | */ |
| 379 | | - protected function executeAction() { |
| | 379 | + protected function executeAction() { |
| 380 | 380 | // First add the id to the top element |
| 381 | 381 | $requestid = $this->getParameter('requestid'); |
| 382 | 382 | if(!is_null($requestid)) |
| — | — | @@ -425,10 +425,10 @@ |
| 426 | 426 | |
| 427 | 427 | if ($this->mPrinter->getNeedsRawData()) |
| 428 | 428 | $this->getResult()->setRawMode(); |
| 429 | | - } |
| 430 | | - |
| | 429 | + } |
| 431 | 430 | // Execute |
| 432 | 431 | $module->profileIn(); |
| | 432 | + |
| 433 | 433 | $module->execute(); |
| 434 | 434 | wfRunHooks('APIAfterExecute', array(&$module)); |
| 435 | 435 | $module->profileOut(); |
| Index: branches/new-upload/phase3/includes/api/ApiUpload.php |
| — | — | @@ -63,7 +63,39 @@ |
| 64 | 64 | //if getAPIresult did not exit report the status error: |
| 65 | 65 | if( isset( $this->mUpload->status[ 'error' ] ) ) |
| 66 | 66 | $this->dieUsageMsg( $this->mUpload->status[ 'error' ] ); |
| 67 | | - |
| | 67 | + }else if( $this->mParams['internalSession'] ){ |
| | 68 | + |
| | 69 | + wfDebug("execFromSession: $tempPath"); |
| | 70 | + //get the params from the init session: |
| | 71 | + |
| | 72 | + $fileSize = filesize($tempPath); |
| | 73 | + |
| | 74 | + $this->mUpload = new UploadFromUpload(); |
| | 75 | + $this->mUpload->initialize( $this->mParams['filename'], $tempPath, $fileSize); |
| | 76 | + if( !isset( $this->mUpload ) ) |
| | 77 | + $this->dieUsage( 'No upload module set', 'nomodule' ); |
| | 78 | + |
| | 79 | + }else if( $this->mParams['httpstatus'] && $this->mParams['sessionkey']){ |
| | 80 | + //return the status of the given upload session_key: |
| | 81 | + if(!isset($_SESSION['wsDownload'][ $this->mParams['sessionkey'] ])){ |
| | 82 | + return $this->getResult()->addValue( null, $this->getModuleName(), |
| | 83 | + array( 'error' => 'invalid-session-key' |
| | 84 | + )); |
| | 85 | + } |
| | 86 | + $sd = & $_SESSION['wsDownload'][ $this->mParams['sessionkey'] ]; |
| | 87 | + //keep passing down the upload sessionkey |
| | 88 | + $statusResult = array( |
| | 89 | + 'upload_session_key' => $this->mParams['sessionkey'] |
| | 90 | + ); |
| | 91 | + |
| | 92 | + //put values into the final apiResult if available |
| | 93 | + if( isset($sd['apiUploadResult'])) $statusResult['apiUploadResult'] = $sd['apiUploadResult']; |
| | 94 | + if( isset($sd['loaded']) ) $statusResult['loaded'] = $sd['loaded']; |
| | 95 | + if( isset($sd['content_length']) ) $statusResult['content_length'] = $sd['content_length']; |
| | 96 | + |
| | 97 | + return $this->getResult()->addValue( null, $this->getModuleName(), |
| | 98 | + $statusResult |
| | 99 | + ); |
| 68 | 100 | }else if( $this->mParams['sessionkey'] ) { |
| 69 | 101 | // Stashed upload |
| 70 | 102 | $this->mUpload = new UploadFromStash(); |
| — | — | @@ -116,25 +148,7 @@ |
| 117 | 149 | |
| 118 | 150 | //finish up the exec command: |
| 119 | 151 | $this->doExecUpload(); |
| 120 | | - } |
| 121 | | - /** |
| 122 | | - * alternate entry point |
| 123 | | - */ |
| 124 | | - function execFromSession($tempPath){ |
| 125 | | - wfDebug("execFromSession: $tempPath"); |
| 126 | | - //get the params from the init session: |
| 127 | | - $this->mParams = $this->extractRequestParams(); |
| 128 | | - |
| 129 | | - $fileSize = filesize($tempPath); |
| 130 | | - |
| 131 | | - $this->mUpload = new UploadFromUpload(); |
| 132 | | - $this->mUpload->initialize( $this->mParams['filename'], $tempPath, $fileSize); |
| 133 | | - if( !isset( $this->mUpload ) ) |
| 134 | | - $this->dieUsage( 'No upload module set', 'nomodule' ); |
| 135 | | - |
| 136 | | - //finish up the exec command as a normal request: |
| 137 | | - $this->doExecUpload(); |
| 138 | | - } |
| | 152 | + } |
| 139 | 153 | function doExecUpload(){ |
| 140 | 154 | global $wgUser; |
| 141 | 155 | //Check whether the user has the appropriate permissions to upload anyway |
| — | — | @@ -260,7 +274,7 @@ |
| 261 | 275 | 'filename' => null, |
| 262 | 276 | 'file' => null, |
| 263 | 277 | 'chunk' => null, |
| 264 | | - 'url' => null, |
| | 278 | + 'url' => null, |
| 265 | 279 | 'enablechunks' => null, |
| 266 | 280 | 'comment' => array( |
| 267 | 281 | ApiBase :: PARAM_DFLT => '' |
| — | — | @@ -269,6 +283,7 @@ |
| 270 | 284 | 'ignorewarnings' => false, |
| 271 | 285 | 'done' => false, |
| 272 | 286 | 'sessionkey' => null, |
| | 287 | + 'httpstatus' => null, |
| 273 | 288 | 'chunksessionkey'=> null |
| 274 | 289 | ); |
| 275 | 290 | } |
| — | — | @@ -285,6 +300,7 @@ |
| 286 | 301 | 'ignorewarnings' => 'Ignore any warnings', |
| 287 | 302 | 'done' => 'When used with "chunks", Is sent to notify the api The last chunk is being uploaded.', |
| 288 | 303 | 'sessionkey' => 'Session key in case there were any warnings.', |
| | 304 | + 'httpstatus' => 'When set to true, will return the status of a given sessionKey (used for progress meters)', |
| 289 | 305 | 'chunksessionkey'=> 'Used to sync uploading of chunks', |
| 290 | 306 | ); |
| 291 | 307 | } |
| Index: branches/new-upload/phase3/includes/UploadFromUrl.php |
| — | — | @@ -26,8 +26,8 @@ |
| 27 | 27 | |
| 28 | 28 | $local_file = tempnam( $wgTmpDirectory, 'WEBUPLOAD' ); |
| 29 | 29 | parent::initialize( $name, $local_file, 0, true ); |
| 30 | | - |
| 31 | | - $this->mUrl = trim( $url ); |
| | 30 | + |
| | 31 | + $this->mUrl = trim( $url ); |
| 32 | 32 | } |
| 33 | 33 | public function isAsync(){ |
| 34 | 34 | return $this->dl_mode == Http::ASYNC_DOWNLOAD; |
| Index: branches/new-upload/phase3/includes/HttpFunctions.php |
| — | — | @@ -31,14 +31,15 @@ |
| 32 | 32 | global $wgPhpCliPath, $wgMaxUploadSize; |
| 33 | 33 | //do a quick check to HEAD to insure the file size is not > $wgMaxUploadSize to large no need to download it |
| 34 | 34 | $head = get_headers($url, 1); |
| 35 | | - if(isset($head['Content-Length']) && $head['Content-Length'] > $wgMaxUploadSize){ |
| | 35 | + if($head['Content-Length'] > $wgMaxUploadSize){ |
| 36 | 36 | return Status::newFatal('requested file length ' . $head['Content-Length'] . ' is greater than $wgMaxUploadSize: ' . $wgMaxUploadSize); |
| 37 | 37 | } |
| | 38 | + $content_length = (isset($head['Content-Length']))?$head['Content-Length']:null; |
| 38 | 39 | //check if we can find phpCliPath (for doing a background shell request to php to do the download: |
| 39 | 40 | if( $wgPhpCliPath && wfShellExecEnabled() && $dl_mode == self::ASYNC_DOWNLOAD){ |
| 40 | 41 | wfDebug("\ASYNC_DOWNLOAD\n"); |
| 41 | 42 | //setup session and shell call: |
| 42 | | - return self::initBackgroundDownload( $url, $target_file_path ); |
| | 43 | + return self::initBackgroundDownload( $url, $target_file_path, $content_length ); |
| 43 | 44 | }else if( $dl_mode== self::SYNC_DOWNLOAD ){ |
| 44 | 45 | wfDebug("\nSYNC_DOWNLOAD\n"); |
| 45 | 46 | //SYNC_DOWNLOAD download as much as we can in the time we have to execute |
| — | — | @@ -53,7 +54,7 @@ |
| 54 | 55 | * should write to a file location and give updates |
| 55 | 56 | * |
| 56 | 57 | */ |
| 57 | | - private function initBackgroundDownload( $url, $target_file_path ){ |
| | 58 | + private function initBackgroundDownload( $url, $target_file_path, $content_length = null ){ |
| 58 | 59 | global $wgMaxUploadSize, $IP, $wgPhpCliPath; |
| 59 | 60 | $status = Status::newGood(); |
| 60 | 61 | |
| — | — | @@ -63,8 +64,15 @@ |
| 64 | 65 | |
| 65 | 66 | //store the url and target path: |
| 66 | 67 | $_SESSION[ 'wsDownload' ][$upload_session_key]['url'] = $url; |
| 67 | | - $_SESSION[ 'wsDownload' ][$upload_session_key]['target_file_path'] = $target_file_path; |
| | 68 | + $_SESSION[ 'wsDownload' ][$upload_session_key]['target_file_path'] = $target_file_path; |
| 68 | 69 | |
| | 70 | + if($content_length) |
| | 71 | + $_SESSION[ 'wsDownload' ][$upload_session_key]['content_length'] = $content_length; |
| | 72 | + |
| | 73 | + //set initial loaded bytes: |
| | 74 | + $_SESSION[ 'wsDownload' ][$upload_session_key]['loaded'] = 0; |
| | 75 | + |
| | 76 | + |
| 69 | 77 | //run the background download request: |
| 70 | 78 | $cmd = $wgPhpCliPath . ' ' . $IP . "/maintenance/http_session_download.php --sid {$session_id} --usk {$upload_session_key}"; |
| 71 | 79 | $pid = wfShellBackgroundExec($cmd , $retval); |
| — | — | @@ -76,7 +84,7 @@ |
| 77 | 85 | //update the status value with the $upload_session_key (for the user to check on the status of the upload) |
| 78 | 86 | $status->value = $upload_session_key; |
| 79 | 87 | |
| 80 | | - //return good status with |
| | 88 | + //return good status |
| 81 | 89 | return $status; |
| 82 | 90 | } |
| 83 | 91 | function getUploadSessionKey(){ |
| — | — | @@ -92,7 +100,7 @@ |
| 93 | 101 | * (a given client could have started a few http uploads at once) |
| 94 | 102 | */ |
| 95 | 103 | public static function doSessionIdDownload( $session_id, $upload_session_key ){ |
| 96 | | - global $wgUser; |
| | 104 | + global $wgUser, $wgEnableWriteAPI; |
| 97 | 105 | wfDebug("\n\ndoSessionIdDownload\n\n"); |
| 98 | 106 | //set session to the provided key: |
| 99 | 107 | session_id($session_id); |
| — | — | @@ -109,45 +117,46 @@ |
| 110 | 118 | $wgUser = User::newFromSession(); |
| 111 | 119 | |
| 112 | 120 | //grab the session data to setup the request: |
| 113 | | - $sessionData = $_SESSION[ 'wsDownload' ][$upload_session_key]; |
| | 121 | + $sd =& $_SESSION[ 'wsDownload' ][$upload_session_key]; |
| 114 | 122 | //close down the session so we can other http queries can get session updates: |
| 115 | 123 | session_write_close(); |
| 116 | 124 | |
| 117 | | - $req = new HttpRequest( $sessionData['url'], array( |
| 118 | | - 'target_file_path' => $sessionData['target_file_path'], |
| | 125 | + $req = new HttpRequest( $sd['url'], array( |
| | 126 | + 'target_file_path' => $sd['target_file_path'], |
| 119 | 127 | 'upload_session_key' => $upload_session_key |
| 120 | 128 | ) ); |
| 121 | 129 | //run the actual request .. (this can take some time) |
| 122 | | - wfDebug("do Request: " . $sessionData['url'] . ' tf: ' .$sessionData['target_file_path'] ); |
| | 130 | + wfDebug("do Request: " . $sd['url'] . ' tf: ' . $sd['target_file_path'] ); |
| 123 | 131 | $status = $req->doRequest(); |
| 124 | | - |
| | 132 | + wfDebug("done with req status is: ". $status->isOK(). ' '.$status->value. "\n"); |
| 125 | 133 | if( $status->isOK() ){ |
| 126 | 134 | //start up the session again: |
| 127 | 135 | if( session_start() === false){ |
| 128 | 136 | wfDebug( __METHOD__ . ' ERROR:: Could not start session'); |
| 129 | | - } |
| | 137 | + } |
| 130 | 138 | //re-grab the updated session data: |
| 131 | | - $sessionData = $_SESSION[ 'wsDownload' ][$upload_session_key]; |
| 132 | | - $reqData = $sessionData['mParam']; |
| | 139 | + $sd =& $_SESSION[ 'wsDownload' ][$upload_session_key]; |
| | 140 | + |
| | 141 | + //setup the faxRequest |
| | 142 | + $fauxReqData = $sd['mParams']; |
| | 143 | + $fauxReqData['action'] = 'upload'; |
| | 144 | + $fauxReqData['format'] = 'json'; |
| | 145 | + $fauxReqData['internalSession'] = true; |
| | 146 | + $fauxReqData['sessionkey'] = $upload_session_key; |
| | 147 | + //evil but no other clean way about it: |
| 133 | 148 | |
| 134 | | - $reqData['action'] = 'upload'; |
| 135 | 149 | |
| 136 | | - wfDebug('running FauxRequest: ' . print_r($reqData, true) ); |
| 137 | | - |
| 138 | | - $faxReq = new FauxRequest($reqData, true); |
| | 150 | + $faxReq = new FauxRequest($fauxReqData, true); |
| 139 | 151 | $processor = new ApiMain($faxReq, $wgEnableWriteAPI); |
| 140 | | - |
| 141 | | - //init the mUpload var for the $processor |
| 142 | | - $processor->execFromSession($sessionData['target_file_path']); |
| 143 | | - |
| | 152 | + //init the mUpload var for the $processor |
| 144 | 153 | ob_start(); |
| 145 | | - $processor->doExecUpload(); |
| | 154 | + $processor->execute(); |
| 146 | 155 | $apiUploadResult = ob_get_contents(); |
| 147 | 156 | ob_get_clean(); |
| 148 | 157 | |
| 149 | | - |
| | 158 | + wfDebug(":: $apiUploadResult \n" ); |
| 150 | 159 | //the status updates runner will grab the result form the session: |
| 151 | | - $_SESSION[ 'wsDownload' ][$upload_session_key]['apiUploadResult'] = $apiUploadResult; |
| | 160 | + $sd['apiUploadResult'] = $apiUploadResult; |
| 152 | 161 | session_write_close(); |
| 153 | 162 | } |
| 154 | 163 | } |
| — | — | @@ -222,9 +231,7 @@ |
| 223 | 232 | private function doCurlReq(){ |
| 224 | 233 | global $wgHTTPFileTimeout, $wgHTTPProxy, $wgTitle; |
| 225 | 234 | |
| 226 | | - $status = Status::newGood(); |
| 227 | | - wfDebug("\ncurReq: $this->url (sleep 1 sec)\n"); |
| 228 | | - sleep(1); |
| | 235 | + $status = Status::newGood(); |
| 229 | 236 | $c = curl_init( $this->url ); |
| 230 | 237 | |
| 231 | 238 | //proxy setup: |
| — | — | @@ -278,8 +285,7 @@ |
| 279 | 286 | } |
| 280 | 287 | } catch (Exception $e) { |
| 281 | 288 | //do something with curl exec error? |
| 282 | | - } |
| 283 | | - wfDebug("\nDONE WITH curl_exec \n"); |
| | 289 | + } |
| 284 | 290 | //if direct request output the results to the stats value: |
| 285 | 291 | if( !$this->target_file_path && $status->isOK() ){ |
| 286 | 292 | $status->value = ob_get_contents(); |
| — | — | @@ -344,8 +350,9 @@ |
| 345 | 351 | var $session_id = null; |
| 346 | 352 | var $session_update_interval = 0; //how offten to update the session while downloading |
| 347 | 353 | |
| 348 | | - function simpleFileWriter($target_file_path, $session_id=false){ |
| | 354 | + function simpleFileWriter($target_file_path, $upload_session_key){ |
| 349 | 355 | $this->target_file_path = $target_file_path; |
| | 356 | + $this->upload_session_key = $upload_session_key; |
| 350 | 357 | $this->status = Status::newGood(); |
| 351 | 358 | //open the file: |
| 352 | 359 | $this->fp = fopen( $this->target_file_path, 'w'); |
| — | — | @@ -356,8 +363,15 @@ |
| 357 | 364 | $this->prevTime = time(); |
| 358 | 365 | } |
| 359 | 366 | public function callbackWriteBody($ch, $data_packet){ |
| 360 | | - global $wgMaxUploadSize; |
| 361 | | - wfDebug("\ncallbackWriteBody::" . strlen($data_packet) . "\n"); |
| | 367 | + global $wgMaxUploadSize; |
| | 368 | + |
| | 369 | + //write out the content |
| | 370 | + if( fwrite($this->fp, $data_packet) === false){ |
| | 371 | + wfDebug(__METHOD__ ." ::could-not-write-to-file\n"); |
| | 372 | + $this->status = Status::newFatal('HTTP::could-not-write-to-file'); |
| | 373 | + return 0; |
| | 374 | + } |
| | 375 | + |
| 362 | 376 | //check file size: |
| 363 | 377 | clearstatcache(); |
| 364 | 378 | $this->current_fsize = filesize( $this->target_file_path); |
| — | — | @@ -367,19 +381,12 @@ |
| 368 | 382 | $this->status = Status::newFatal('HTTP::file-has-grown-beyond-upload-limit-killing: downloaded more than ' . |
| 369 | 383 | Language::formatSize($wgMaxUploadSize) . ' '); |
| 370 | 384 | return 0; |
| 371 | | - } |
| 372 | | - wfDebug("passed fsize check\n"); |
| 373 | | - //write out the content |
| 374 | | - if( fwrite($this->fp, $data_packet) === false){ |
| 375 | | - wfDebug(__METHOD__ ." ::could-not-write-to-file\n"); |
| 376 | | - $this->status = Status::newFatal('HTTP::could-not-write-to-file'); |
| 377 | | - return 0; |
| 378 | | - } |
| 379 | | - wfDebug("did fwrite oky\n"); |
| 380 | | - |
| 381 | | - wfDebug("\n" .'check if we should update: ' . time() . ' - ' .$this->prevTime . ' > '. $this->session_update_interval . "\n"); |
| 382 | | - //if more than 2 second have passed update_session_progress |
| 383 | | - if($this->upload_session_key && (time() - $this->prevTime) > $this->session_update_interval ) { |
| | 385 | + } |
| | 386 | + //wfDebug('simulating slow download....'."\n"); |
| | 387 | + sleep(1); |
| | 388 | + |
| | 389 | + //if more than session_update_interval second have passed update_session_progress |
| | 390 | + if($this->upload_session_key && ( (time() - $this->prevTime) > $this->session_update_interval )) { |
| 384 | 391 | $this->prevTime = time(); |
| 385 | 392 | $session_status = $this->update_session_progress(); |
| 386 | 393 | if( !$session_status->isOK() ){ |
| — | — | @@ -387,8 +394,8 @@ |
| 388 | 395 | wfDebug( __METHOD__ . ' update session failed or was canceled'); |
| 389 | 396 | return 0; |
| 390 | 397 | } |
| 391 | | - } |
| 392 | | - return strlen($data_packet); |
| | 398 | + } |
| | 399 | + return strlen( $data_packet ); |
| 393 | 400 | } |
| 394 | 401 | public function update_session_progress(){ |
| 395 | 402 | $status = Status::newGood(); |
| — | — | @@ -396,8 +403,8 @@ |
| 397 | 404 | if( session_start() === false){ |
| 398 | 405 | wfDebug( __METHOD__ . ' could not start session'); |
| 399 | 406 | exit(0); |
| 400 | | - } |
| 401 | | - $sd =& $_SESSION[ 'wsDownload' ][$this->upload_session_key]; |
| | 407 | + } |
| | 408 | + $sd =& $_SESSION[ 'wsDownload' ][ $this->upload_session_key ]; |
| 402 | 409 | //check if the user canceled the request: |
| 403 | 410 | if( $sd['user_cancel'] == true ){ |
| 404 | 411 | //kill the download |
| — | — | @@ -405,12 +412,15 @@ |
| 406 | 413 | } |
| 407 | 414 | //update the progress bytes download so far: |
| 408 | 415 | $sd['loaded'] = $this->current_fsize; |
| 409 | | - wfDebug('set session loaded amount to: ' . $sd['loaded']); |
| | 416 | + wfDebug('set session loaded amount to: ' . $sd['loaded'] . "\n"); |
| 410 | 417 | //close down the session so we can other http queries can get session updates: |
| 411 | 418 | session_write_close(); |
| 412 | 419 | return $status; |
| 413 | 420 | } |
| 414 | | - public function close(){ |
| | 421 | + public function close(){ |
| | 422 | + //do a final session update: |
| | 423 | + $this->update_session_progress(); |
| | 424 | + //close up the file handle: |
| 415 | 425 | if(false === fclose( $this->fp )){ |
| 416 | 426 | $this->status = Status::newFatal('HTTP::could-not-close-file'); |
| 417 | 427 | } |
| Index: branches/new-upload/phase3/api.php |
| — | — | @@ -72,7 +72,6 @@ |
| 73 | 73 | * but we don't care here, as that is handled by the ctor. |
| 74 | 74 | */ |
| 75 | 75 | $processor = new ApiMain($wgRequest, $wgEnableWriteAPI); |
| 76 | | - |
| 77 | 76 | // Process data & print results |
| 78 | 77 | $processor->execute(); |
| 79 | 78 | |