r48335 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r48334‎ | r48335 | r48336 >
Date:00:37, 12 March 2009
Author:dale
Status:deferred (Comments)
Tags:
Comment:
updates for upload api chunk support
Modified paths:
  • /branches/new-upload/phase3/includes/UploadBase.php (modified) (history)
  • /branches/new-upload/phase3/includes/UploadFromChunks.php (modified) (history)
  • /branches/new-upload/phase3/includes/api/ApiUpload.php (modified) (history)
  • /branches/new-upload/phase3/includes/filerepo/FSRepo.php (modified) (history)
  • /branches/new-upload/phase3/includes/filerepo/FileRepo.php (modified) (history)
  • /branches/new-upload/phase3/includes/filerepo/RepoGroup.php (modified) (history)
  • /branches/new-upload/phase3/languages/messages/MessagesEn.php (modified) (history)

Diff [purge]

Index: branches/new-upload/phase3/includes/filerepo/FSRepo.php
@@ -211,7 +211,22 @@
212212 }
213213 return $status;
214214 }
215 -
 215+ function append( $srcPath, $toAppendPath ){
 216+ $status = $this->newGood();
 217+ //make sure files are there:
 218+ if ( !is_file( $srcPath ) )
 219+ $status->fatal( 'filenotfound', $srcPath );
 220+
 221+ if ( !is_file( $toAppendPath ) )
 222+ $status->fatal( 'filenotfound', $toAppendPath );
 223+
 224+ //I assume fopen($src, 'a') fopen ($$toAppend .. etc would be faster / more memory friendly
 225+ //but ideally we don't append "big" files so it does not matter
 226+ if( ! file_put_contents( $srcPath, file_get_contents( $toAppendPath ), FILE_APPEND ) )
 227+ $status->fatal( 'fileappenderror', $toAppendPath, $srcPath);
 228+
 229+ return $status;
 230+ }
216231 /**
217232 * Take all available measures to prevent web accessibility of new deleted
218233 * directories, in case the user has not configured offline storage
@@ -243,7 +258,15 @@
244259 $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel;
245260 return $result;
246261 }
 262+ /*append to a temporary file (used in chunks uploads) */
 263+ function appendToTemp( $srcPath, $appendDataPath ) {
 264+ //open the source file
247265
 266+ $result = $this->append( $srcPath, $appendDataPath );
 267+ $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel;
 268+ return $result;
 269+ }
 270+
248271 /**
249272 * Remove a temporary file or mark it for garbage collection
250273 * @param string $virtualUrl The virtual URL returned by storeTemp
Index: branches/new-upload/phase3/includes/filerepo/FileRepo.php
@@ -336,7 +336,7 @@
337337 }
338338 return $status;
339339 }
340 -
 340+
341341 /**
342342 * Store a batch of files
343343 *
Index: branches/new-upload/phase3/includes/filerepo/RepoGroup.php
@@ -148,7 +148,7 @@
149149 $this->initialiseRepos();
150150 }
151151 if ( $index === 'local' ) {
152 - return $this->localRepo;
 152+ return $this->localRepo;
153153 } elseif ( isset( $this->foreignRepos[$index] ) ) {
154154 return $this->foreignRepos[$index];
155155 } else {
Index: branches/new-upload/phase3/includes/api/ApiUpload.php
@@ -32,7 +32,7 @@
3333 * @ingroup API
3434 */
3535 class ApiUpload extends ApiBase {
36 -
 36+
3737 public function __construct($main, $action) {
3838 parent :: __construct($main, $action);
3939 }
@@ -43,7 +43,7 @@
4444 $this->mParams = $this->extractRequestParams();
4545 $request = $this->getMain()->getRequest();
4646 // Add the uploaded file to the params array
47 - $this->mParams['file'] = $request->getFileName( 'file' );
 47+ $this->mParams['file'] = $request->getFileName( 'file' );
4848
4949 // Check whether upload is enabled
5050 if( !UploadBase::isEnabled() )
@@ -52,12 +52,21 @@
5353 // One and only one of the following parameters is needed
5454 $this->requireOnlyOneParameter( $this->mParams,
5555 'sessionkey', 'file', 'url', 'enablechunks' );
 56+
 57+ if( $this->mParams['enablechunks'] ){
 58+ //chunks upload enabled
 59+ $this->mUpload = new UploadFromChunks();
 60+ $this->mUpload->initializeFromParams( $this->mParams );
5661
57 - if( $this->mParams['sessionkey'] ) {
 62+ if( isset( $this->mUpload->status[ 'error' ] ) )
 63+ $this->dieUsageMsg( $this->mUpload->status[ 'error' ] );
 64+
 65+ } else if( $this->mParams['sessionkey'] ) {
5866 // Stashed upload
5967 $this->mUpload = new UploadFromStash();
6068 $this->mUpload->initialize( $this->mParams['sessionkey'] );
61 - } else {
 69+
 70+ }else{
6271 // Upload from url or file or start a chunks request
6372
6473 // Parameter filename is required
@@ -75,11 +84,12 @@
7685 } elseif( isset( $this->mParams['url'] ) ) {
7786 $this->mUpload = new UploadFromUrl();
7887 $this->mUpload->initialize( $this->mParams['filename'], $this->mParams['url'] );
79 - }elseif (isset( $this->mParams['enablechunks'])) {
80 - $this->mUpload = new UploadFromChunks();
81 - $this->mUpload->initializeFromParams( $this->mParams );
8288 }
83 - }
 89+ }
 90+ if( !isset( $this->mUpload ) )
 91+ $this->dieUsageMsg( array( 'no upload module set' ) );
 92+
 93+
8494 // Check whether the user has the appropriate permissions to upload anyway
8595 $permission = $this->mUpload->isAllowed( $wgUser );
8696
@@ -115,35 +125,35 @@
116126 $result['result'] = 'Failure';
117127 $result['error'] = 'permission-denied';
118128 return $result;
119 - }
 129+ }
120130
121131 $verification = $this->mUpload->verifyUpload( $resultDetails );
122 - if( $verification != UploadFromBase::OK ) {
 132+ if( $verification != UploadBase::OK ) {
123133 $result['result'] = 'Failure';
124134 switch( $verification ) {
125 - case UploadFromBase::EMPTY_FILE:
 135+ case UploadBase::EMPTY_FILE:
126136 $result['error'] = 'empty-file';
127137 break;
128 - case UploadFromBase::FILETYPE_MISSING:
 138+ case UploadBase::FILETYPE_MISSING:
129139 $result['error'] = 'filetype-missing';
130140 break;
131 - case UploadFromBase::FILETYPE_BADTYPE:
 141+ case UploadBase::FILETYPE_BADTYPE:
132142 global $wgFileExtensions;
133143 $result['error'] = 'filetype-banned';
134144 $result['filetype'] = $resultDetails['finalExt'];
135145 $result['allowed-filetypes'] = $wgFileExtensions;
136146 break;
137 - case UploadFromBase::MIN_LENGHT_PARTNAME:
 147+ case UploadBase::MIN_LENGHT_PARTNAME:
138148 $result['error'] = 'filename-tooshort';
139149 break;
140 - case UploadFromBase::ILLEGAL_FILENAME:
 150+ case UploadBase::ILLEGAL_FILENAME:
141151 $result['error'] = 'illegal-filename';
142152 $result['filename'] = $resultDetails['filtered'];
143153 break;
144 - case UploadFromBase::OVERWRITE_EXISTING_FILE:
 154+ case UploadBase::OVERWRITE_EXISTING_FILE:
145155 $result['error'] = 'overwrite';
146156 break;
147 - case UploadFromBase::VERIFICATION_ERROR:
 157+ case UploadBase::VERIFICATION_ERROR:
148158 $result['error'] = 'verification-error';
149159 $args = $resultDetails['veri'];
150160 $code = array_shift( $args );
@@ -151,7 +161,7 @@
152162 $result['args'] = $args;
153163 $this->getResult()->setIndexedTagName( $result['args'], 'arg' );
154164 break;
155 - case UploadFromBase::UPLOAD_VERIFICATION_ERROR:
 165+ case UploadBase::UPLOAD_VERIFICATION_ERROR:
156166 $result['error'] = 'upload-verification-error';
157167 $result['upload-verification-error'] = $resultDetails['error'];
158168 break;
@@ -178,8 +188,14 @@
179189 $result['sessionkey'] = $sessionKey;
180190 return $result;
181191 }
182 - }
 192+ }
183193
 194+ //check for special API upload response:
 195+ $upApiResult = $this->mUpload->getAPIresult( $this->mParams['comment'], $this->mParams['watch'] );
 196+ if( $upApiResult != UploadBase::OK ) //if we have a result override return it
 197+ return $upApiResult;
 198+
 199+ //do the upload
184200 $status = $this->mUpload->performUpload( $this->mParams['comment'],
185201 $this->mParams['comment'], $this->mParams['watch'], $wgUser );
186202
@@ -231,8 +247,7 @@
232248 'comment' => 'Upload comment or initial page text',
233249 'watch' => 'Watch the page',
234250 'ignorewarnings' => 'Ignore any warnings',
235 - 'enablechunks' => 'Boolean If we are in chunk mode; accepts many small file POSTs',
236 - 'chunk_inx'=> 'The index of the chunk being uploaded. Used to order the build of a single file',
 251+ 'enablechunks' => 'Boolean If we are in chunk mode; accepts many small file POSTs',
237252 'done' => 'When used with "chunks", Is sent to notify the api The last chunk is being uploaded.',
238253 'sessionkey' => 'Session key in case there were any warnings, or uploading chunks'
239254 );
Index: branches/new-upload/phase3/includes/UploadFromChunks.php
@@ -9,23 +9,154 @@
1010 * more info at: http://firefogg.org/dev/chunk_post.html
1111 */
1212 class UploadFromChunks extends UploadBase {
13 - var $chunk_state; //init, chunk, done
14 - function initializeFromParams( $param ) {
15 - //start of a chunk request init the upload check destination file name
16 - //setup chunk folder
17 - if( !$param['sessionkey'] && !$param['chunk_inx'] && !$parm['done'] ){
18 -
 13+
 14+ var $chunk_mode; //init, chunk, done
 15+ var $mSessionKey = false;
 16+ var $status = array();
 17+
 18+ const INIT = 1;
 19+ const CHUNK = 2;
 20+ const DONE = 3;
 21+
 22+ function initializeFromParams( $param ) {
 23+ $this->initFromSessionKey( $param['sessionkey'] );
 24+
 25+ //set the chunk mode:
 26+ if( !$this->mSessionKey && !$parm['done'] ){
 27+ //session key not set init the chunk upload system:
 28+ $this->chunk_mode = UploadFromChunks::INIT;
 29+ }else if( $this->mSessionKey && !$parm['done']){
 30+ //this is a chunk piece
 31+ $this->chunk_mode = UploadFromChunks::CHUNK;
 32+
 33+ }else if( $this->mSessionKey && $parm['done']){
 34+ //this is the last chunk
 35+ $this->chunk_mode = UploadFromChunks::DONE;
 36+ }
 37+ return $this->status;
 38+ }
 39+
 40+ function initFromSessionKey( $sessionKey ){
 41+ if( !$sessionKey || empty( $sessionKey ) ){
 42+ return false;
1943 }
 44+ $this->mSessionKey = $sessionKey;
 45+ if( isset( $_SESSION['wsUploadData'][$this->mSessionKey]['version'] ) &&
 46+ $_SESSION['wsUploadData'][$this->mSessionKey]['version'] == self::SESSION_VERSION ) {
 47+ //update the local object from the session
 48+ $this->mComment = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mComment' ];
 49+ $this->mWatch = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mWatch' ];
 50+ $this->mFilteredName = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mFilteredName' ];
 51+ $this->mTempAppendPath = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mTempAppendPath' ];
 52+ }else{
 53+ $this->status = Array( 'error'=> 'missing session data');
 54+ return false;
 55+ }
2056
21 - //we are receiving a chunk process as an upload and stash it the folder with its index number.
22 - if( $param['sessionkey'] && $param['chunk_inx'] && !$parm['done']){
 57+ }
 58+ static function isValidRequest( $request ) {
 59+ $sessionData = $request->getSessionData('wsUploadData');
 60+ if(! self::isValidSessionKey(
 61+ $request->getInt( 'wpSessionKey' ),
 62+ $sessionData) )
 63+ return false;
 64+ //check for the file:
 65+ return (bool)$request->getFileTempName( 'file' );
 66+ }
 67+
 68+ /* check warnings depending on chunk_mode*/
 69+ function checkWarnings(){
 70+ $warning = array();
 71+ return $warning;
 72+ }
 73+
 74+ /* Verify whether the upload is sane.
 75+ * Returns self::OK or else an array with error information
 76+ */
 77+ function verifyUpload( $resultDetails ) {
 78+ /*
 79+ * check the internal chunk mode for alternative Verify path
 80+ * (for now just return "OK"
 81+ */
 82+ if( $this->chunk_mode == UploadFromChunks::INIT)
 83+ return self::OK;
 84+
 85+ return parent::verifyUpload( $resultDetails );
 86+ }
2387
 88+ function setupChunkSession( $comment, $watch ) {
 89+ $key = $this->getSessionKey();
 90+ //since we can't pass things along in POST store them in the Session:
 91+ $_SESSION['wsUploadData'][$key] = array(
 92+ 'mComment' => $comment,
 93+ 'mWatch' => $watch,
 94+ 'mFilteredName' => $this->mFilteredName,
 95+ 'mTempAppendPath' => null,
 96+ 'version' => self::SESSION_VERSION,
 97+ );
 98+ return $key;
 99+ }
 100+
 101+ //lets us return an api result (as flow for chunk uploads is kind of different than others.
 102+ function getAPIresult($comment, $watch){
 103+ if( $this->chunk_mode == UploadFromChunks::INIT ){
 104+ //verifyUpload & checkWarnings have already run .. just create the upload store return the upload session key
 105+ return array(
 106+ 'sessionkey'=> $this->setupChunkSession( $comment, $watch )
 107+ );
 108+ }else if( $this->chunk_mode == UploadFromChunks::CHUNK ){
 109+
 110+ $this->doChunkAppend();
 111+
 112+ //return success:
 113+ return array(
 114+ 'result' => 1
 115+ );
 116+
 117+ }else if( $this->chunk_mode == UploadFromChunks::DONE ){
 118+ //append the last chunk:
 119+ $this->doChunkAppend();
 120+ //process the upload normally:
 121+ return UploadFrom::OK;
 122+ }
 123+ }
 124+ //append the given chunk to the temporary uploaded file. (if no temporary uploaded file exists created it.
 125+ function doChunkAppend(){
 126+ //if we don't have a mTempAppendPath to append to generate that:
 127+ if( ! $this->mTempAppendPath ){
 128+ //make a chunk store path. (append tmp file to chunk)
 129+ print "save Temp: " . $this->mTempPath . ' '. $this->mDestName . "\n";
 130+ if( isset( $this->mDestName ) ){
 131+ $stash = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
 132+ if( !$stash ) {
 133+ # Couldn't save the file.
 134+ return false;
 135+ }
 136+ //update the mDestName
 137+ $this->mTempAppendPath = $stash;
 138+ $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ] = $this->mTempAppendPath;
 139+ }
 140+ }else{
 141+ //make sure the file exists:
 142+ if( is_file( $this->mTempAppendPath ) ){
 143+ print "append: " . $this->mTempPath . ' to ' . $this->mTempAppendPath . "\n";
 144+ $this->appendToUploadFile( $this->mTempAppendPath, $this->mTempPath );
 145+ }
24146 }
 147+ //@@todo api should allow for "comment vs page text"
25148
26 - //this is the last chunk
27 - if( $param['sessionkey'] && $param['chunk_inx'] && $parm['done']){
28 -
 149+
 150+ //do the chunk append:
 151+
 152+ //do the actual upload:
 153+
 154+ }
 155+
 156+ function checkAPIresultOverride(){
 157+ if( $this->chunk_mode == UploadFromChunks::INIT ){
 158+ return true;
 159+ }else{
 160+ return false;
29161 }
30 -
31162 }
32163 }
Index: branches/new-upload/phase3/includes/UploadBase.php
@@ -21,12 +21,13 @@
2222 const UPLOAD_VERIFICATION_ERROR = 11;
2323 const UPLOAD_WARNING = 12;
2424 const INTERNAL_ERROR = 13;
 25+ const MIN_LENGHT_PARTNAME = 14;
2526
2627 const SESSION_VERSION = 2;
2728
2829 /**
2930 * Returns true if uploads are enabled.
30 - * Can be overriden by subclasses.
 31+ * Can be override by subclasses.
3132 */
3233 static function isEnabled() {
3334 global $wgEnableUploads;
@@ -319,9 +320,8 @@
320321 $status = $this->mLocalFile->upload( $this->mTempPath, $comment, $pageText,
321322 File::DELETE_SOURCE, $this->mFileProps, false, $user );
322323
323 - if( $status->isGood() && $watch ) {
324 - $user->addWatch( $this->mLocalFile->getTitle() );
325 - }
 324+ if( $status->isGood() && $watch )
 325+ $user->addWatch( $this->mLocalFile->getTitle() );
326326
327327 if( $status->isGood() )
328328 wfRunHooks( 'UploadComplete', array( &$this ) );
@@ -418,12 +418,17 @@
419419 * @return string - full path the stashed file, or false on failure
420420 * @access private
421421 */
422 - function saveTempUploadedFile( $saveName, $tempName ) {
423 - global $wgOut;
 422+ function saveTempUploadedFile( $saveName, $tempName ) {
424423 $repo = RepoGroup::singleton()->getLocalRepo();
425424 $status = $repo->storeTemp( $saveName, $tempName );
426425 return $status;
427426 }
 427+ /* append to a stached file */
 428+ function appendToUploadFile($srcPath, $toAppendPath ){
 429+ $repo = RepoGroup::singleton()->getLocalRepo();
 430+ $status = $repo->append($srcPath, $toAppendPath);
 431+ return $status;
 432+ }
428433
429434 /**
430435 * Stash a file in a temporary directory for later processing,
@@ -435,20 +440,29 @@
436441 * @access private
437442 */
438443 function stashSession() {
439 - $status = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
 444+ $stash = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
440445
441 - if( !$status->isGood() ) {
 446+ if( !$stash ) {
442447 # Couldn't save the file.
443448 return false;
444449 }
445450
446 - return array(
447 - 'mTempPath' => $status->value,
 451+ $key = $this->getSessionKey ();
 452+ $_SESSION['wsUploadData'][$key] = array(
 453+ 'mTempPath' => $stash,
448454 'mFileSize' => $this->mFileSize,
 455+ 'mSrcName' => $this->mSrcName,
449456 'mFileProps' => $this->mFileProps,
450457 'version' => self::SESSION_VERSION,
451458 );
 459+ return $key;
452460 }
 461+ //pull session Key gen from stash in cases where we want to start an upload without much information
 462+ function getSessionKey(){
 463+ $key = mt_rand( 0, 0x7fffffff );
 464+ $_SESSION['wsUploadData'][$key] = array();
 465+ return $key;
 466+ }
453467
454468 /**
455469 * Remove a temporarily kept file stashed by saveTempUploadedFile().
@@ -844,6 +858,10 @@
845859 return true;
846860
847861 }
 862+ /* allow for getAPIresult override (normally just return UploadFrom::OK to continue form processing */
 863+ function getAPIresult(){
 864+ return UploadFrom::OK;
 865+ }
848866
849867 /**
850868 * Check if a user is the last uploader
Index: branches/new-upload/phase3/languages/messages/MessagesEn.php
@@ -823,6 +823,7 @@
824824 'readonly_lag' => 'The database has been automatically locked while the slave database servers catch up to the master',
825825 'internalerror' => 'Internal error',
826826 'internalerror_info' => 'Internal error: $1',
 827+'fileappenderror' => 'Could not append $1 to $2',
827828 'filecopyerror' => 'Could not copy file "$1" to "$2".',
828829 'filerenameerror' => 'Could not rename file "$1" to "$2".',
829830 'filedeleteerror' => 'Could not delete file "$1".',

Comments

#Comment by Catrope (talk | contribs)   08:10, 12 March 2009
if( !isset( $this->mUpload ) )		
	$this->dieUsageMsg( array( 'no upload module set' ) );

dieUsageMsg() doesn't work this way. You can either specify the error code and text like this:

$this->dieUsage( 'No upload module set', 'nomodule' );

Either way, I'm not sure if this error will actually occur ever; if it can't happen, you should remove the dieUsage() call. Or use a message key like this:

$this->dieUsageMsg( array( 'nouploadmodule' ) );

and define the nomodule message before that with:

ApiBase::$messageMap['nouploadmodule'] = array( 'code' => 'nomodule', 'info' => 'No upload module set' );
#Comment by Catrope (talk | contribs)   08:11, 12 March 2009

WTF, CR somehow cut my last line:

Either way, I'm not sure when this error will actually occur; if it can't happen, you should remove the dieUsageMsg() call.

#Comment by Mdale (talk | contribs)   18:07, 12 March 2009

thanks for checking that out... put the suggested fix in.. will also try and make sure that error does not happen (somehow it _was_ happening)

Status & tagging log