r49675 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r49674‎ | r49675 | r49676 >
Date:20:26, 20 April 2009
Author:dale
Status:deferred
Tags:
Comment:
* some fixes for chunk uploads (still not 100% there yet)
* cleaned up the way chunks extends the UploadBase
Modified paths:
  • /branches/new-upload/phase3/includes/DefaultSettings.php (modified) (history)
  • /branches/new-upload/phase3/includes/UploadBase.php (modified) (history)
  • /branches/new-upload/phase3/includes/UploadFromChunks.php (modified) (history)
  • /branches/new-upload/phase3/includes/api/ApiUpload.php (modified) (history)
  • /branches/new-upload/phase3/includes/filerepo/FSRepo.php (modified) (history)

Diff [purge]

Index: branches/new-upload/phase3/includes/filerepo/FSRepo.php
@@ -220,8 +220,7 @@
221221 if ( !is_file( $toAppendPath ) )
222222 $status->fatal( 'filenotfound', $toAppendPath );
223223
224 - //I assume fopen($src, 'a') fopen ($$toAppend .. etc would be faster / more memory friendly
225 - //but ideally we don't append "big" files so it does not matter
 224+ //do the append:
226225 if( ! file_put_contents( $srcPath, file_get_contents( $toAppendPath ), FILE_APPEND ) )
227226 $status->fatal( 'fileappenderror', $toAppendPath, $srcPath);
228227
@@ -260,10 +259,8 @@
261260 }
262261 /*append to a temporary file (used in chunks uploads) */
263262 function appendToTemp( $srcPath, $appendDataPath ) {
264 - //open the source file
265 -
266 - $result = $this->append( $srcPath, $appendDataPath );
267 - $result->value = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel;
 263+ //append to the "shared" temporary file
 264+ $result = $this->append( $srcPath, $appendDataPath );
268265 return $result;
269266 }
270267
Index: branches/new-upload/phase3/includes/api/ApiUpload.php
@@ -27,7 +27,7 @@
2828 require_once ("ApiBase.php");
2929 }
3030
31 -
 31+
3232 /**
3333 * @ingroup API
3434 */
@@ -38,10 +38,13 @@
3939 }
4040
4141 public function execute() {
42 - global $wgUser;
 42+ global $wgUser;
 43+
 44+
4345 $this->getMain()->requestWriteMode();
4446 $this->mParams = $this->extractRequestParams();
45 - $request = $this->getMain()->getRequest();
 47+ $request = $this->getMain()->getRequest();
 48+
4649 // Add the uploaded file to the params array
4750 $this->mParams['file'] = $request->getFileName( 'file' );
4851
@@ -56,19 +59,17 @@
5760 if( $this->mParams['enablechunks'] ){
5861 //chunks upload enabled
5962 $this->mUpload = new UploadFromChunks();
60 - $this->mUpload->initializeFromParams( $this->mParams );
61 -
62 - if( isset( $this->mUpload->status[ 'error' ] ) )
63 - $this->dieUsageMsg( $this->mUpload->status[ 'error' ] );
64 -
 63+ $this->mUpload->initializeFromParams( $this->mParams, $request );
 64+ //if getAPIresult did not exit report the status error:
 65+ if( isset( $this->mUpload->status[ 'error' ] ) )
 66+ $this->dieUsageMsg( $this->mUpload->status[ 'error' ] );
 67+
6568 } else if( $this->mParams['sessionkey'] ) {
6669 // Stashed upload
6770 $this->mUpload = new UploadFromStash();
68 - $this->mUpload->initialize( $this->mParams['sessionkey'] );
69 -
 71+ $this->mUpload->initialize( $this->mParams['sessionkey'] );
7072 }else{
71 - // Upload from url or file or start a chunks request
72 -
 73+ // Upload from url or file
7374 // Parameter filename is required
7475 if( !isset( $this->mParams['filename'] ) )
7576 $this->dieUsageMsg( array( 'missingparam', 'filename' ) );
@@ -77,15 +78,16 @@
7879 if( isset( $this->mParams['file'] ) ) {
7980 $this->mUpload = new UploadFromUpload();
8081 $this->mUpload->initialize(
 82+ $request->getFileName( 'file' ),
8183 $request->getFileTempName( 'file' ),
82 - $request->getFileSize( 'file' ),
83 - $request->getFileName( 'file' )
 84+ $request->getFileSize( 'file' )
8485 );
8586 } elseif( isset( $this->mParams['url'] ) ) {
8687 $this->mUpload = new UploadFromUrl();
8788 $this->mUpload->initialize( $this->mParams['filename'], $this->mParams['url'] );
8889 }
8990 }
 91+
9092 if( !isset( $this->mUpload ) )
9193 $this->dieUsage( 'No upload module set', 'nomodule' );
9294
@@ -99,8 +101,7 @@
100102 $this->dieUsageMsg( array( 'mustbeloggedin', 'upload' ) );
101103 else
102104 $this->dieUsageMsg( array( 'badaccess-groups' ) );
103 - }
104 -
 105+ }
105106 // Perform the upload
106107 $result = $this->performUpload();
107108
@@ -111,7 +112,7 @@
112113 }
113114
114115 private function performUpload() {
115 - global $wgUser;
 116+ global $wgUser;
116117 $result = array();
117118 $resultDetails = null;
118119
@@ -122,7 +123,7 @@
123124 return $result;
124125 }
125126
126 - $verification = $this->mUpload->verifyUpload( $resultDetails );
 127+ $verification = $this->mUpload->verifyUpload( $resultDetails );
127128 if( $verification != UploadBase::OK ) {
128129 $result['result'] = 'Failure';
129130 switch( $verification ) {
@@ -166,8 +167,7 @@
167168 break;
168169 }
169170 return $result;
170 - }
171 -
 171+ }
172172 if( !$this->mParams['ignorewarnings'] ) {
173173 $warnings = $this->mUpload->checkWarnings();
174174 if( $warnings ) {
@@ -183,13 +183,7 @@
184184 $result['sessionkey'] = $sessionKey;
185185 return $result;
186186 }
187 - }
188 -
189 - //check for special API upload response:
190 - $upApiResult = $this->mUpload->getAPIresult( $this->mParams['comment'], $this->mParams['watch'] );
191 - if( $upApiResult != UploadBase::OK ) //if we have a result override return it
192 - return $upApiResult;
193 -
 187+ }
194188 //do the upload
195189 $status = $this->mUpload->performUpload( $this->mParams['comment'],
196190 $this->mParams['comment'], $this->mParams['watch'], $wgUser );
@@ -222,6 +216,7 @@
223217 return array (
224218 'filename' => null,
225219 'file' => null,
 220+ 'chunk' => null,
226221 'url' => null,
227222 'comment' => array(
228223 ApiBase :: PARAM_DFLT => ''
@@ -231,6 +226,7 @@
232227 'enablechunks' => false,
233228 'done' => false,
234229 'sessionkey' => null,
 230+ 'chunksessionkey'=> null,
235231 );
236232 }
237233
@@ -244,7 +240,8 @@
245241 'ignorewarnings' => 'Ignore any warnings',
246242 'enablechunks' => 'Boolean If we are in chunk mode; accepts many small file POSTs',
247243 'done' => 'When used with "chunks", Is sent to notify the api The last chunk is being uploaded.',
248 - 'sessionkey' => 'Session key in case there were any warnings, or uploading chunks'
 244+ 'sessionkey' => 'Session key in case there were any warnings.',
 245+ 'chunksessionkey'=> 'Used to sync uploading of chunks',
249246 );
250247 }
251248
Index: branches/new-upload/phase3/includes/UploadFromChunks.php
@@ -18,28 +18,33 @@
1919 const CHUNK = 2;
2020 const DONE = 3;
2121
22 - function initializeFromParams( $param ) {
23 - $this->initFromSessionKey( $param['sessionkey'] );
24 -
 22+ function initializeFromParams( &$param , &$request) {
 23+ $this->initFromSessionKey( $param['chunksessionkey'] );
2524 //set the chunk mode:
26 - if( !$this->mSessionKey && !$parm['done'] ){
 25+ if( !$this->mSessionKey && !$param['done'] ){
2726 //session key not set init the chunk upload system:
28 - $this->chunk_mode = UploadFromChunks::INIT;
29 - }else if( $this->mSessionKey && !$parm['done']){
 27+ $this->chunk_mode = UploadFromChunks::INIT;
 28+ $this->mDesiredDestName = $param['filename'];
 29+
 30+ }else if( $this->mSessionKey && !$param['done']){
3031 //this is a chunk piece
3132 $this->chunk_mode = UploadFromChunks::CHUNK;
32 -
33 - }else if( $this->mSessionKey && $parm['done']){
 33+
 34+ //set chunk related vars:
 35+ $this->mTempPath = $request->getFileTempName( 'chunk' );
 36+ $this->mFileSize = $request->getFileSize( 'chunk' );
 37+
 38+ }else if( $this->mSessionKey && $param['done']){
3439 //this is the last chunk
3540 $this->chunk_mode = UploadFromChunks::DONE;
3641 }
3742 return $this->status;
3843 }
3944
40 - function initFromSessionKey( $sessionKey ){
 45+ function initFromSessionKey( $sessionKey ){
4146 if( !$sessionKey || empty( $sessionKey ) ){
4247 return false;
43 - }
 48+ }
4449 $this->mSessionKey = $sessionKey;
4550 if( isset( $_SESSION['wsUploadData'][$this->mSessionKey]['version'] ) &&
4651 $_SESSION['wsUploadData'][$this->mSessionKey]['version'] == self::SESSION_VERSION ) {
@@ -48,11 +53,11 @@
4954 $this->mWatch = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mWatch' ];
5055 $this->mFilteredName = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mFilteredName' ];
5156 $this->mTempAppendPath = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mTempAppendPath' ];
 57+ $this->mDesiredDestName = $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mDesiredDestName' ];
5258 }else{
5359 $this->status = Array( 'error'=> 'missing session data');
5460 return false;
55 - }
56 -
 61+ }
5762 }
5863 static function isValidRequest( $request ) {
5964 $sessionData = $request->getSessionData('wsUploadData');
@@ -69,88 +74,113 @@
7075 $warning = array();
7176 return $warning;
7277 }
73 -
 78+ function isEmptyFile(){
 79+ //does not apply to chunk init
 80+ if( $this->chunk_mode == UploadFromChunks::INIT ){
 81+ return false;
 82+ }else{
 83+ return parent::isEmptyFile();
 84+ }
 85+ }
7486 /* Verify whether the upload is sane.
7587 * Returns self::OK or else an array with error information
7688 */
77 - function verifyUpload( $resultDetails ) {
78 - /*
79 - * check the internal chunk mode for alternative Verify path
80 - * (for now just return "OK"
81 - */
82 - if( $this->chunk_mode == UploadFromChunks::INIT)
 89+ function verifyUpload( $resultDetails ) {
 90+ //no checks on chunk upload mode:
 91+ if( $this->chunk_mode == UploadFromChunks::INIT )
8392 return self::OK;
84 -
85 - return parent::verifyUpload( $resultDetails );
 93+
 94+ //verify on init and last chunk request
 95+ if( $this->chunk_mode == UploadFromChunks::CHUNK ||
 96+ $this->chunk_mode == UploadFromChunks::DONE )
 97+ return parent::verifyUpload( $resultDetails );
8698 }
87 -
 99+ //only run verifyFile on completed uploaded chunks
 100+ function verifyFile( $tmpFile ){
 101+ if( $this->chunk_mode == UploadFromChunks::DONE){
 102+ return parent::verifyFile($tmpFile);
 103+ }else{
 104+ return true;
 105+ }
 106+ }
88107 function setupChunkSession( $comment, $watch ) {
89 - $key = $this->getSessionKey();
90 - //since we can't pass things along in POST store them in the Session:
91 - $_SESSION['wsUploadData'][$key] = array(
 108+ $this->mSessionKey = $this->getSessionKey();
 109+
 110+ $_SESSION['wsUploadData'][ $this->mSessionKey ] = array(
92111 'mComment' => $comment,
93112 'mWatch' => $watch,
94113 'mFilteredName' => $this->mFilteredName,
95 - 'mTempAppendPath' => null,
 114+ 'mTempAppendPath' => null, //the repo append path (not temporary local node mTempPath)
 115+ 'mDesiredDestName' => $this->mDesiredDestName,
96116 'version' => self::SESSION_VERSION,
97117 );
98 - return $key;
 118+
 119+ return $this->mSessionKey;
99120 }
100121
101122 //lets us return an api result (as flow for chunk uploads is kind of different than others.
102 - function getAPIresult($comment, $watch){
103 - if( $this->chunk_mode == UploadFromChunks::INIT ){
104 - //verifyUpload & checkWarnings have already run .. just create the upload store return the upload session key
 123+ function performUpload($summary='', $comment='', $watch='', $user){
 124+ global $wgServer, $wgScriptPath;
 125+ if( $this->chunk_mode == UploadFromChunks::INIT ){
 126+
 127+ //firefogg expects a specific result per:
 128+ //http://www.firefogg.org/dev/chunk_post.html
 129+ print "{\"uploadUrl\": \"{$wgServer}{$wgScriptPath}/api.php?action=upload&format=json&enablechunks=true&chunksessionkey=".
 130+ $this->setupChunkSession( $comment, $watch ) . "\" }";
 131+ exit(0);
 132+
 133+ /*
 134+ * @@todo would be more ideal to have firefogg pass results back to the client to construct next chunk url
105135 return array(
106136 'sessionkey'=> $this->setupChunkSession( $comment, $watch )
107137 );
108 - }else if( $this->chunk_mode == UploadFromChunks::CHUNK ){
109 -
110 - $this->doChunkAppend();
111 -
112 - //return success:
113 - return array(
114 - 'result' => 1
115 - );
116 -
 138+ */
 139+ }else if( $this->chunk_mode == UploadFromChunks::CHUNK ){
 140+ $status = $this->doChunkAppend();
 141+ if( $status->isOK() ){
 142+ //return success:
 143+ //firefogg expects a specific result per:
 144+ //http://www.firefogg.org/dev/chunk_post.html
 145+ print "{\"result\": 1}";
 146+ exit(0);
 147+ /*return array(
 148+ 'result' => 1
 149+ );*/
 150+ }else{
 151+ return $status;
 152+ }
117153 }else if( $this->chunk_mode == UploadFromChunks::DONE ){
118154 //append the last chunk:
119155 if( $this->doChunkAppend() ){
120156 //process the upload normally:
121 - return UploadFrom::OK;
 157+ return Status::newGood('chunk upload done');
122158 }
123159 }
124160 }
125161 //append the given chunk to the temporary uploaded file. (if no temporary uploaded file exists created it.
126162 function doChunkAppend(){
127 - //if we don't have a mTempAppendPath to append to generate that:
 163+ //if we don't have a mTempAppendPath to generate a file from the chunk packaged var:
128164 if( ! $this->mTempAppendPath ){
129 - //make a chunk store path. (append tmp file to chunk)
130 - print "save Temp: " . $this->mTempPath . ' '. $this->mDestName . "\n";
131 - if( isset( $this->mDestName ) ){
132 - $stash = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
133 - if( !$stash ) {
134 - # Couldn't save the file.
135 - return false;
136 - }
137 - //update the mDestName
138 - $this->mTempAppendPath = $stash;
139 - $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ] = $this->mTempAppendPath;
140 - }
 165+ //die();
 166+ //get temp name:
 167+ //make a chunk store path. (append tmp file to chunk)
 168+ $status = $this->saveTempUploadedFile( $this->mDestName, $this->mTempPath );
 169+
 170+ if( $status->isOK() ) {
 171+ $this->mTempAppendPath = $status->value;
 172+ $_SESSION[ 'wsUploadData' ][ $this->mSessionKey ][ 'mTempAppendPath' ] = $this->mTempAppendPath;
 173+ print "did save to $status->value \n";
 174+ }
 175+ return $status;
141176 }else{
142177 //make sure the file exists:
143178 if( is_file( $this->mTempAppendPath ) ){
144179 print "append: " . $this->mTempPath . ' to ' . $this->mTempAppendPath . "\n";
145 - $this->appendToUploadFile( $this->mTempAppendPath, $this->mTempPath );
 180+ $status = $this->appendToUploadFile( $this->mTempAppendPath, $this->mTempPath );
 181+ return $status;
 182+ }else{
 183+ return Status::newFatal('chunk-file-append-missing');
146184 }
147185 }
148 - }
149 -
150 - function checkAPIresultOverride(){
151 - if( $this->chunk_mode == UploadFromChunks::INIT ){
152 - return true;
153 - }else{
154 - return false;
155 - }
156 - }
 186+ }
157187 }
Index: branches/new-upload/phase3/includes/DefaultSettings.php
@@ -437,6 +437,7 @@
438438 */
439439 $wgMaxUploadSize = 1024*1024*100; # 100MB
440440
 441+
441442 /**
442443 * Point the upload navigation link to an external URL
443444 * Useful if you want to use a shared repository by default
Index: branches/new-upload/phase3/includes/UploadBase.php
@@ -100,19 +100,23 @@
101101 function fetchFile() {
102102 return self::OK;
103103 }
104 -
 104+ //return the file size
 105+ function isEmptyFile(){
 106+ return empty( $this->mFileSize);
 107+ }
105108 /**
106109 * Verify whether the upload is sane.
107110 * Returns self::OK or else an array with error information
108111 */
109 - function verifyUpload() {
 112+ function verifyUpload() {
110113 /**
111114 * If there was no filename or a zero size given, give up quick.
112115 */
113 - if( empty( $this->mFileSize ) )
 116+
 117+ if( $this->isEmptyFile() )
114118 return array( 'status' => self::EMPTY_FILE );
115 -
116 - $nt = $this->getTitle();
 119+
 120+ $nt = $this->getTitle();
117121 if( is_null( $nt ) ) {
118122 $result = array( 'status' => $this->mTitleError );
119123 if( $this->mTitleError == self::ILLEGAL_FILENAME )
@@ -122,20 +126,20 @@
123127 return $result;
124128 }
125129 $this->mLocalFile = wfLocalFile( $nt );
126 - $this->mDestName = $this->mLocalFile->getName();
127 -
 130+ $this->mDestName = $this->mLocalFile->getName();
 131+
128132 /**
129133 * In some cases we may forbid overwriting of existing files.
130134 */
131135 $overwrite = $this->checkOverwrite();
132136 if( $overwrite !== true )
133137 return array( 'status' => self::OVERWRITE_EXISTING_FILE, 'overwrite' => $overwrite );
134 -
 138+
135139 /**
136140 * Look at the contents of the file; if we can recognize the
137141 * type but it's corrupt or data of the wrong type, we should
138142 * probably not accept it.
139 - */
 143+ */
140144 $verification = $this->verifyFile( $this->mTempPath );
141145
142146 if( $verification !== true ) {
@@ -423,7 +427,7 @@
424428 $status = $repo->storeTemp( $saveName, $tempName );
425429 return $status;
426430 }
427 - /* append to a stached file */
 431+ /* append to a stashed file */
428432 function appendToUploadFile($srcPath, $toAppendPath ){
429433 $repo = RepoGroup::singleton()->getLocalRepo();
430434 $status = $repo->append($srcPath, $toAppendPath);
@@ -858,11 +862,6 @@
859863 return true;
860864
861865 }
862 - /* allow for getAPIresult override (normally just return UploadFrom::OK to continue form processing */
863 - function getAPIresult(){
864 - return UploadFrom::OK;
865 - }
866 -
867866 /**
868867 * Check if a user is the last uploader
869868 *

Status & tagging log