| Index: branches/REL1_18/phase3/includes/upload/UploadFromStash.php |
| — | — | @@ -9,13 +9,18 @@ |
| 10 | 10 | |
| 11 | 11 | class UploadFromStash extends UploadBase { |
| 12 | 12 | protected $mFileKey, $mVirtualTempPath, $mFileProps, $mSourceType; |
| 13 | | - |
| | 13 | + |
| 14 | 14 | // an instance of UploadStash |
| 15 | 15 | private $stash; |
| 16 | | - |
| | 16 | + |
| 17 | 17 | //LocalFile repo |
| 18 | 18 | private $repo; |
| 19 | | - |
| | 19 | + |
| | 20 | + /** |
| | 21 | + * @param $user User |
| | 22 | + * @param $stash UploadStash |
| | 23 | + * @param $repo FileRepo |
| | 24 | + */ |
| 20 | 25 | public function __construct( $user = false, $stash = false, $repo = false ) { |
| 21 | 26 | // user object. sometimes this won't exist, as when running from cron. |
| 22 | 27 | $this->user = $user; |
| — | — | @@ -29,16 +34,25 @@ |
| 30 | 35 | if( $stash ) { |
| 31 | 36 | $this->stash = $stash; |
| 32 | 37 | } else { |
| 33 | | - wfDebug( __METHOD__ . " creating new UploadStash instance for " . $user->getId() . "\n" ); |
| | 38 | + if( $user ) { |
| | 39 | + wfDebug( __METHOD__ . " creating new UploadStash instance for " . $user->getId() . "\n" ); |
| | 40 | + } else { |
| | 41 | + wfDebug( __METHOD__ . " creating new UploadStash instance with no user\n" ); |
| | 42 | + } |
| | 43 | + |
| 34 | 44 | $this->stash = new UploadStash( $this->repo, $this->user ); |
| 35 | 45 | } |
| 36 | 46 | |
| 37 | 47 | return true; |
| 38 | 48 | } |
| 39 | | - |
| | 49 | + |
| | 50 | + /** |
| | 51 | + * @param $key string |
| | 52 | + * @return bool |
| | 53 | + */ |
| 40 | 54 | public static function isValidKey( $key ) { |
| 41 | 55 | // this is checked in more detail in UploadStash |
| 42 | | - return preg_match( UploadStash::KEY_FORMAT_REGEX, $key ); |
| | 56 | + return (bool)preg_match( UploadStash::KEY_FORMAT_REGEX, $key ); |
| 43 | 57 | } |
| 44 | 58 | |
| 45 | 59 | /** |
| — | — | @@ -47,16 +61,23 @@ |
| 48 | 62 | * @return Boolean |
| 49 | 63 | */ |
| 50 | 64 | public static function isValidRequest( $request ) { |
| 51 | | - return self::isValidKey( $request->getText( 'wpFileKey' ) || $request->getText( 'wpSessionKey' ) ); |
| | 65 | + // this passes wpSessionKey to getText() as a default when wpFileKey isn't set. |
| | 66 | + // wpSessionKey has no default which guarantees failure if both are missing |
| | 67 | + // (though that should have been caught earlier) |
| | 68 | + return self::isValidKey( $request->getText( 'wpFileKey', $request->getText( 'wpSessionKey' ) ) ); |
| 52 | 69 | } |
| 53 | 70 | |
| | 71 | + /** |
| | 72 | + * @param $key string |
| | 73 | + * @param $name string |
| | 74 | + */ |
| 54 | 75 | public function initialize( $key, $name = 'upload_file' ) { |
| 55 | 76 | /** |
| 56 | 77 | * Confirming a temporarily stashed upload. |
| 57 | 78 | * We don't want path names to be forged, so we keep |
| 58 | 79 | * them in the session on the server and just give |
| 59 | 80 | * an opaque key to the user agent. |
| 60 | | - */ |
| | 81 | + */ |
| 61 | 82 | $metadata = $this->stash->getMetadata( $key ); |
| 62 | 83 | $this->initializePathInfo( $name, |
| 63 | 84 | $this->getRealPath ( $metadata['us_path'] ), |
| — | — | @@ -74,17 +95,20 @@ |
| 75 | 96 | * @param $request WebRequest |
| 76 | 97 | */ |
| 77 | 98 | public function initializeFromRequest( &$request ) { |
| 78 | | - $fileKey = $request->getText( 'wpFileKey' ) || $request->getText( 'wpSessionKey' ); |
| | 99 | + // sends wpSessionKey as a default when wpFileKey is missing |
| | 100 | + $fileKey = $request->getText( 'wpFileKey', $request->getText( 'wpSessionKey' ) ); |
| 79 | 101 | |
| 80 | | - $desiredDestName = $request->getText( 'wpDestFile' ); |
| 81 | | - if( !$desiredDestName ) { |
| 82 | | - $desiredDestName = $request->getText( 'wpUploadFile' ) || $request->getText( 'filename' ); |
| 83 | | - } |
| | 102 | + // chooses one of wpDestFile, wpUploadFile, filename in that order. |
| | 103 | + $desiredDestName = $request->getText( 'wpDestFile', $request->getText( 'wpUploadFile', $request->getText( 'filename' ) ) ); |
| | 104 | + |
| 84 | 105 | return $this->initialize( $fileKey, $desiredDestName ); |
| 85 | 106 | } |
| 86 | 107 | |
| 87 | | - public function getSourceType() { |
| 88 | | - return $this->mSourceType; |
| | 108 | + /** |
| | 109 | + * @return string |
| | 110 | + */ |
| | 111 | + public function getSourceType() { |
| | 112 | + return $this->mSourceType; |
| 89 | 113 | } |
| 90 | 114 | |
| 91 | 115 | /** |
| — | — | @@ -97,20 +121,23 @@ |
| 98 | 122 | } |
| 99 | 123 | |
| 100 | 124 | /** |
| 101 | | - * There is no need to stash the image twice |
| | 125 | + * Stash the file. |
| | 126 | + * |
| | 127 | + * @return UploadStashFile |
| 102 | 128 | */ |
| 103 | | - public function stashFile( $key = null ) { |
| 104 | | - if ( !empty( $this->mLocalFile ) ) { |
| 105 | | - return $this->mLocalFile; |
| 106 | | - } |
| 107 | | - return parent::stashFile( $key ); |
| | 129 | + public function stashFile() { |
| | 130 | + // replace mLocalFile with an instance of UploadStashFile, which adds some methods |
| | 131 | + // that are useful for stashed files. |
| | 132 | + $this->mLocalFile = parent::stashFile(); |
| | 133 | + return $this->mLocalFile; |
| 108 | 134 | } |
| 109 | 135 | |
| 110 | 136 | /** |
| 111 | | - * Alias for stashFile |
| | 137 | + * This should return the key instead of the UploadStashFile instance, for backward compatibility. |
| | 138 | + * @return String |
| 112 | 139 | */ |
| 113 | | - public function stashSession( $key = null ) { |
| 114 | | - return $this->stashFile( $key ); |
| | 140 | + public function stashSession() { |
| | 141 | + return $this->stashFile()->getFileKey(); |
| 115 | 142 | } |
| 116 | 143 | |
| 117 | 144 | /** |
| — | — | @@ -123,11 +150,15 @@ |
| 124 | 151 | |
| 125 | 152 | /** |
| 126 | 153 | * Perform the upload, then remove the database record afterward. |
| | 154 | + * @param $comment string |
| | 155 | + * @param $pageText string |
| | 156 | + * @param $watch bool |
| | 157 | + * @param $user User |
| | 158 | + * @return Status |
| 127 | 159 | */ |
| 128 | 160 | public function performUpload( $comment, $pageText, $watch, $user ) { |
| 129 | 161 | $rv = parent::performUpload( $comment, $pageText, $watch, $user ); |
| 130 | 162 | $this->unsaveUploadedFile(); |
| 131 | 163 | return $rv; |
| 132 | 164 | } |
| 133 | | - |
| 134 | | -} |
| \ No newline at end of file |
| | 165 | +} |
| Index: branches/REL1_18/phase3/includes/upload/UploadStash.php |
| — | — | @@ -10,6 +10,9 @@ |
| 11 | 11 | * We accomplish this using a database table, with ownership checking as you might expect. See SpecialUploadStash, which |
| 12 | 12 | * implements a web interface to some files stored this way. |
| 13 | 13 | * |
| | 14 | + * UploadStash right now is *mostly* intended to show you one user's slice of the entire stash. The user parameter is only optional |
| | 15 | + * because there are few cases where we clean out the stash from an automated script. In the future we might refactor this. |
| | 16 | + * |
| 14 | 17 | * UploadStash represents the entire stash of temporary files. |
| 15 | 18 | * UploadStashFile is a filestore for the actual physical disk files. |
| 16 | 19 | * UploadFromStash extends UploadBase, and represents a single stashed file as it is moved from the stash to the regular file repository |
| — | — | @@ -19,13 +22,6 @@ |
| 20 | 23 | // Format of the key for files -- has to be suitable as a filename itself (e.g. ab12cd34ef.jpg) |
| 21 | 24 | const KEY_FORMAT_REGEX = '/^[\w-\.]+\.\w*$/'; |
| 22 | 25 | |
| 23 | | - // When a given stashed file can't be loaded, wait for the slaves to catch up. If they're more than MAX_LAG |
| 24 | | - // behind, throw an exception instead. (at what point is broken better than slow?) |
| 25 | | - const MAX_LAG = 30; |
| 26 | | - |
| 27 | | - // Age of the repository in hours. That is, after how long will files be assumed abandoned and deleted? |
| 28 | | - const REPO_AGE = 6; |
| 29 | | - |
| 30 | 26 | /** |
| 31 | 27 | * repository that this uses to store temp files |
| 32 | 28 | * public because we sometimes need to get a LocalFile within the same repo. |
| — | — | @@ -73,6 +69,7 @@ |
| 74 | 70 | |
| 75 | 71 | /** |
| 76 | 72 | * Get a file and its metadata from the stash. |
| | 73 | + * The noAuth param is a bit janky but is required for automated scripts which clean out the stash. |
| 77 | 74 | * |
| 78 | 75 | * @param $key String: key under which file information is stored |
| 79 | 76 | * @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts. |
| — | — | @@ -94,26 +91,10 @@ |
| 95 | 92 | } |
| 96 | 93 | } |
| 97 | 94 | |
| 98 | | - $dbr = $this->repo->getSlaveDb(); |
| 99 | | - |
| 100 | 95 | if ( !isset( $this->fileMetadata[$key] ) ) { |
| 101 | | - // try this first. if it fails to find the row, check for lag, wait, try again. if its still missing, throw an exception. |
| 102 | | - // this more complex solution keeps things moving for page loads with many requests |
| 103 | | - // (ie. validating image ownership) when replag is high |
| 104 | 96 | if ( !$this->fetchFileMetadata( $key ) ) { |
| 105 | | - $lag = $dbr->getLag(); |
| 106 | | - if ( $lag > 0 && $lag <= self::MAX_LAG ) { |
| 107 | | - // if there's not too much replication lag, just wait for the slave to catch up to our last insert. |
| 108 | | - sleep( ceil( $lag ) ); |
| 109 | | - } elseif ( $lag > self::MAX_LAG ) { |
| 110 | | - // that's a lot of lag to introduce into the middle of the UI. |
| 111 | | - throw new UploadStashMaxLagExceededException( |
| 112 | | - 'Couldn\'t load stashed file metadata, and replication lag is above threshold: (MAX_LAG=' . self::MAX_LAG . ')' |
| 113 | | - ); |
| 114 | | - } |
| 115 | | - |
| 116 | | - // now that the waiting has happened, try again |
| 117 | | - $this->fetchFileMetadata( $key ); |
| | 97 | + // If nothing was received, it's likely due to replication lag. Check the master to see if the record is there. |
| | 98 | + $this->fetchFileMetadata( $key, DB_MASTER ); |
| 118 | 99 | } |
| 119 | 100 | |
| 120 | 101 | if ( !isset( $this->fileMetadata[$key] ) ) { |
| — | — | @@ -172,13 +153,12 @@ |
| 173 | 154 | * |
| 174 | 155 | * @param $path String: path to file you want stashed |
| 175 | 156 | * @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null) |
| 176 | | - * @param $key String: optional, unique key for this file. Used for directory hashing when storing, otherwise not important |
| 177 | 157 | * @throws UploadStashBadPathException |
| 178 | 158 | * @throws UploadStashFileException |
| 179 | 159 | * @throws UploadStashNotLoggedInException |
| 180 | 160 | * @return UploadStashFile: file, or null on failure |
| 181 | 161 | */ |
| 182 | | - public function stashFile( $path, $sourceType = null, $key = null ) { |
| | 162 | + public function stashFile( $path, $sourceType = null ) { |
| 183 | 163 | if ( ! file_exists( $path ) ) { |
| 184 | 164 | wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" ); |
| 185 | 165 | throw new UploadStashBadPathException( "path doesn't exist" ); |
| — | — | @@ -198,17 +178,16 @@ |
| 199 | 179 | } |
| 200 | 180 | |
| 201 | 181 | // If no key was supplied, make one. a mysql insertid would be totally reasonable here, except |
| 202 | | - // that some users of this function might expect to supply the key instead of using the generated one. |
| 203 | | - if ( is_null( $key ) ) { |
| 204 | | - // some things that when combined will make a suitably unique key. |
| 205 | | - // see: http://www.jwz.org/doc/mid.html |
| 206 | | - list ($usec, $sec) = explode( ' ', microtime() ); |
| 207 | | - $usec = substr($usec, 2); |
| 208 | | - $key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' . |
| 209 | | - wfBaseConvert( mt_rand(), 10, 36 ) . '.'. |
| 210 | | - $this->userId . '.' . |
| 211 | | - $extension; |
| 212 | | - } |
| | 182 | + // that for historical reasons, the key is this random thing instead. At least it's not guessable. |
| | 183 | + // |
| | 184 | + // some things that when combined will make a suitably unique key. |
| | 185 | + // see: http://www.jwz.org/doc/mid.html |
| | 186 | + list ($usec, $sec) = explode( ' ', microtime() ); |
| | 187 | + $usec = substr($usec, 2); |
| | 188 | + $key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' . |
| | 189 | + wfBaseConvert( mt_rand(), 10, 36 ) . '.'. |
| | 190 | + $this->userId . '.' . |
| | 191 | + $extension; |
| 213 | 192 | |
| 214 | 193 | $this->fileProps[$key] = $fileProps; |
| 215 | 194 | |
| — | — | @@ -249,31 +228,8 @@ |
| 250 | 229 | wfDebug( __METHOD__ . " inserting $stashPath under $key\n" ); |
| 251 | 230 | $dbw = $this->repo->getMasterDb(); |
| 252 | 231 | |
| 253 | | - // select happens on the master so this can all be in a transaction, which |
| 254 | | - // avoids a race condition that's likely with multiple people uploading from the same |
| 255 | | - // set of files |
| 256 | | - $dbw->begin(); |
| 257 | | - // first, check to see if it's already there. |
| 258 | | - $row = $dbw->selectRow( |
| 259 | | - 'uploadstash', |
| 260 | | - 'us_user, us_timestamp', |
| 261 | | - array( 'us_key' => $key ), |
| 262 | | - __METHOD__ |
| 263 | | - ); |
| 264 | | - |
| 265 | | - // The current user can't have this key if: |
| 266 | | - // - the key is owned by someone else and |
| 267 | | - // - the age of the key is less than REPO_AGE |
| 268 | | - if ( is_object( $row ) ) { |
| 269 | | - if ( $row->us_user != $this->userId && |
| 270 | | - $row->wfTimestamp( TS_UNIX, $row->us_timestamp ) > time() - UploadStash::REPO_AGE * 3600 |
| 271 | | - ) { |
| 272 | | - $dbw->rollback(); |
| 273 | | - throw new UploadStashWrongOwnerException( "Attempting to upload a duplicate of a file that someone else has stashed" ); |
| 274 | | - } |
| 275 | | - } |
| 276 | | - |
| 277 | 232 | $this->fileMetadata[$key] = array( |
| | 233 | + 'us_id' => $dbw->nextSequenceValue( 'uploadstash_us_id_seq' ), |
| 278 | 234 | 'us_user' => $this->userId, |
| 279 | 235 | 'us_key' => $key, |
| 280 | 236 | 'us_orig_path' => $path, |
| — | — | @@ -290,14 +246,11 @@ |
| 291 | 247 | 'us_status' => 'finished' |
| 292 | 248 | ); |
| 293 | 249 | |
| 294 | | - // if a row exists but previous checks on it passed, let the current user take over this key. |
| 295 | | - $dbw->replace( |
| | 250 | + $dbw->insert( |
| 296 | 251 | 'uploadstash', |
| 297 | | - 'us_key', |
| 298 | 252 | $this->fileMetadata[$key], |
| 299 | 253 | __METHOD__ |
| 300 | 254 | ); |
| 301 | | - $dbw->commit(); |
| 302 | 255 | |
| 303 | 256 | // store the insertid in the class variable so immediate retrieval (possibly laggy) isn't necesary. |
| 304 | 257 | $this->fileMetadata[$key]['us_id'] = $dbw->insertId(); |
| — | — | @@ -320,7 +273,7 @@ |
| 321 | 274 | throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' ); |
| 322 | 275 | } |
| 323 | 276 | |
| 324 | | - wfDebug( __METHOD__ . " clearing all rows for user $userId\n" ); |
| | 277 | + wfDebug( __METHOD__ . ' clearing all rows for user ' . $this->userId . "\n" ); |
| 325 | 278 | $dbw = $this->repo->getMasterDb(); |
| 326 | 279 | $dbw->delete( |
| 327 | 280 | 'uploadstash', |
| — | — | @@ -414,7 +367,7 @@ |
| 415 | 368 | $res = $dbr->select( |
| 416 | 369 | 'uploadstash', |
| 417 | 370 | 'us_key', |
| 418 | | - array( 'us_key' => $key ), |
| | 371 | + array( 'us_user' => $this->userId ), |
| 419 | 372 | __METHOD__ |
| 420 | 373 | ); |
| 421 | 374 | |
| — | — | @@ -468,9 +421,16 @@ |
| 469 | 422 | * @param $key String: key |
| 470 | 423 | * @return boolean |
| 471 | 424 | */ |
| 472 | | - protected function fetchFileMetadata( $key ) { |
| | 425 | + protected function fetchFileMetadata( $key, $readFromDB = DB_SLAVE ) { |
| 473 | 426 | // populate $fileMetadata[$key] |
| 474 | | - $dbr = $this->repo->getSlaveDb(); |
| | 427 | + $dbr = null; |
| | 428 | + if( $readFromDB === DB_MASTER ) { |
| | 429 | + // sometimes reading from the master is necessary, if there's replication lag. |
| | 430 | + $dbr = $this->repo->getMasterDb(); |
| | 431 | + } else { |
| | 432 | + $dbr = $this->repo->getSlaveDb(); |
| | 433 | + } |
| | 434 | + |
| 475 | 435 | $row = $dbr->selectRow( |
| 476 | 436 | 'uploadstash', |
| 477 | 437 | '*', |
| — | — | @@ -483,22 +443,7 @@ |
| 484 | 444 | return false; |
| 485 | 445 | } |
| 486 | 446 | |
| 487 | | - $this->fileMetadata[$key] = array( |
| 488 | | - 'us_user' => $row->us_user, |
| 489 | | - 'us_key' => $row->us_key, |
| 490 | | - 'us_orig_path' => $row->us_orig_path, |
| 491 | | - 'us_path' => $row->us_path, |
| 492 | | - 'us_size' => $row->us_size, |
| 493 | | - 'us_sha1' => $row->us_sha1, |
| 494 | | - 'us_mime' => $row->us_mime, |
| 495 | | - 'us_media_type' => $row->us_media_type, |
| 496 | | - 'us_image_width' => $row->us_image_width, |
| 497 | | - 'us_image_height' => $row->us_image_height, |
| 498 | | - 'us_image_bits' => $row->us_image_bits, |
| 499 | | - 'us_source_type' => $row->us_source_type, |
| 500 | | - 'us_timestamp' => $row->us_timestamp, |
| 501 | | - 'us_status' => $row->us_status |
| 502 | | - ); |
| | 447 | + $this->fileMetadata[$key] = (array)$row; |
| 503 | 448 | |
| 504 | 449 | return true; |
| 505 | 450 | } |
| — | — | @@ -698,5 +643,4 @@ |
| 699 | 644 | class UploadStashZeroLengthFileException extends MWException {}; |
| 700 | 645 | class UploadStashNotLoggedInException extends MWException {}; |
| 701 | 646 | class UploadStashWrongOwnerException extends MWException {}; |
| 702 | | -class UploadStashMaxLagExceededException extends MWException {}; |
| 703 | 647 | class UploadStashNoSuchKeyException extends MWException {}; |
| Property changes on: branches/REL1_18/phase3/includes/upload/UploadStash.php |
| ___________________________________________________________________ |
| Modified: svn:mergeinfo |
| 704 | 648 | Reverse-merged /trunk/phase3/includes/upload/UploadStash.php:r92580,92713,92765,92884,92886-92887,92894,92898,92907,92932,93065,93149,93151,93233-93234,93258,93266,93516-93518,93818-93822,93834,93847,93858,93891,93935-93936,94068,94155,94235,94346,94372,94422,94425,94444,94448,94456,94498,94601,94728,94825,94862,94995-94997 |