| Index: trunk/phase3/maintenance/cleanupUploadStash.php |
| — | — | @@ -0,0 +1,74 @@ |
| | 2 | +<?php |
| | 3 | +/** |
| | 4 | + * Remove old or broken uploads from temporary uploaded file storage, |
| | 5 | + * clean up associated database records |
| | 6 | + * |
| | 7 | + * Copyright © 2011, Wikimedia Foundation |
| | 8 | + * |
| | 9 | + * This program is free software; you can redistribute it and/or modify |
| | 10 | + * it under the terms of the GNU General Public License as published by |
| | 11 | + * the Free Software Foundation; either version 2 of the License, or |
| | 12 | + * (at your option) any later version. |
| | 13 | + * |
| | 14 | + * This program is distributed in the hope that it will be useful, |
| | 15 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| | 16 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| | 17 | + * GNU General Public License for more details. |
| | 18 | + * |
| | 19 | + * You should have received a copy of the GNU General Public License along |
| | 20 | + * with this program; if not, write to the Free Software Foundation, Inc., |
| | 21 | + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
| | 22 | + * http://www.gnu.org/copyleft/gpl.html |
| | 23 | + * |
| | 24 | + * @file |
| | 25 | + * @author Ian Baker <ibaker@wikimedia.org> |
| | 26 | + * @ingroup Maintenance |
| | 27 | + */ |
| | 28 | + |
| | 29 | +require_once( dirname( __FILE__ ) . '/Maintenance.php' ); |
| | 30 | + |
| | 31 | +class UploadStashCleanup extends Maintenance { |
| | 32 | + |
| | 33 | + public function __construct() { |
| | 34 | + parent::__construct(); |
| | 35 | + $this->mDescription = "Clean up abandoned files in temporary uploaded file stash"; |
| | 36 | + } |
| | 37 | + |
| | 38 | + public function execute() { |
| | 39 | + $dbr = wfGetDB( DB_SLAVE ); |
| | 40 | + |
| | 41 | + $this->output( "Getting list of files to clean up...\n" ); |
| | 42 | + $res = $dbr->select( |
| | 43 | + 'uploadstash', |
| | 44 | + 'us_key', |
| | 45 | + 'us_timestamp < ' . wfTimestamp( TS_MW, time() - UploadStash::REPO_AGE * 3600 ), |
| | 46 | + __METHOD__ |
| | 47 | + ); |
| | 48 | + |
| | 49 | + if( !is_object( $res ) ) { |
| | 50 | + // nothing to do. |
| | 51 | + return false; |
| | 52 | + } |
| | 53 | + |
| | 54 | + // finish the read before starting writes. |
| | 55 | + $keys = array(); |
| | 56 | + while( $row = $dbr->fetchRow( $res ) ) { |
| | 57 | + array_push( $keys, $row['us_key'] ); |
| | 58 | + } |
| | 59 | + |
| | 60 | + $this->output( 'Removing ' . count($keys) . " file(s)...\n" ); |
| | 61 | + // this could be done some other, more direct/efficient way, but using |
| | 62 | + // UploadStash's own methods means it's less likely to fall accidentally |
| | 63 | + // out-of-date someday |
| | 64 | + $repo = RepoGroup::singleton()->getLocalRepo(); |
| | 65 | + $stash = new UploadStash( $repo ); |
| | 66 | + |
| | 67 | + foreach( $keys as $key ) { |
| | 68 | + $stash->getFile( $key, true ); |
| | 69 | + $stash->removeFileNoAuth( $key ); |
| | 70 | + } |
| | 71 | + } |
| | 72 | +} |
| | 73 | + |
| | 74 | +$maintClass = "UploadStashCleanup"; |
| | 75 | +require_once( RUN_MAINTENANCE_IF_MAIN ); |
| \ No newline at end of file |
| Property changes on: trunk/phase3/maintenance/cleanupUploadStash.php |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| 1 | 76 | + native |
| Index: trunk/phase3/includes/upload/UploadStash.php |
| — | — | @@ -23,6 +23,9 @@ |
| 24 | 24 | // behind, throw an exception instead. (at what point is broken better than slow?) |
| 25 | 25 | const MAX_LAG = 30; |
| 26 | 26 | |
| | 27 | + // Age of the repository in hours. That is, after how long will files be assumed abandoned and deleted? |
| | 28 | + const REPO_AGE = 6; |
| | 29 | + |
| 27 | 30 | /** |
| 28 | 31 | * repository that this uses to store temp files |
| 29 | 32 | * public because we sometimes need to get a LocalFile within the same repo. |
| — | — | @@ -55,24 +58,27 @@ |
| 56 | 59 | * Get a file and its metadata from the stash. |
| 57 | 60 | * |
| 58 | 61 | * @param $key String: key under which file information is stored |
| | 62 | + * @param $noauth Boolean (optional) Don't check authentication. Used by maintenance scripts. |
| 59 | 63 | * @throws UploadStashFileNotFoundException |
| 60 | 64 | * @throws UploadStashNotLoggedInException |
| 61 | 65 | * @throws UploadStashWrongOwnerException |
| 62 | 66 | * @throws UploadStashBadPathException |
| 63 | 67 | * @return UploadStashFile |
| 64 | 68 | */ |
| 65 | | - public function getFile( $key ) { |
| | 69 | + public function getFile( $key, $noAuth = false ) { |
| 66 | 70 | global $wgUser; |
| 67 | 71 | |
| 68 | 72 | if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) { |
| 69 | 73 | throw new UploadStashBadPathException( "key '$key' is not in a proper format" ); |
| 70 | 74 | } |
| 71 | | - |
| 72 | | - $userId = $wgUser->getId(); |
| 73 | | - if( !$userId ) { |
| 74 | | - throw new UploadStashNotLoggedInException( 'No user is logged in, files must belong to users' ); |
| | 75 | + |
| | 76 | + if( !$noAuth ) { |
| | 77 | + $userId = $wgUser->getId(); |
| | 78 | + if( !$userId ) { |
| | 79 | + throw new UploadStashNotLoggedInException( 'No user is logged in, files must belong to users' ); |
| | 80 | + } |
| 75 | 81 | } |
| 76 | | - |
| | 82 | + |
| 77 | 83 | if ( !isset( $this->fileMetadata[$key] ) ) { |
| 78 | 84 | // try this first. if it fails to find the row, check for lag, wait, try again. if its still missing, throw an exception. |
| 79 | 85 | // this more complex solution keeps things moving for page loads with many requests |
| — | — | @@ -108,10 +114,12 @@ |
| 109 | 115 | $this->fileProps[$key] = File::getPropsFromPath( $path ); |
| 110 | 116 | } |
| 111 | 117 | |
| 112 | | - if( $this->fileMetadata[$key]['us_user'] != $userId ) { |
| 113 | | - throw new UploadStashWrongOwnerException( "This file ($key) doesn't belong to the current user." ); |
| | 118 | + if( !$noAuth ) { |
| | 119 | + if( $this->fileMetadata[$key]['us_user'] != $userId ) { |
| | 120 | + throw new UploadStashWrongOwnerException( "This file ($key) doesn't belong to the current user." ); |
| | 121 | + } |
| 114 | 122 | } |
| 115 | | - |
| | 123 | + |
| 116 | 124 | return $this->files[$key]; |
| 117 | 125 | } |
| 118 | 126 | |
| — | — | @@ -275,14 +283,14 @@ |
| 276 | 284 | return true; |
| 277 | 285 | } |
| 278 | 286 | |
| 279 | | - |
| 280 | 287 | /** |
| 281 | 288 | * Remove a particular file from the stash. Also removes it from the repo. |
| 282 | 289 | * |
| 283 | 290 | * @throws UploadStashNotLoggedInException |
| | 291 | + * @throws UploadStashWrongOwnerException |
| 284 | 292 | * @return boolean: success |
| 285 | 293 | */ |
| 286 | | - public function removeFile( $key ) { |
| | 294 | + public function removeFile( $key ){ |
| 287 | 295 | global $wgUser; |
| 288 | 296 | |
| 289 | 297 | $userId = $wgUser->getId(); |
| — | — | @@ -290,7 +298,6 @@ |
| 291 | 299 | throw new UploadStashNotLoggedInException( 'No user is logged in, files must belong to users' ); |
| 292 | 300 | } |
| 293 | 301 | |
| 294 | | - wfDebug( __METHOD__ . " clearing row $key for user $userId\n" ); |
| 295 | 302 | $dbw = wfGetDB( DB_MASTER ); |
| 296 | 303 | |
| 297 | 304 | // this is a cheap query. it runs on the master so that this function still works when there's lag. |
| — | — | @@ -306,11 +313,28 @@ |
| 307 | 314 | throw new UploadStashWrongOwnerException( "Can't delete: the file ($key) doesn't belong to this user." ); |
| 308 | 315 | } |
| 309 | 316 | |
| | 317 | + return $this->removeFileNoAuth( $key ); |
| | 318 | + } |
| | 319 | + |
| | 320 | + |
| | 321 | + /** |
| | 322 | + * Remove a file (see removeFile), but doesn't check ownership first. |
| | 323 | + * |
| | 324 | + * @return boolean: success |
| | 325 | + */ |
| | 326 | + public function removeFileNoAuth( $key ) { |
| | 327 | + wfDebug( __METHOD__ . " clearing row $key\n" ); |
| | 328 | + |
| | 329 | + $dbw = wfGetDB( DB_MASTER ); |
| | 330 | + |
| | 331 | + // this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script |
| | 332 | + $dbw->begin(); |
| 310 | 333 | $dbw->delete( |
| 311 | 334 | 'uploadstash', |
| 312 | | - array( 'us_key' => $key, 'us_user' => $userId ), |
| | 335 | + array( 'us_key' => $key), |
| 313 | 336 | __METHOD__ |
| 314 | 337 | ); |
| | 338 | + $dbw->commit(); |
| 315 | 339 | |
| 316 | 340 | // TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed) |
| 317 | 341 | // for now, ignore. |