| Index: trunk/phase3/maintenance/language/messages.inc |
| — | — | @@ -1367,7 +1367,8 @@ |
| 1368 | 1368 | 'backend-fail-synced', |
| 1369 | 1369 | 'backend-fail-connect', |
| 1370 | 1370 | 'backend-fail-internal', |
| 1371 | | - 'backend-fail-contenttype' |
| | 1371 | + 'backend-fail-contenttype', |
| | 1372 | + 'backend-fail-batchsize' |
| 1372 | 1373 | ), |
| 1373 | 1374 | |
| 1374 | 1375 | 'lockmanager-errors' => array( |
| Index: trunk/phase3/includes/filerepo/backend/FileOp.php |
| — | — | @@ -34,6 +34,10 @@ |
| 35 | 35 | const STATE_CHECKED = 2; |
| 36 | 36 | const STATE_ATTEMPTED = 3; |
| 37 | 37 | |
| | 38 | + /* Timeout related parameters */ |
| | 39 | + const MAX_BATCH_SIZE = 1000; |
| | 40 | + const TIME_LIMIT_SEC = 300; // 5 minutes |
| | 41 | + |
| 38 | 42 | /** |
| 39 | 43 | * Build a new file operation transaction |
| 40 | 44 | * |
| — | — | @@ -52,6 +56,10 @@ |
| 53 | 57 | |
| 54 | 58 | /** |
| 55 | 59 | * Allow stale data for file reads and existence checks. |
| | 60 | + * |
| | 61 | + * Note that we don't want to mix stale and non-stale reads |
| | 62 | + * because stat calls are cached: if we read X without 'latest' |
| | 63 | + * and then read it with 'latest', the data may still be stale. |
| 56 | 64 | * |
| 57 | 65 | * @return void |
| 58 | 66 | */ |
| — | — | @@ -80,6 +88,12 @@ |
| 81 | 89 | $allowStale = !empty( $opts['allowStale'] ); |
| 82 | 90 | $ignoreErrors = !empty( $opts['force'] ); |
| 83 | 91 | |
| | 92 | + $n = count( $performOps ); |
| | 93 | + if ( $n > self::MAX_BATCH_SIZE ) { |
| | 94 | + $status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE ); |
| | 95 | + return $status; |
| | 96 | + } |
| | 97 | + |
| 84 | 98 | $predicates = FileOp::newPredicates(); // account for previous op in prechecks |
| 85 | 99 | // Do pre-checks for each operation; abort on failure... |
| 86 | 100 | foreach ( $performOps as $index => $fileOp ) { |
| — | — | @@ -97,6 +111,11 @@ |
| 98 | 112 | } |
| 99 | 113 | } |
| 100 | 114 | |
| | 115 | + // Restart PHP's execution timer and set the timeout to safe amount. |
| | 116 | + // This handles cases where the operations take a long time or where we are |
| | 117 | + // already running low on time left. The old timeout is restored afterwards. |
| | 118 | + $scopedTimeLimit = new FileOpScopedPHPTimeout( self::TIME_LIMIT_SEC ); |
| | 119 | + |
| 101 | 120 | // Attempt each operation... |
| 102 | 121 | foreach ( $performOps as $index => $fileOp ) { |
| 103 | 122 | if ( $fileOp->failed() ) { |
| — | — | @@ -326,6 +345,39 @@ |
| 327 | 346 | } |
| 328 | 347 | |
| 329 | 348 | /** |
| | 349 | + * FileOp helper class to expand PHP execution time for a function. |
| | 350 | + * On construction, set_time_limit() is called and set to $seconds. |
| | 351 | + * When the object goes out of scope, the timer is restarted, with |
| | 352 | + * the original time limit minus the time the object existed. |
| | 353 | + */ |
| | 354 | +class FileOpScopedPHPTimeout { |
| | 355 | + protected $startTime; // integer seconds |
| | 356 | + protected $oldTimeout; // integer seconds |
| | 357 | + |
| | 358 | + /** |
| | 359 | + * @param $seconds integer |
| | 360 | + */ |
| | 361 | + public function __construct( $seconds ) { |
| | 362 | + if ( ini_get( 'max_execution_time' ) > 0 ) { // CLI uses 0 |
| | 363 | + $this->oldTimeout = ini_set( 'max_execution_time', $seconds ); |
| | 364 | + } |
| | 365 | + $this->startTime = time(); |
| | 366 | + } |
| | 367 | + |
| | 368 | + /* |
| | 369 | + * Restore the original timeout. |
| | 370 | + * This does not account for the timer value on __construct(). |
| | 371 | + */ |
| | 372 | + public function __destruct() { |
| | 373 | + if ( $this->oldTimeout ) { |
| | 374 | + $elapsed = time() - $this->startTime; |
| | 375 | + // Note: a limit of 0 is treated as "forever" |
| | 376 | + set_time_limit( max( 1, $this->oldTimeout - $elapsed ) ); |
| | 377 | + } |
| | 378 | + } |
| | 379 | +} |
| | 380 | + |
| | 381 | +/** |
| 330 | 382 | * Store a file into the backend from a file on the file system. |
| 331 | 383 | * Parameters similar to FileBackend::storeInternal(), which include: |
| 332 | 384 | * src : source path on file system |
| — | — | @@ -345,6 +397,11 @@ |
| 346 | 398 | $status->fatal( 'backend-fail-notexists', $this->params['src'] ); |
| 347 | 399 | return $status; |
| 348 | 400 | } |
| | 401 | + // Check if the source file is too big |
| | 402 | + if ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) { |
| | 403 | + $status->fatal( 'backend-fail-store', $this->params['dst'] ); |
| | 404 | + return $status; |
| | 405 | + } |
| 349 | 406 | // Check if destination file exists |
| 350 | 407 | $status->merge( $this->precheckDestExistence( $predicates ) ); |
| 351 | 408 | if ( !$status->isOK() ) { |
| — | — | @@ -395,6 +452,11 @@ |
| 396 | 453 | |
| 397 | 454 | protected function doPrecheck( array &$predicates ) { |
| 398 | 455 | $status = Status::newGood(); |
| | 456 | + // Check if the source data is too big |
| | 457 | + if ( strlen( $this->params['content'] ) > $this->backend->maxFileSizeInternal() ) { |
| | 458 | + $status->fatal( 'backend-fail-create', $this->params['dst'] ); |
| | 459 | + return $status; |
| | 460 | + } |
| 399 | 461 | // Check if destination file exists |
| 400 | 462 | $status->merge( $this->precheckDestExistence( $predicates ) ); |
| 401 | 463 | if ( !$status->isOK() ) { |
| Index: trunk/phase3/includes/filerepo/backend/FileBackend.php |
| — | — | @@ -592,7 +592,20 @@ |
| 593 | 593 | /** @var Array */ |
| 594 | 594 | protected $shardViaHashLevels = array(); // (container name => integer) |
| 595 | 595 | |
| | 596 | + protected $maxFileSize = 1000000000; // integer bytes (1GB) |
| | 597 | + |
| 596 | 598 | /** |
| | 599 | + * Get the maximum allowable file size given backend |
| | 600 | + * medium restrictions and basic performance constraints. |
| | 601 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 602 | + * |
| | 603 | + * @return integer Bytes |
| | 604 | + */ |
| | 605 | + final public function maxFileSizeInternal() { |
| | 606 | + return $this->maxFileSize; |
| | 607 | + } |
| | 608 | + |
| | 609 | + /** |
| 597 | 610 | * Create a file in the backend with the given contents. |
| 598 | 611 | * Do not call this function from places outside FileBackend and FileOp. |
| 599 | 612 | * |
| — | — | @@ -605,8 +618,12 @@ |
| 606 | 619 | * @return Status |
| 607 | 620 | */ |
| 608 | 621 | final public function createInternal( array $params ) { |
| 609 | | - $status = $this->doCreateInternal( $params ); |
| 610 | | - $this->clearCache( array( $params['dst'] ) ); |
| | 622 | + if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { |
| | 623 | + $status = Status::newFatal( 'backend-fail-create', $params['dst'] ); |
| | 624 | + } else { |
| | 625 | + $status = $this->doCreateInternal( $params ); |
| | 626 | + $this->clearCache( array( $params['dst'] ) ); |
| | 627 | + } |
| 611 | 628 | return $status; |
| 612 | 629 | } |
| 613 | 630 | |
| — | — | @@ -628,8 +645,12 @@ |
| 629 | 646 | * @return Status |
| 630 | 647 | */ |
| 631 | 648 | final public function storeInternal( array $params ) { |
| 632 | | - $status = $this->doStoreInternal( $params ); |
| 633 | | - $this->clearCache( array( $params['dst'] ) ); |
| | 649 | + if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { |
| | 650 | + $status = Status::newFatal( 'backend-fail-store', $params['dst'] ); |
| | 651 | + } else { |
| | 652 | + $status = $this->doStoreInternal( $params ); |
| | 653 | + $this->clearCache( array( $params['dst'] ) ); |
| | 654 | + } |
| 634 | 655 | return $status; |
| 635 | 656 | } |
| 636 | 657 | |
| Index: trunk/phase3/includes/AutoLoader.php |
| — | — | @@ -511,6 +511,7 @@ |
| 512 | 512 | 'MySqlLockManager'=> 'includes/filerepo/backend/lockmanager/DBLockManager.php', |
| 513 | 513 | 'NullLockManager' => 'includes/filerepo/backend/lockmanager/LockManager.php', |
| 514 | 514 | 'FileOp' => 'includes/filerepo/backend/FileOp.php', |
| | 515 | + 'FileOpScopedPHPTimeout' => 'includes/filerepo/backend/FileOp.php', |
| 515 | 516 | 'StoreFileOp' => 'includes/filerepo/backend/FileOp.php', |
| 516 | 517 | 'CopyFileOp' => 'includes/filerepo/backend/FileOp.php', |
| 517 | 518 | 'MoveFileOp' => 'includes/filerepo/backend/FileOp.php', |
| Index: trunk/phase3/languages/messages/MessagesEn.php |
| — | — | @@ -2258,6 +2258,7 @@ |
| 2259 | 2259 | 'backend-fail-connect' => 'Could not connect to file backend "$1".', |
| 2260 | 2260 | 'backend-fail-internal' => 'An unknown error occurred in file backend "$1".', |
| 2261 | 2261 | 'backend-fail-contenttype' => 'Could not determine the content type of file to store at "$1".', |
| | 2262 | +'backend-fail-batchsize' => 'Backend given a batch of $1 file {{PLURAL:$1|operation|operations}}; the limit is $2.', |
| 2262 | 2263 | |
| 2263 | 2264 | # Lock manager |
| 2264 | 2265 | 'lockmanager-notlocked' => 'Could not unlock "$1"; it is not locked.', |