| Index: trunk/phase3/includes/filerepo/backend/FileBackendStore.php |
| — | — | @@ -0,0 +1,1060 @@ |
| | 2 | +<?php |
| | 3 | +/** |
| | 4 | + * @file |
| | 5 | + * @ingroup FileBackend |
| | 6 | + * @author Aaron Schulz |
| | 7 | + */ |
| | 8 | + |
| | 9 | +/** |
| | 10 | + * @brief Base class for all backends using particular storage medium. |
| | 11 | + * |
| | 12 | + * This class defines the methods as abstract that subclasses must implement. |
| | 13 | + * Outside callers should *not* use functions with "Internal" in the name. |
| | 14 | + * |
| | 15 | + * The FileBackend operations are implemented using basic functions |
| | 16 | + * such as storeInternal(), copyInternal(), deleteInternal() and the like. |
| | 17 | + * This class is also responsible for path resolution and sanitization. |
| | 18 | + * |
| | 19 | + * @ingroup FileBackend |
| | 20 | + * @since 1.19 |
| | 21 | + */ |
| | 22 | +abstract class FileBackendStore extends FileBackend { |
| | 23 | + /** @var Array Map of paths to small (RAM/disk) cache items */ |
| | 24 | + protected $cache = array(); // (storage path => key => value) |
| | 25 | + protected $maxCacheSize = 100; // integer; max paths with entries |
| | 26 | + /** @var Array Map of paths to large (RAM/disk) cache items */ |
| | 27 | + protected $expensiveCache = array(); // (storage path => key => value) |
| | 28 | + protected $maxExpensiveCacheSize = 10; // integer; max paths with entries |
| | 29 | + |
| | 30 | + /** @var Array Map of container names to sharding settings */ |
| | 31 | + protected $shardViaHashLevels = array(); // (container name => config array) |
| | 32 | + |
| | 33 | + protected $maxFileSize = 1000000000; // integer bytes (1GB) |
| | 34 | + |
| | 35 | + /** |
| | 36 | + * Get the maximum allowable file size given backend |
| | 37 | + * medium restrictions and basic performance constraints. |
| | 38 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 39 | + * |
| | 40 | + * @return integer Bytes |
| | 41 | + */ |
| | 42 | + final public function maxFileSizeInternal() { |
| | 43 | + return $this->maxFileSize; |
| | 44 | + } |
| | 45 | + |
| | 46 | + /** |
| | 47 | + * Check if a file can be created at a given storage path. |
| | 48 | + * FS backends should check if the parent directory exists and the file is writable. |
| | 49 | + * Backends using key/value stores should check if the container exists. |
| | 50 | + * |
| | 51 | + * @param $storagePath string |
| | 52 | + * @return bool |
| | 53 | + */ |
| | 54 | + abstract public function isPathUsableInternal( $storagePath ); |
| | 55 | + |
| | 56 | + /** |
| | 57 | + * Create a file in the backend with the given contents. |
| | 58 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 59 | + * |
| | 60 | + * $params include: |
| | 61 | + * content : the raw file contents |
| | 62 | + * dst : destination storage path |
| | 63 | + * overwrite : overwrite any file that exists at the destination |
| | 64 | + * |
| | 65 | + * @param $params Array |
| | 66 | + * @return Status |
| | 67 | + */ |
| | 68 | + final public function createInternal( array $params ) { |
| | 69 | + wfProfileIn( __METHOD__ ); |
| | 70 | + if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { |
| | 71 | + $status = Status::newFatal( 'backend-fail-create', $params['dst'] ); |
| | 72 | + } else { |
| | 73 | + $status = $this->doCreateInternal( $params ); |
| | 74 | + $this->clearCache( array( $params['dst'] ) ); |
| | 75 | + } |
| | 76 | + wfProfileOut( __METHOD__ ); |
| | 77 | + return $status; |
| | 78 | + } |
| | 79 | + |
| | 80 | + /** |
| | 81 | + * @see FileBackendStore::createInternal() |
| | 82 | + */ |
| | 83 | + abstract protected function doCreateInternal( array $params ); |
| | 84 | + |
| | 85 | + /** |
| | 86 | + * Store a file into the backend from a file on disk. |
| | 87 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 88 | + * |
| | 89 | + * $params include: |
| | 90 | + * src : source path on disk |
| | 91 | + * dst : destination storage path |
| | 92 | + * overwrite : overwrite any file that exists at the destination |
| | 93 | + * |
| | 94 | + * @param $params Array |
| | 95 | + * @return Status |
| | 96 | + */ |
| | 97 | + final public function storeInternal( array $params ) { |
| | 98 | + wfProfileIn( __METHOD__ ); |
| | 99 | + if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { |
| | 100 | + $status = Status::newFatal( 'backend-fail-store', $params['dst'] ); |
| | 101 | + } else { |
| | 102 | + $status = $this->doStoreInternal( $params ); |
| | 103 | + $this->clearCache( array( $params['dst'] ) ); |
| | 104 | + } |
| | 105 | + wfProfileOut( __METHOD__ ); |
| | 106 | + return $status; |
| | 107 | + } |
| | 108 | + |
| | 109 | + /** |
| | 110 | + * @see FileBackendStore::storeInternal() |
| | 111 | + */ |
| | 112 | + abstract protected function doStoreInternal( array $params ); |
| | 113 | + |
| | 114 | + /** |
| | 115 | + * Copy a file from one storage path to another in the backend. |
| | 116 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 117 | + * |
| | 118 | + * $params include: |
| | 119 | + * src : source storage path |
| | 120 | + * dst : destination storage path |
| | 121 | + * overwrite : overwrite any file that exists at the destination |
| | 122 | + * |
| | 123 | + * @param $params Array |
| | 124 | + * @return Status |
| | 125 | + */ |
| | 126 | + final public function copyInternal( array $params ) { |
| | 127 | + wfProfileIn( __METHOD__ ); |
| | 128 | + $status = $this->doCopyInternal( $params ); |
| | 129 | + $this->clearCache( array( $params['dst'] ) ); |
| | 130 | + wfProfileOut( __METHOD__ ); |
| | 131 | + return $status; |
| | 132 | + } |
| | 133 | + |
| | 134 | + /** |
| | 135 | + * @see FileBackendStore::copyInternal() |
| | 136 | + */ |
| | 137 | + abstract protected function doCopyInternal( array $params ); |
| | 138 | + |
| | 139 | + /** |
| | 140 | + * Delete a file at the storage path. |
| | 141 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 142 | + * |
| | 143 | + * $params include: |
| | 144 | + * src : source storage path |
| | 145 | + * ignoreMissingSource : do nothing if the source file does not exist |
| | 146 | + * |
| | 147 | + * @param $params Array |
| | 148 | + * @return Status |
| | 149 | + */ |
| | 150 | + final public function deleteInternal( array $params ) { |
| | 151 | + wfProfileIn( __METHOD__ ); |
| | 152 | + $status = $this->doDeleteInternal( $params ); |
| | 153 | + $this->clearCache( array( $params['src'] ) ); |
| | 154 | + wfProfileOut( __METHOD__ ); |
| | 155 | + return $status; |
| | 156 | + } |
| | 157 | + |
| | 158 | + /** |
| | 159 | + * @see FileBackendStore::deleteInternal() |
| | 160 | + */ |
| | 161 | + abstract protected function doDeleteInternal( array $params ); |
| | 162 | + |
| | 163 | + /** |
| | 164 | + * Move a file from one storage path to another in the backend. |
| | 165 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 166 | + * |
| | 167 | + * $params include: |
| | 168 | + * src : source storage path |
| | 169 | + * dst : destination storage path |
| | 170 | + * overwrite : overwrite any file that exists at the destination |
| | 171 | + * |
| | 172 | + * @param $params Array |
| | 173 | + * @return Status |
| | 174 | + */ |
| | 175 | + final public function moveInternal( array $params ) { |
| | 176 | + wfProfileIn( __METHOD__ ); |
| | 177 | + $status = $this->doMoveInternal( $params ); |
| | 178 | + $this->clearCache( array( $params['src'], $params['dst'] ) ); |
| | 179 | + wfProfileOut( __METHOD__ ); |
| | 180 | + return $status; |
| | 181 | + } |
| | 182 | + |
| | 183 | + /** |
| | 184 | + * @see FileBackendStore::moveInternal() |
| | 185 | + * @return Status |
| | 186 | + */ |
| | 187 | + protected function doMoveInternal( array $params ) { |
| | 188 | + // Copy source to dest |
| | 189 | + $status = $this->copyInternal( $params ); |
| | 190 | + if ( $status->isOK() ) { |
| | 191 | + // Delete source (only fails due to races or medium going down) |
| | 192 | + $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); |
| | 193 | + $status->setResult( true, $status->value ); // ignore delete() errors |
| | 194 | + } |
| | 195 | + return $status; |
| | 196 | + } |
| | 197 | + |
| | 198 | + /** |
| | 199 | + * @see FileBackend::concatenate() |
| | 200 | + * @return Status |
| | 201 | + */ |
| | 202 | + final public function concatenate( array $params ) { |
| | 203 | + wfProfileIn( __METHOD__ ); |
| | 204 | + $status = Status::newGood(); |
| | 205 | + |
| | 206 | + // Try to lock the source files for the scope of this function |
| | 207 | + $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); |
| | 208 | + if ( $status->isOK() ) { |
| | 209 | + // Actually do the concatenation |
| | 210 | + $status->merge( $this->doConcatenate( $params ) ); |
| | 211 | + } |
| | 212 | + |
| | 213 | + wfProfileOut( __METHOD__ ); |
| | 214 | + return $status; |
| | 215 | + } |
| | 216 | + |
| | 217 | + /** |
| | 218 | + * @see FileBackendStore::concatenate() |
| | 219 | + * @return Status |
| | 220 | + */ |
| | 221 | + protected function doConcatenate( array $params ) { |
| | 222 | + $status = Status::newGood(); |
| | 223 | + $tmpPath = $params['dst']; // convenience |
| | 224 | + |
| | 225 | + // Check that the specified temp file is valid... |
| | 226 | + wfSuppressWarnings(); |
| | 227 | + $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) ); |
| | 228 | + wfRestoreWarnings(); |
| | 229 | + if ( !$ok ) { // not present or not empty |
| | 230 | + $status->fatal( 'backend-fail-opentemp', $tmpPath ); |
| | 231 | + return $status; |
| | 232 | + } |
| | 233 | + |
| | 234 | + // Build up the temp file using the source chunks (in order)... |
| | 235 | + $tmpHandle = fopen( $tmpPath, 'ab' ); |
| | 236 | + if ( $tmpHandle === false ) { |
| | 237 | + $status->fatal( 'backend-fail-opentemp', $tmpPath ); |
| | 238 | + return $status; |
| | 239 | + } |
| | 240 | + foreach ( $params['srcs'] as $virtualSource ) { |
| | 241 | + // Get a local FS version of the chunk |
| | 242 | + $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) ); |
| | 243 | + if ( !$tmpFile ) { |
| | 244 | + $status->fatal( 'backend-fail-read', $virtualSource ); |
| | 245 | + return $status; |
| | 246 | + } |
| | 247 | + // Get a handle to the local FS version |
| | 248 | + $sourceHandle = fopen( $tmpFile->getPath(), 'r' ); |
| | 249 | + if ( $sourceHandle === false ) { |
| | 250 | + fclose( $tmpHandle ); |
| | 251 | + $status->fatal( 'backend-fail-read', $virtualSource ); |
| | 252 | + return $status; |
| | 253 | + } |
| | 254 | + // Append chunk to file (pass chunk size to avoid magic quotes) |
| | 255 | + if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { |
| | 256 | + fclose( $sourceHandle ); |
| | 257 | + fclose( $tmpHandle ); |
| | 258 | + $status->fatal( 'backend-fail-writetemp', $tmpPath ); |
| | 259 | + return $status; |
| | 260 | + } |
| | 261 | + fclose( $sourceHandle ); |
| | 262 | + } |
| | 263 | + if ( !fclose( $tmpHandle ) ) { |
| | 264 | + $status->fatal( 'backend-fail-closetemp', $tmpPath ); |
| | 265 | + return $status; |
| | 266 | + } |
| | 267 | + |
| | 268 | + clearstatcache(); // temp file changed |
| | 269 | + |
| | 270 | + return $status; |
| | 271 | + } |
| | 272 | + |
| | 273 | + /** |
| | 274 | + * @see FileBackend::doPrepare() |
| | 275 | + * @return Status |
| | 276 | + */ |
| | 277 | + final protected function doPrepare( array $params ) { |
| | 278 | + wfProfileIn( __METHOD__ ); |
| | 279 | + |
| | 280 | + $status = Status::newGood(); |
| | 281 | + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| | 282 | + if ( $dir === null ) { |
| | 283 | + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| | 284 | + wfProfileOut( __METHOD__ ); |
| | 285 | + return $status; // invalid storage path |
| | 286 | + } |
| | 287 | + |
| | 288 | + if ( $shard !== null ) { // confined to a single container/shard |
| | 289 | + $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); |
| | 290 | + } else { // directory is on several shards |
| | 291 | + wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| | 292 | + list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| | 293 | + foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| | 294 | + $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| | 295 | + } |
| | 296 | + } |
| | 297 | + |
| | 298 | + wfProfileOut( __METHOD__ ); |
| | 299 | + return $status; |
| | 300 | + } |
| | 301 | + |
| | 302 | + /** |
| | 303 | + * @see FileBackendStore::doPrepare() |
| | 304 | + * @return Status |
| | 305 | + */ |
| | 306 | + protected function doPrepareInternal( $container, $dir, array $params ) { |
| | 307 | + return Status::newGood(); |
| | 308 | + } |
| | 309 | + |
| | 310 | + /** |
| | 311 | + * @see FileBackend::doSecure() |
| | 312 | + * @return Status |
| | 313 | + */ |
| | 314 | + final protected function doSecure( array $params ) { |
| | 315 | + wfProfileIn( __METHOD__ ); |
| | 316 | + $status = Status::newGood(); |
| | 317 | + |
| | 318 | + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| | 319 | + if ( $dir === null ) { |
| | 320 | + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| | 321 | + wfProfileOut( __METHOD__ ); |
| | 322 | + return $status; // invalid storage path |
| | 323 | + } |
| | 324 | + |
| | 325 | + if ( $shard !== null ) { // confined to a single container/shard |
| | 326 | + $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); |
| | 327 | + } else { // directory is on several shards |
| | 328 | + wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| | 329 | + list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| | 330 | + foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| | 331 | + $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| | 332 | + } |
| | 333 | + } |
| | 334 | + |
| | 335 | + wfProfileOut( __METHOD__ ); |
| | 336 | + return $status; |
| | 337 | + } |
| | 338 | + |
| | 339 | + /** |
| | 340 | + * @see FileBackendStore::doSecure() |
| | 341 | + * @return Status |
| | 342 | + */ |
| | 343 | + protected function doSecureInternal( $container, $dir, array $params ) { |
| | 344 | + return Status::newGood(); |
| | 345 | + } |
| | 346 | + |
| | 347 | + /** |
| | 348 | + * @see FileBackend::doClean() |
| | 349 | + * @return Status |
| | 350 | + */ |
| | 351 | + final protected function doClean( array $params ) { |
| | 352 | + wfProfileIn( __METHOD__ ); |
| | 353 | + $status = Status::newGood(); |
| | 354 | + |
| | 355 | + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| | 356 | + if ( $dir === null ) { |
| | 357 | + $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| | 358 | + wfProfileOut( __METHOD__ ); |
| | 359 | + return $status; // invalid storage path |
| | 360 | + } |
| | 361 | + |
| | 362 | + // Attempt to lock this directory... |
| | 363 | + $filesLockEx = array( $params['dir'] ); |
| | 364 | + $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); |
| | 365 | + if ( !$status->isOK() ) { |
| | 366 | + wfProfileOut( __METHOD__ ); |
| | 367 | + return $status; // abort |
| | 368 | + } |
| | 369 | + |
| | 370 | + if ( $shard !== null ) { // confined to a single container/shard |
| | 371 | + $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); |
| | 372 | + } else { // directory is on several shards |
| | 373 | + wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| | 374 | + list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| | 375 | + foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| | 376 | + $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| | 377 | + } |
| | 378 | + } |
| | 379 | + |
| | 380 | + wfProfileOut( __METHOD__ ); |
| | 381 | + return $status; |
| | 382 | + } |
| | 383 | + |
| | 384 | + /** |
| | 385 | + * @see FileBackendStore::doClean() |
| | 386 | + * @return Status |
| | 387 | + */ |
| | 388 | + protected function doCleanInternal( $container, $dir, array $params ) { |
| | 389 | + return Status::newGood(); |
| | 390 | + } |
| | 391 | + |
| | 392 | + /** |
| | 393 | + * @see FileBackend::fileExists() |
| | 394 | + * @return bool|null |
| | 395 | + */ |
| | 396 | + final public function fileExists( array $params ) { |
| | 397 | + wfProfileIn( __METHOD__ ); |
| | 398 | + $stat = $this->getFileStat( $params ); |
| | 399 | + wfProfileOut( __METHOD__ ); |
| | 400 | + return ( $stat === null ) ? null : (bool)$stat; // null => failure |
| | 401 | + } |
| | 402 | + |
| | 403 | + /** |
| | 404 | + * @see FileBackend::getFileTimestamp() |
| | 405 | + * @return bool |
| | 406 | + */ |
| | 407 | + final public function getFileTimestamp( array $params ) { |
| | 408 | + wfProfileIn( __METHOD__ ); |
| | 409 | + $stat = $this->getFileStat( $params ); |
| | 410 | + wfProfileOut( __METHOD__ ); |
| | 411 | + return $stat ? $stat['mtime'] : false; |
| | 412 | + } |
| | 413 | + |
| | 414 | + /** |
| | 415 | + * @see FileBackend::getFileSize() |
| | 416 | + * @return bool |
| | 417 | + */ |
| | 418 | + final public function getFileSize( array $params ) { |
| | 419 | + wfProfileIn( __METHOD__ ); |
| | 420 | + $stat = $this->getFileStat( $params ); |
| | 421 | + wfProfileOut( __METHOD__ ); |
| | 422 | + return $stat ? $stat['size'] : false; |
| | 423 | + } |
| | 424 | + |
| | 425 | + /** |
| | 426 | + * @see FileBackend::getFileStat() |
| | 427 | + * @return bool |
| | 428 | + */ |
| | 429 | + final public function getFileStat( array $params ) { |
| | 430 | + wfProfileIn( __METHOD__ ); |
| | 431 | + $path = self::normalizeStoragePath( $params['src'] ); |
| | 432 | + if ( $path === null ) { |
| | 433 | + wfProfileOut( __METHOD__ ); |
| | 434 | + return false; // invalid storage path |
| | 435 | + } |
| | 436 | + $latest = !empty( $params['latest'] ); |
| | 437 | + if ( isset( $this->cache[$path]['stat'] ) ) { |
| | 438 | + // If we want the latest data, check that this cached |
| | 439 | + // value was in fact fetched with the latest available data. |
| | 440 | + if ( !$latest || $this->cache[$path]['stat']['latest'] ) { |
| | 441 | + wfProfileOut( __METHOD__ ); |
| | 442 | + return $this->cache[$path]['stat']; |
| | 443 | + } |
| | 444 | + } |
| | 445 | + wfProfileIn( __METHOD__ . '-miss' ); |
| | 446 | + $stat = $this->doGetFileStat( $params ); |
| | 447 | + wfProfileOut( __METHOD__ . '-miss' ); |
| | 448 | + if ( is_array( $stat ) ) { // don't cache negatives |
| | 449 | + $this->trimCache(); // limit memory |
| | 450 | + $this->cache[$path]['stat'] = $stat; |
| | 451 | + $this->cache[$path]['stat']['latest'] = $latest; |
| | 452 | + } |
| | 453 | + wfProfileOut( __METHOD__ ); |
| | 454 | + return $stat; |
| | 455 | + } |
| | 456 | + |
| | 457 | + /** |
| | 458 | + * @see FileBackendStore::getFileStat() |
| | 459 | + */ |
| | 460 | + abstract protected function doGetFileStat( array $params ); |
| | 461 | + |
| | 462 | + /** |
| | 463 | + * @see FileBackend::getFileContents() |
| | 464 | + * @return bool|string |
| | 465 | + */ |
| | 466 | + public function getFileContents( array $params ) { |
| | 467 | + wfProfileIn( __METHOD__ ); |
| | 468 | + $tmpFile = $this->getLocalReference( $params ); |
| | 469 | + if ( !$tmpFile ) { |
| | 470 | + wfProfileOut( __METHOD__ ); |
| | 471 | + return false; |
| | 472 | + } |
| | 473 | + wfSuppressWarnings(); |
| | 474 | + $data = file_get_contents( $tmpFile->getPath() ); |
| | 475 | + wfRestoreWarnings(); |
| | 476 | + wfProfileOut( __METHOD__ ); |
| | 477 | + return $data; |
| | 478 | + } |
| | 479 | + |
| | 480 | + /** |
| | 481 | + * @see FileBackend::getFileSha1Base36() |
| | 482 | + * @return bool|string |
| | 483 | + */ |
| | 484 | + final public function getFileSha1Base36( array $params ) { |
| | 485 | + wfProfileIn( __METHOD__ ); |
| | 486 | + $path = $params['src']; |
| | 487 | + if ( isset( $this->cache[$path]['sha1'] ) ) { |
| | 488 | + wfProfileOut( __METHOD__ ); |
| | 489 | + return $this->cache[$path]['sha1']; |
| | 490 | + } |
| | 491 | + wfProfileIn( __METHOD__ . '-miss' ); |
| | 492 | + $hash = $this->doGetFileSha1Base36( $params ); |
| | 493 | + wfProfileOut( __METHOD__ . '-miss' ); |
| | 494 | + if ( $hash ) { // don't cache negatives |
| | 495 | + $this->trimCache(); // limit memory |
| | 496 | + $this->cache[$path]['sha1'] = $hash; |
| | 497 | + } |
| | 498 | + wfProfileOut( __METHOD__ ); |
| | 499 | + return $hash; |
| | 500 | + } |
| | 501 | + |
| | 502 | + /** |
| | 503 | + * @see FileBackendStore::getFileSha1Base36() |
| | 504 | + * @return bool |
| | 505 | + */ |
| | 506 | + protected function doGetFileSha1Base36( array $params ) { |
| | 507 | + $fsFile = $this->getLocalReference( $params ); |
| | 508 | + if ( !$fsFile ) { |
| | 509 | + return false; |
| | 510 | + } else { |
| | 511 | + return $fsFile->getSha1Base36(); |
| | 512 | + } |
| | 513 | + } |
| | 514 | + |
| | 515 | + /** |
| | 516 | + * @see FileBackend::getFileProps() |
| | 517 | + * @return Array |
| | 518 | + */ |
| | 519 | + final public function getFileProps( array $params ) { |
| | 520 | + wfProfileIn( __METHOD__ ); |
| | 521 | + $fsFile = $this->getLocalReference( $params ); |
| | 522 | + $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); |
| | 523 | + wfProfileOut( __METHOD__ ); |
| | 524 | + return $props; |
| | 525 | + } |
| | 526 | + |
| | 527 | + /** |
| | 528 | + * @see FileBackend::getLocalReference() |
| | 529 | + * @return TempFSFile|null |
| | 530 | + */ |
| | 531 | + public function getLocalReference( array $params ) { |
| | 532 | + wfProfileIn( __METHOD__ ); |
| | 533 | + $path = $params['src']; |
| | 534 | + if ( isset( $this->expensiveCache[$path]['localRef'] ) ) { |
| | 535 | + wfProfileOut( __METHOD__ ); |
| | 536 | + return $this->expensiveCache[$path]['localRef']; |
| | 537 | + } |
| | 538 | + $tmpFile = $this->getLocalCopy( $params ); |
| | 539 | + if ( $tmpFile ) { // don't cache negatives |
| | 540 | + $this->trimExpensiveCache(); // limit memory |
| | 541 | + $this->expensiveCache[$path]['localRef'] = $tmpFile; |
| | 542 | + } |
| | 543 | + wfProfileOut( __METHOD__ ); |
| | 544 | + return $tmpFile; |
| | 545 | + } |
| | 546 | + |
| | 547 | + /** |
| | 548 | + * @see FileBackend::streamFile() |
| | 549 | + * @return Status |
| | 550 | + */ |
| | 551 | + final public function streamFile( array $params ) { |
| | 552 | + wfProfileIn( __METHOD__ ); |
| | 553 | + $status = Status::newGood(); |
| | 554 | + |
| | 555 | + $info = $this->getFileStat( $params ); |
| | 556 | + if ( !$info ) { // let StreamFile handle the 404 |
| | 557 | + $status->fatal( 'backend-fail-notexists', $params['src'] ); |
| | 558 | + } |
| | 559 | + |
| | 560 | + // Set output buffer and HTTP headers for stream |
| | 561 | + $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array(); |
| | 562 | + $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders ); |
| | 563 | + if ( $res == StreamFile::NOT_MODIFIED ) { |
| | 564 | + // do nothing; client cache is up to date |
| | 565 | + } elseif ( $res == StreamFile::READY_STREAM ) { |
| | 566 | + wfProfileIn( __METHOD__ . '-send' ); |
| | 567 | + $status = $this->doStreamFile( $params ); |
| | 568 | + wfProfileOut( __METHOD__ . '-send' ); |
| | 569 | + } else { |
| | 570 | + $status->fatal( 'backend-fail-stream', $params['src'] ); |
| | 571 | + } |
| | 572 | + |
| | 573 | + wfProfileOut( __METHOD__ ); |
| | 574 | + return $status; |
| | 575 | + } |
| | 576 | + |
| | 577 | + /** |
| | 578 | + * @see FileBackendStore::streamFile() |
| | 579 | + * @return Status |
| | 580 | + */ |
| | 581 | + protected function doStreamFile( array $params ) { |
| | 582 | + $status = Status::newGood(); |
| | 583 | + |
| | 584 | + $fsFile = $this->getLocalReference( $params ); |
| | 585 | + if ( !$fsFile ) { |
| | 586 | + $status->fatal( 'backend-fail-stream', $params['src'] ); |
| | 587 | + } elseif ( !readfile( $fsFile->getPath() ) ) { |
| | 588 | + $status->fatal( 'backend-fail-stream', $params['src'] ); |
| | 589 | + } |
| | 590 | + |
| | 591 | + return $status; |
| | 592 | + } |
| | 593 | + |
| | 594 | + /** |
| | 595 | + * @copydoc FileBackend::getFileList() |
| | 596 | + * @return Array|null|Traversable |
| | 597 | + */ |
| | 598 | + final public function getFileList( array $params ) { |
| | 599 | + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| | 600 | + if ( $dir === null ) { // invalid storage path |
| | 601 | + return null; |
| | 602 | + } |
| | 603 | + if ( $shard !== null ) { |
| | 604 | + // File listing is confined to a single container/shard |
| | 605 | + return $this->getFileListInternal( $fullCont, $dir, $params ); |
| | 606 | + } else { |
| | 607 | + wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| | 608 | + // File listing spans multiple containers/shards |
| | 609 | + list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| | 610 | + return new FileBackendStoreShardListIterator( $this, |
| | 611 | + $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); |
| | 612 | + } |
| | 613 | + } |
| | 614 | + |
| | 615 | + /** |
| | 616 | + * Do not call this function from places outside FileBackend |
| | 617 | + * |
| | 618 | + * @see FileBackendStore::getFileList() |
| | 619 | + * |
| | 620 | + * @param $container string Resolved container name |
| | 621 | + * @param $dir string Resolved path relative to container |
| | 622 | + * @param $params Array |
| | 623 | + * @return Traversable|Array|null |
| | 624 | + */ |
| | 625 | + abstract public function getFileListInternal( $container, $dir, array $params ); |
| | 626 | + |
| | 627 | + /** |
| | 628 | + * Get the list of supported operations and their corresponding FileOp classes. |
| | 629 | + * |
| | 630 | + * @return Array |
| | 631 | + */ |
| | 632 | + protected function supportedOperations() { |
| | 633 | + return array( |
| | 634 | + 'store' => 'StoreFileOp', |
| | 635 | + 'copy' => 'CopyFileOp', |
| | 636 | + 'move' => 'MoveFileOp', |
| | 637 | + 'delete' => 'DeleteFileOp', |
| | 638 | + 'create' => 'CreateFileOp', |
| | 639 | + 'null' => 'NullFileOp' |
| | 640 | + ); |
| | 641 | + } |
| | 642 | + |
| | 643 | + /** |
| | 644 | + * Return a list of FileOp objects from a list of operations. |
| | 645 | + * Do not call this function from places outside FileBackend. |
| | 646 | + * |
| | 647 | + * The result must have the same number of items as the input. |
| | 648 | + * An exception is thrown if an unsupported operation is requested. |
| | 649 | + * |
| | 650 | + * @param $ops Array Same format as doOperations() |
| | 651 | + * @return Array List of FileOp objects |
| | 652 | + * @throws MWException |
| | 653 | + */ |
| | 654 | + final public function getOperations( array $ops ) { |
| | 655 | + $supportedOps = $this->supportedOperations(); |
| | 656 | + |
| | 657 | + $performOps = array(); // array of FileOp objects |
| | 658 | + // Build up ordered array of FileOps... |
| | 659 | + foreach ( $ops as $operation ) { |
| | 660 | + $opName = $operation['op']; |
| | 661 | + if ( isset( $supportedOps[$opName] ) ) { |
| | 662 | + $class = $supportedOps[$opName]; |
| | 663 | + // Get params for this operation |
| | 664 | + $params = $operation; |
| | 665 | + // Append the FileOp class |
| | 666 | + $performOps[] = new $class( $this, $params ); |
| | 667 | + } else { |
| | 668 | + throw new MWException( "Operation `$opName` is not supported." ); |
| | 669 | + } |
| | 670 | + } |
| | 671 | + |
| | 672 | + return $performOps; |
| | 673 | + } |
| | 674 | + |
| | 675 | + /** |
| | 676 | + * @see FileBackend::doOperationsInternal() |
| | 677 | + * @return Status |
| | 678 | + */ |
| | 679 | + protected function doOperationsInternal( array $ops, array $opts ) { |
| | 680 | + wfProfileIn( __METHOD__ ); |
| | 681 | + $status = Status::newGood(); |
| | 682 | + |
| | 683 | + // Build up a list of FileOps... |
| | 684 | + $performOps = $this->getOperations( $ops ); |
| | 685 | + |
| | 686 | + // Acquire any locks as needed... |
| | 687 | + if ( empty( $opts['nonLocking'] ) ) { |
| | 688 | + // Build up a list of files to lock... |
| | 689 | + $filesLockEx = $filesLockSh = array(); |
| | 690 | + foreach ( $performOps as $fileOp ) { |
| | 691 | + $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() ); |
| | 692 | + $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() ); |
| | 693 | + } |
| | 694 | + // Optimization: if doing an EX lock anyway, don't also set an SH one |
| | 695 | + $filesLockSh = array_diff( $filesLockSh, $filesLockEx ); |
| | 696 | + // Get a shared lock on the parent directory of each path changed |
| | 697 | + $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) ); |
| | 698 | + // Try to lock those files for the scope of this function... |
| | 699 | + $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status ); |
| | 700 | + $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); |
| | 701 | + if ( !$status->isOK() ) { |
| | 702 | + wfProfileOut( __METHOD__ ); |
| | 703 | + return $status; // abort |
| | 704 | + } |
| | 705 | + } |
| | 706 | + |
| | 707 | + // Clear any cache entries (after locks acquired) |
| | 708 | + $this->clearCache(); |
| | 709 | + |
| | 710 | + // Actually attempt the operation batch... |
| | 711 | + $subStatus = FileOp::attemptBatch( $performOps, $opts ); |
| | 712 | + |
| | 713 | + // Merge errors into status fields |
| | 714 | + $status->merge( $subStatus ); |
| | 715 | + $status->success = $subStatus->success; // not done in merge() |
| | 716 | + |
| | 717 | + wfProfileOut( __METHOD__ ); |
| | 718 | + return $status; |
| | 719 | + } |
| | 720 | + |
| | 721 | + /** |
| | 722 | + * @see FileBackend::clearCache() |
| | 723 | + */ |
| | 724 | + final public function clearCache( array $paths = null ) { |
| | 725 | + if ( is_array( $paths ) ) { |
| | 726 | + $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); |
| | 727 | + $paths = array_filter( $paths, 'strlen' ); // remove nulls |
| | 728 | + } |
| | 729 | + if ( $paths === null ) { |
| | 730 | + $this->cache = array(); |
| | 731 | + $this->expensiveCache = array(); |
| | 732 | + } else { |
| | 733 | + foreach ( $paths as $path ) { |
| | 734 | + unset( $this->cache[$path] ); |
| | 735 | + unset( $this->expensiveCache[$path] ); |
| | 736 | + } |
| | 737 | + } |
| | 738 | + $this->doClearCache( $paths ); |
| | 739 | + } |
| | 740 | + |
| | 741 | + /** |
| | 742 | + * Clears any additional stat caches for storage paths |
| | 743 | + * |
| | 744 | + * @see FileBackend::clearCache() |
| | 745 | + * |
| | 746 | + * @param $paths Array Storage paths (optional) |
| | 747 | + * @return void |
| | 748 | + */ |
| | 749 | + protected function doClearCache( array $paths = null ) {} |
| | 750 | + |
| | 751 | + /** |
| | 752 | + * Prune the inexpensive cache if it is too big to add an item |
| | 753 | + * |
| | 754 | + * @return void |
| | 755 | + */ |
| | 756 | + protected function trimCache() { |
| | 757 | + if ( count( $this->cache ) >= $this->maxCacheSize ) { |
| | 758 | + reset( $this->cache ); |
| | 759 | + unset( $this->cache[key( $this->cache )] ); |
| | 760 | + } |
| | 761 | + } |
| | 762 | + |
| | 763 | + /** |
| | 764 | + * Prune the expensive cache if it is too big to add an item |
| | 765 | + * |
| | 766 | + * @return void |
| | 767 | + */ |
| | 768 | + protected function trimExpensiveCache() { |
| | 769 | + if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) { |
| | 770 | + reset( $this->expensiveCache ); |
| | 771 | + unset( $this->expensiveCache[key( $this->expensiveCache )] ); |
| | 772 | + } |
| | 773 | + } |
| | 774 | + |
| | 775 | + /** |
| | 776 | + * Check if a container name is valid. |
| | 777 | + * This checks for for length and illegal characters. |
| | 778 | + * |
| | 779 | + * @param $container string |
| | 780 | + * @return bool |
| | 781 | + */ |
| | 782 | + final protected static function isValidContainerName( $container ) { |
| | 783 | + // This accounts for Swift and S3 restrictions while leaving room |
| | 784 | + // for things like '.xxx' (hex shard chars) or '.seg' (segments). |
| | 785 | + // This disallows directory separators or traversal characters. |
| | 786 | + // Note that matching strings URL encode to the same string; |
| | 787 | + // in Swift, the length restriction is *after* URL encoding. |
| | 788 | + return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container ); |
| | 789 | + } |
| | 790 | + |
| | 791 | + /** |
| | 792 | + * Splits a storage path into an internal container name, |
| | 793 | + * an internal relative file name, and a container shard suffix. |
| | 794 | + * Any shard suffix is already appended to the internal container name. |
| | 795 | + * This also checks that the storage path is valid and within this backend. |
| | 796 | + * |
| | 797 | + * If the container is sharded but a suffix could not be determined, |
| | 798 | + * this means that the path can only refer to a directory and can only |
| | 799 | + * be scanned by looking in all the container shards. |
| | 800 | + * |
| | 801 | + * @param $storagePath string |
| | 802 | + * @return Array (container, path, container suffix) or (null, null, null) if invalid |
| | 803 | + */ |
| | 804 | + final protected function resolveStoragePath( $storagePath ) { |
| | 805 | + list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); |
| | 806 | + if ( $backend === $this->name ) { // must be for this backend |
| | 807 | + $relPath = self::normalizeContainerPath( $relPath ); |
| | 808 | + if ( $relPath !== null ) { |
| | 809 | + // Get shard for the normalized path if this container is sharded |
| | 810 | + $cShard = $this->getContainerShard( $container, $relPath ); |
| | 811 | + // Validate and sanitize the relative path (backend-specific) |
| | 812 | + $relPath = $this->resolveContainerPath( $container, $relPath ); |
| | 813 | + if ( $relPath !== null ) { |
| | 814 | + // Prepend any wiki ID prefix to the container name |
| | 815 | + $container = $this->fullContainerName( $container ); |
| | 816 | + if ( self::isValidContainerName( $container ) ) { |
| | 817 | + // Validate and sanitize the container name (backend-specific) |
| | 818 | + $container = $this->resolveContainerName( "{$container}{$cShard}" ); |
| | 819 | + if ( $container !== null ) { |
| | 820 | + return array( $container, $relPath, $cShard ); |
| | 821 | + } |
| | 822 | + } |
| | 823 | + } |
| | 824 | + } |
| | 825 | + } |
| | 826 | + return array( null, null, null ); |
| | 827 | + } |
| | 828 | + |
| | 829 | + /** |
| | 830 | + * Like resolveStoragePath() except null values are returned if |
| | 831 | + * the container is sharded and the shard could not be determined. |
| | 832 | + * |
| | 833 | + * @see FileBackendStore::resolveStoragePath() |
| | 834 | + * |
| | 835 | + * @param $storagePath string |
| | 836 | + * @return Array (container, path) or (null, null) if invalid |
| | 837 | + */ |
| | 838 | + final protected function resolveStoragePathReal( $storagePath ) { |
| | 839 | + list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); |
| | 840 | + if ( $cShard !== null ) { |
| | 841 | + return array( $container, $relPath ); |
| | 842 | + } |
| | 843 | + return array( null, null ); |
| | 844 | + } |
| | 845 | + |
| | 846 | + /** |
| | 847 | + * Get the container name shard suffix for a given path. |
| | 848 | + * Any empty suffix means the container is not sharded. |
| | 849 | + * |
| | 850 | + * @param $container string Container name |
| | 851 | + * @param $relStoragePath string Storage path relative to the container |
| | 852 | + * @return string|null Returns null if shard could not be determined |
| | 853 | + */ |
| | 854 | + final protected function getContainerShard( $container, $relPath ) { |
| | 855 | + list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); |
| | 856 | + if ( $levels == 1 || $levels == 2 ) { |
| | 857 | + // Hash characters are either base 16 or 36 |
| | 858 | + $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; |
| | 859 | + // Get a regex that represents the shard portion of paths. |
| | 860 | + // The concatenation of the captures gives us the shard. |
| | 861 | + if ( $levels === 1 ) { // 16 or 36 shards per container |
| | 862 | + $hashDirRegex = '(' . $char . ')'; |
| | 863 | + } else { // 256 or 1296 shards per container |
| | 864 | + if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") |
| | 865 | + $hashDirRegex = $char . '/(' . $char . '{2})'; |
| | 866 | + } else { // short hash dir format (e.g. "a/b/c") |
| | 867 | + $hashDirRegex = '(' . $char . ')/(' . $char . ')'; |
| | 868 | + } |
| | 869 | + } |
| | 870 | + // Allow certain directories to be above the hash dirs so as |
| | 871 | + // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). |
| | 872 | + // They must be 2+ chars to avoid any hash directory ambiguity. |
| | 873 | + $m = array(); |
| | 874 | + if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { |
| | 875 | + return '.' . implode( '', array_slice( $m, 1 ) ); |
| | 876 | + } |
| | 877 | + return null; // failed to match |
| | 878 | + } |
| | 879 | + return ''; // no sharding |
| | 880 | + } |
| | 881 | + |
| | 882 | + /** |
| | 883 | + * Get the sharding config for a container. |
| | 884 | + * If greater than 0, then all file storage paths within |
| | 885 | + * the container are required to be hashed accordingly. |
| | 886 | + * |
| | 887 | + * @param $container string |
| | 888 | + * @return Array (integer levels, integer base, repeat flag) or (0, 0, false) |
| | 889 | + */ |
| | 890 | + final protected function getContainerHashLevels( $container ) { |
| | 891 | + if ( isset( $this->shardViaHashLevels[$container] ) ) { |
| | 892 | + $config = $this->shardViaHashLevels[$container]; |
| | 893 | + $hashLevels = (int)$config['levels']; |
| | 894 | + if ( $hashLevels == 1 || $hashLevels == 2 ) { |
| | 895 | + $hashBase = (int)$config['base']; |
| | 896 | + if ( $hashBase == 16 || $hashBase == 36 ) { |
| | 897 | + return array( $hashLevels, $hashBase, $config['repeat'] ); |
| | 898 | + } |
| | 899 | + } |
| | 900 | + } |
| | 901 | + return array( 0, 0, false ); // no sharding |
| | 902 | + } |
| | 903 | + |
| | 904 | + /** |
| | 905 | + * Get a list of full container shard suffixes for a container |
| | 906 | + * |
| | 907 | + * @param $container string |
| | 908 | + * @return Array |
| | 909 | + */ |
| | 910 | + final protected function getContainerSuffixes( $container ) { |
| | 911 | + $shards = array(); |
| | 912 | + list( $digits, $base ) = $this->getContainerHashLevels( $container ); |
| | 913 | + if ( $digits > 0 ) { |
| | 914 | + $numShards = pow( $base, $digits ); |
| | 915 | + for ( $index = 0; $index < $numShards; $index++ ) { |
| | 916 | + $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits ); |
| | 917 | + } |
| | 918 | + } |
| | 919 | + return $shards; |
| | 920 | + } |
| | 921 | + |
| | 922 | + /** |
| | 923 | + * Get the full container name, including the wiki ID prefix |
| | 924 | + * |
| | 925 | + * @param $container string |
| | 926 | + * @return string |
| | 927 | + */ |
| | 928 | + final protected function fullContainerName( $container ) { |
| | 929 | + if ( $this->wikiId != '' ) { |
| | 930 | + return "{$this->wikiId}-$container"; |
| | 931 | + } else { |
| | 932 | + return $container; |
| | 933 | + } |
| | 934 | + } |
| | 935 | + |
| | 936 | + /** |
| | 937 | + * Resolve a container name, checking if it's allowed by the backend. |
| | 938 | + * This is intended for internal use, such as encoding illegal chars. |
| | 939 | + * Subclasses can override this to be more restrictive. |
| | 940 | + * |
| | 941 | + * @param $container string |
| | 942 | + * @return string|null |
| | 943 | + */ |
| | 944 | + protected function resolveContainerName( $container ) { |
| | 945 | + return $container; |
| | 946 | + } |
| | 947 | + |
| | 948 | + /** |
| | 949 | + * Resolve a relative storage path, checking if it's allowed by the backend. |
| | 950 | + * This is intended for internal use, such as encoding illegal chars or perhaps |
| | 951 | + * getting absolute paths (e.g. FS based backends). Note that the relative path |
| | 952 | + * may be the empty string (e.g. the path is simply to the container). |
| | 953 | + * |
| | 954 | + * @param $container string Container name |
| | 955 | + * @param $relStoragePath string Storage path relative to the container |
| | 956 | + * @return string|null Path or null if not valid |
| | 957 | + */ |
| | 958 | + protected function resolveContainerPath( $container, $relStoragePath ) { |
| | 959 | + return $relStoragePath; |
| | 960 | + } |
| | 961 | +} |
| | 962 | + |
| | 963 | +/** |
| | 964 | + * FileBackendStore helper function to handle file listings that span container shards. |
| | 965 | + * Do not use this class from places outside of FileBackendStore. |
| | 966 | + * |
| | 967 | + * @ingroup FileBackend |
| | 968 | + */ |
| | 969 | +class FileBackendStoreShardListIterator implements Iterator { |
| | 970 | + /* @var FileBackendStore */ |
| | 971 | + protected $backend; |
| | 972 | + /* @var Array */ |
| | 973 | + protected $params; |
| | 974 | + /* @var Array */ |
| | 975 | + protected $shardSuffixes; |
| | 976 | + protected $container; // string |
| | 977 | + protected $directory; // string |
| | 978 | + |
| | 979 | + /* @var Traversable */ |
| | 980 | + protected $iter; |
| | 981 | + protected $curShard = 0; // integer |
| | 982 | + protected $pos = 0; // integer |
| | 983 | + |
| | 984 | + /** |
| | 985 | + * @param $backend FileBackendStore |
| | 986 | + * @param $container string Full storage container name |
| | 987 | + * @param $dir string Storage directory relative to container |
| | 988 | + * @param $suffixes Array List of container shard suffixes |
| | 989 | + * @param $params Array |
| | 990 | + */ |
| | 991 | + public function __construct( |
| | 992 | + FileBackendStore $backend, $container, $dir, array $suffixes, array $params |
| | 993 | + ) { |
| | 994 | + $this->backend = $backend; |
| | 995 | + $this->container = $container; |
| | 996 | + $this->directory = $dir; |
| | 997 | + $this->shardSuffixes = $suffixes; |
| | 998 | + $this->params = $params; |
| | 999 | + } |
| | 1000 | + |
| | 1001 | + public function current() { |
| | 1002 | + if ( is_array( $this->iter ) ) { |
| | 1003 | + return current( $this->iter ); |
| | 1004 | + } else { |
| | 1005 | + return $this->iter->current(); |
| | 1006 | + } |
| | 1007 | + } |
| | 1008 | + |
| | 1009 | + public function key() { |
| | 1010 | + return $this->pos; |
| | 1011 | + } |
| | 1012 | + |
| | 1013 | + public function next() { |
| | 1014 | + ++$this->pos; |
| | 1015 | + if ( is_array( $this->iter ) ) { |
| | 1016 | + next( $this->iter ); |
| | 1017 | + } else { |
| | 1018 | + $this->iter->next(); |
| | 1019 | + } |
| | 1020 | + // Find the next non-empty shard if no elements are left |
| | 1021 | + $this->nextShardIteratorIfNotValid(); |
| | 1022 | + } |
| | 1023 | + |
| | 1024 | + /** |
| | 1025 | + * If the iterator for this container shard is out of items, |
| | 1026 | + * then move on to the next container that has items. |
| | 1027 | + * If there are none, then it advances to the last container. |
| | 1028 | + */ |
| | 1029 | + protected function nextShardIteratorIfNotValid() { |
| | 1030 | + while ( !$this->valid() ) { |
| | 1031 | + if ( ++$this->curShard >= count( $this->shardSuffixes ) ) { |
| | 1032 | + break; // no more container shards |
| | 1033 | + } |
| | 1034 | + $this->setIteratorFromCurrentShard(); |
| | 1035 | + } |
| | 1036 | + } |
| | 1037 | + |
| | 1038 | + protected function setIteratorFromCurrentShard() { |
| | 1039 | + $suffix = $this->shardSuffixes[$this->curShard]; |
| | 1040 | + $this->iter = $this->backend->getFileListInternal( |
| | 1041 | + "{$this->container}{$suffix}", $this->directory, $this->params ); |
| | 1042 | + } |
| | 1043 | + |
| | 1044 | + public function rewind() { |
| | 1045 | + $this->pos = 0; |
| | 1046 | + $this->curShard = 0; |
| | 1047 | + $this->setIteratorFromCurrentShard(); |
| | 1048 | + // Find the next non-empty shard if this one has no elements |
| | 1049 | + $this->nextShardIteratorIfNotValid(); |
| | 1050 | + } |
| | 1051 | + |
| | 1052 | + public function valid() { |
| | 1053 | + if ( $this->iter == null ) { |
| | 1054 | + return false; // some failure? |
| | 1055 | + } elseif ( is_array( $this->iter ) ) { |
| | 1056 | + return ( current( $this->iter ) !== false ); // no paths can have this value |
| | 1057 | + } else { |
| | 1058 | + return $this->iter->valid(); |
| | 1059 | + } |
| | 1060 | + } |
| | 1061 | +} |
| Property changes on: trunk/phase3/includes/filerepo/backend/FileBackendStore.php |
| ___________________________________________________________________ |
| Added: svn:eol-style |
| 1 | 1062 | + native |
| Index: trunk/phase3/includes/filerepo/backend/FileBackend.php |
| — | — | @@ -713,1057 +713,3 @@ |
| 714 | 714 | return strtolower( $i ? substr( $path, $i + 1 ) : '' ); |
| 715 | 715 | } |
| 716 | 716 | } |
| 717 | | - |
| 718 | | -/** |
| 719 | | - * @brief Base class for all backends using particular storage medium. |
| 720 | | - * |
| 721 | | - * This class defines the methods as abstract that subclasses must implement. |
| 722 | | - * Outside callers should *not* use functions with "Internal" in the name. |
| 723 | | - * |
| 724 | | - * The FileBackend operations are implemented using basic functions |
| 725 | | - * such as storeInternal(), copyInternal(), deleteInternal() and the like. |
| 726 | | - * This class is also responsible for path resolution and sanitization. |
| 727 | | - * |
| 728 | | - * @ingroup FileBackend |
| 729 | | - * @since 1.19 |
| 730 | | - */ |
| 731 | | -abstract class FileBackendStore extends FileBackend { |
| 732 | | - /** @var Array Map of paths to small (RAM/disk) cache items */ |
| 733 | | - protected $cache = array(); // (storage path => key => value) |
| 734 | | - protected $maxCacheSize = 100; // integer; max paths with entries |
| 735 | | - /** @var Array Map of paths to large (RAM/disk) cache items */ |
| 736 | | - protected $expensiveCache = array(); // (storage path => key => value) |
| 737 | | - protected $maxExpensiveCacheSize = 10; // integer; max paths with entries |
| 738 | | - |
| 739 | | - /** @var Array Map of container names to sharding settings */ |
| 740 | | - protected $shardViaHashLevels = array(); // (container name => config array) |
| 741 | | - |
| 742 | | - protected $maxFileSize = 1000000000; // integer bytes (1GB) |
| 743 | | - |
| 744 | | - /** |
| 745 | | - * Get the maximum allowable file size given backend |
| 746 | | - * medium restrictions and basic performance constraints. |
| 747 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 748 | | - * |
| 749 | | - * @return integer Bytes |
| 750 | | - */ |
| 751 | | - final public function maxFileSizeInternal() { |
| 752 | | - return $this->maxFileSize; |
| 753 | | - } |
| 754 | | - |
| 755 | | - /** |
| 756 | | - * Check if a file can be created at a given storage path. |
| 757 | | - * FS backends should check if the parent directory exists and the file is writable. |
| 758 | | - * Backends using key/value stores should check if the container exists. |
| 759 | | - * |
| 760 | | - * @param $storagePath string |
| 761 | | - * @return bool |
| 762 | | - */ |
| 763 | | - abstract public function isPathUsableInternal( $storagePath ); |
| 764 | | - |
| 765 | | - /** |
| 766 | | - * Create a file in the backend with the given contents. |
| 767 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 768 | | - * |
| 769 | | - * $params include: |
| 770 | | - * content : the raw file contents |
| 771 | | - * dst : destination storage path |
| 772 | | - * overwrite : overwrite any file that exists at the destination |
| 773 | | - * |
| 774 | | - * @param $params Array |
| 775 | | - * @return Status |
| 776 | | - */ |
| 777 | | - final public function createInternal( array $params ) { |
| 778 | | - wfProfileIn( __METHOD__ ); |
| 779 | | - if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { |
| 780 | | - $status = Status::newFatal( 'backend-fail-create', $params['dst'] ); |
| 781 | | - } else { |
| 782 | | - $status = $this->doCreateInternal( $params ); |
| 783 | | - $this->clearCache( array( $params['dst'] ) ); |
| 784 | | - } |
| 785 | | - wfProfileOut( __METHOD__ ); |
| 786 | | - return $status; |
| 787 | | - } |
| 788 | | - |
| 789 | | - /** |
| 790 | | - * @see FileBackendStore::createInternal() |
| 791 | | - */ |
| 792 | | - abstract protected function doCreateInternal( array $params ); |
| 793 | | - |
| 794 | | - /** |
| 795 | | - * Store a file into the backend from a file on disk. |
| 796 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 797 | | - * |
| 798 | | - * $params include: |
| 799 | | - * src : source path on disk |
| 800 | | - * dst : destination storage path |
| 801 | | - * overwrite : overwrite any file that exists at the destination |
| 802 | | - * |
| 803 | | - * @param $params Array |
| 804 | | - * @return Status |
| 805 | | - */ |
| 806 | | - final public function storeInternal( array $params ) { |
| 807 | | - wfProfileIn( __METHOD__ ); |
| 808 | | - if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { |
| 809 | | - $status = Status::newFatal( 'backend-fail-store', $params['dst'] ); |
| 810 | | - } else { |
| 811 | | - $status = $this->doStoreInternal( $params ); |
| 812 | | - $this->clearCache( array( $params['dst'] ) ); |
| 813 | | - } |
| 814 | | - wfProfileOut( __METHOD__ ); |
| 815 | | - return $status; |
| 816 | | - } |
| 817 | | - |
| 818 | | - /** |
| 819 | | - * @see FileBackendStore::storeInternal() |
| 820 | | - */ |
| 821 | | - abstract protected function doStoreInternal( array $params ); |
| 822 | | - |
| 823 | | - /** |
| 824 | | - * Copy a file from one storage path to another in the backend. |
| 825 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 826 | | - * |
| 827 | | - * $params include: |
| 828 | | - * src : source storage path |
| 829 | | - * dst : destination storage path |
| 830 | | - * overwrite : overwrite any file that exists at the destination |
| 831 | | - * |
| 832 | | - * @param $params Array |
| 833 | | - * @return Status |
| 834 | | - */ |
| 835 | | - final public function copyInternal( array $params ) { |
| 836 | | - wfProfileIn( __METHOD__ ); |
| 837 | | - $status = $this->doCopyInternal( $params ); |
| 838 | | - $this->clearCache( array( $params['dst'] ) ); |
| 839 | | - wfProfileOut( __METHOD__ ); |
| 840 | | - return $status; |
| 841 | | - } |
| 842 | | - |
| 843 | | - /** |
| 844 | | - * @see FileBackendStore::copyInternal() |
| 845 | | - */ |
| 846 | | - abstract protected function doCopyInternal( array $params ); |
| 847 | | - |
| 848 | | - /** |
| 849 | | - * Delete a file at the storage path. |
| 850 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 851 | | - * |
| 852 | | - * $params include: |
| 853 | | - * src : source storage path |
| 854 | | - * ignoreMissingSource : do nothing if the source file does not exist |
| 855 | | - * |
| 856 | | - * @param $params Array |
| 857 | | - * @return Status |
| 858 | | - */ |
| 859 | | - final public function deleteInternal( array $params ) { |
| 860 | | - wfProfileIn( __METHOD__ ); |
| 861 | | - $status = $this->doDeleteInternal( $params ); |
| 862 | | - $this->clearCache( array( $params['src'] ) ); |
| 863 | | - wfProfileOut( __METHOD__ ); |
| 864 | | - return $status; |
| 865 | | - } |
| 866 | | - |
| 867 | | - /** |
| 868 | | - * @see FileBackendStore::deleteInternal() |
| 869 | | - */ |
| 870 | | - abstract protected function doDeleteInternal( array $params ); |
| 871 | | - |
| 872 | | - /** |
| 873 | | - * Move a file from one storage path to another in the backend. |
| 874 | | - * Do not call this function from places outside FileBackend and FileOp. |
| 875 | | - * |
| 876 | | - * $params include: |
| 877 | | - * src : source storage path |
| 878 | | - * dst : destination storage path |
| 879 | | - * overwrite : overwrite any file that exists at the destination |
| 880 | | - * |
| 881 | | - * @param $params Array |
| 882 | | - * @return Status |
| 883 | | - */ |
| 884 | | - final public function moveInternal( array $params ) { |
| 885 | | - wfProfileIn( __METHOD__ ); |
| 886 | | - $status = $this->doMoveInternal( $params ); |
| 887 | | - $this->clearCache( array( $params['src'], $params['dst'] ) ); |
| 888 | | - wfProfileOut( __METHOD__ ); |
| 889 | | - return $status; |
| 890 | | - } |
| 891 | | - |
| 892 | | - /** |
| 893 | | - * @see FileBackendStore::moveInternal() |
| 894 | | - * @return Status |
| 895 | | - */ |
| 896 | | - protected function doMoveInternal( array $params ) { |
| 897 | | - // Copy source to dest |
| 898 | | - $status = $this->copyInternal( $params ); |
| 899 | | - if ( $status->isOK() ) { |
| 900 | | - // Delete source (only fails due to races or medium going down) |
| 901 | | - $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); |
| 902 | | - $status->setResult( true, $status->value ); // ignore delete() errors |
| 903 | | - } |
| 904 | | - return $status; |
| 905 | | - } |
| 906 | | - |
| 907 | | - /** |
| 908 | | - * @see FileBackend::concatenate() |
| 909 | | - * @return Status |
| 910 | | - */ |
| 911 | | - final public function concatenate( array $params ) { |
| 912 | | - wfProfileIn( __METHOD__ ); |
| 913 | | - $status = Status::newGood(); |
| 914 | | - |
| 915 | | - // Try to lock the source files for the scope of this function |
| 916 | | - $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status ); |
| 917 | | - if ( $status->isOK() ) { |
| 918 | | - // Actually do the concatenation |
| 919 | | - $status->merge( $this->doConcatenate( $params ) ); |
| 920 | | - } |
| 921 | | - |
| 922 | | - wfProfileOut( __METHOD__ ); |
| 923 | | - return $status; |
| 924 | | - } |
| 925 | | - |
| 926 | | - /** |
| 927 | | - * @see FileBackendStore::concatenate() |
| 928 | | - * @return Status |
| 929 | | - */ |
| 930 | | - protected function doConcatenate( array $params ) { |
| 931 | | - $status = Status::newGood(); |
| 932 | | - $tmpPath = $params['dst']; // convenience |
| 933 | | - |
| 934 | | - // Check that the specified temp file is valid... |
| 935 | | - wfSuppressWarnings(); |
| 936 | | - $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) ); |
| 937 | | - wfRestoreWarnings(); |
| 938 | | - if ( !$ok ) { // not present or not empty |
| 939 | | - $status->fatal( 'backend-fail-opentemp', $tmpPath ); |
| 940 | | - return $status; |
| 941 | | - } |
| 942 | | - |
| 943 | | - // Build up the temp file using the source chunks (in order)... |
| 944 | | - $tmpHandle = fopen( $tmpPath, 'ab' ); |
| 945 | | - if ( $tmpHandle === false ) { |
| 946 | | - $status->fatal( 'backend-fail-opentemp', $tmpPath ); |
| 947 | | - return $status; |
| 948 | | - } |
| 949 | | - foreach ( $params['srcs'] as $virtualSource ) { |
| 950 | | - // Get a local FS version of the chunk |
| 951 | | - $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) ); |
| 952 | | - if ( !$tmpFile ) { |
| 953 | | - $status->fatal( 'backend-fail-read', $virtualSource ); |
| 954 | | - return $status; |
| 955 | | - } |
| 956 | | - // Get a handle to the local FS version |
| 957 | | - $sourceHandle = fopen( $tmpFile->getPath(), 'r' ); |
| 958 | | - if ( $sourceHandle === false ) { |
| 959 | | - fclose( $tmpHandle ); |
| 960 | | - $status->fatal( 'backend-fail-read', $virtualSource ); |
| 961 | | - return $status; |
| 962 | | - } |
| 963 | | - // Append chunk to file (pass chunk size to avoid magic quotes) |
| 964 | | - if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) { |
| 965 | | - fclose( $sourceHandle ); |
| 966 | | - fclose( $tmpHandle ); |
| 967 | | - $status->fatal( 'backend-fail-writetemp', $tmpPath ); |
| 968 | | - return $status; |
| 969 | | - } |
| 970 | | - fclose( $sourceHandle ); |
| 971 | | - } |
| 972 | | - if ( !fclose( $tmpHandle ) ) { |
| 973 | | - $status->fatal( 'backend-fail-closetemp', $tmpPath ); |
| 974 | | - return $status; |
| 975 | | - } |
| 976 | | - |
| 977 | | - clearstatcache(); // temp file changed |
| 978 | | - |
| 979 | | - return $status; |
| 980 | | - } |
| 981 | | - |
| 982 | | - /** |
| 983 | | - * @see FileBackend::doPrepare() |
| 984 | | - * @return Status |
| 985 | | - */ |
| 986 | | - final protected function doPrepare( array $params ) { |
| 987 | | - wfProfileIn( __METHOD__ ); |
| 988 | | - |
| 989 | | - $status = Status::newGood(); |
| 990 | | - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| 991 | | - if ( $dir === null ) { |
| 992 | | - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| 993 | | - wfProfileOut( __METHOD__ ); |
| 994 | | - return $status; // invalid storage path |
| 995 | | - } |
| 996 | | - |
| 997 | | - if ( $shard !== null ) { // confined to a single container/shard |
| 998 | | - $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); |
| 999 | | - } else { // directory is on several shards |
| 1000 | | - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| 1001 | | - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| 1002 | | - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| 1003 | | - $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| 1004 | | - } |
| 1005 | | - } |
| 1006 | | - |
| 1007 | | - wfProfileOut( __METHOD__ ); |
| 1008 | | - return $status; |
| 1009 | | - } |
| 1010 | | - |
| 1011 | | - /** |
| 1012 | | - * @see FileBackendStore::doPrepare() |
| 1013 | | - * @return Status |
| 1014 | | - */ |
| 1015 | | - protected function doPrepareInternal( $container, $dir, array $params ) { |
| 1016 | | - return Status::newGood(); |
| 1017 | | - } |
| 1018 | | - |
| 1019 | | - /** |
| 1020 | | - * @see FileBackend::doSecure() |
| 1021 | | - * @return Status |
| 1022 | | - */ |
| 1023 | | - final protected function doSecure( array $params ) { |
| 1024 | | - wfProfileIn( __METHOD__ ); |
| 1025 | | - $status = Status::newGood(); |
| 1026 | | - |
| 1027 | | - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| 1028 | | - if ( $dir === null ) { |
| 1029 | | - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| 1030 | | - wfProfileOut( __METHOD__ ); |
| 1031 | | - return $status; // invalid storage path |
| 1032 | | - } |
| 1033 | | - |
| 1034 | | - if ( $shard !== null ) { // confined to a single container/shard |
| 1035 | | - $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); |
| 1036 | | - } else { // directory is on several shards |
| 1037 | | - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| 1038 | | - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| 1039 | | - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| 1040 | | - $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| 1041 | | - } |
| 1042 | | - } |
| 1043 | | - |
| 1044 | | - wfProfileOut( __METHOD__ ); |
| 1045 | | - return $status; |
| 1046 | | - } |
| 1047 | | - |
| 1048 | | - /** |
| 1049 | | - * @see FileBackendStore::doSecure() |
| 1050 | | - * @return Status |
| 1051 | | - */ |
| 1052 | | - protected function doSecureInternal( $container, $dir, array $params ) { |
| 1053 | | - return Status::newGood(); |
| 1054 | | - } |
| 1055 | | - |
| 1056 | | - /** |
| 1057 | | - * @see FileBackend::doClean() |
| 1058 | | - * @return Status |
| 1059 | | - */ |
| 1060 | | - final protected function doClean( array $params ) { |
| 1061 | | - wfProfileIn( __METHOD__ ); |
| 1062 | | - $status = Status::newGood(); |
| 1063 | | - |
| 1064 | | - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| 1065 | | - if ( $dir === null ) { |
| 1066 | | - $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); |
| 1067 | | - wfProfileOut( __METHOD__ ); |
| 1068 | | - return $status; // invalid storage path |
| 1069 | | - } |
| 1070 | | - |
| 1071 | | - // Attempt to lock this directory... |
| 1072 | | - $filesLockEx = array( $params['dir'] ); |
| 1073 | | - $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); |
| 1074 | | - if ( !$status->isOK() ) { |
| 1075 | | - wfProfileOut( __METHOD__ ); |
| 1076 | | - return $status; // abort |
| 1077 | | - } |
| 1078 | | - |
| 1079 | | - if ( $shard !== null ) { // confined to a single container/shard |
| 1080 | | - $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) ); |
| 1081 | | - } else { // directory is on several shards |
| 1082 | | - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| 1083 | | - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| 1084 | | - foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { |
| 1085 | | - $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); |
| 1086 | | - } |
| 1087 | | - } |
| 1088 | | - |
| 1089 | | - wfProfileOut( __METHOD__ ); |
| 1090 | | - return $status; |
| 1091 | | - } |
| 1092 | | - |
| 1093 | | - /** |
| 1094 | | - * @see FileBackendStore::doClean() |
| 1095 | | - * @return Status |
| 1096 | | - */ |
| 1097 | | - protected function doCleanInternal( $container, $dir, array $params ) { |
| 1098 | | - return Status::newGood(); |
| 1099 | | - } |
| 1100 | | - |
| 1101 | | - /** |
| 1102 | | - * @see FileBackend::fileExists() |
| 1103 | | - * @return bool|null |
| 1104 | | - */ |
| 1105 | | - final public function fileExists( array $params ) { |
| 1106 | | - wfProfileIn( __METHOD__ ); |
| 1107 | | - $stat = $this->getFileStat( $params ); |
| 1108 | | - wfProfileOut( __METHOD__ ); |
| 1109 | | - return ( $stat === null ) ? null : (bool)$stat; // null => failure |
| 1110 | | - } |
| 1111 | | - |
| 1112 | | - /** |
| 1113 | | - * @see FileBackend::getFileTimestamp() |
| 1114 | | - * @return bool |
| 1115 | | - */ |
| 1116 | | - final public function getFileTimestamp( array $params ) { |
| 1117 | | - wfProfileIn( __METHOD__ ); |
| 1118 | | - $stat = $this->getFileStat( $params ); |
| 1119 | | - wfProfileOut( __METHOD__ ); |
| 1120 | | - return $stat ? $stat['mtime'] : false; |
| 1121 | | - } |
| 1122 | | - |
| 1123 | | - /** |
| 1124 | | - * @see FileBackend::getFileSize() |
| 1125 | | - * @return bool |
| 1126 | | - */ |
| 1127 | | - final public function getFileSize( array $params ) { |
| 1128 | | - wfProfileIn( __METHOD__ ); |
| 1129 | | - $stat = $this->getFileStat( $params ); |
| 1130 | | - wfProfileOut( __METHOD__ ); |
| 1131 | | - return $stat ? $stat['size'] : false; |
| 1132 | | - } |
| 1133 | | - |
| 1134 | | - /** |
| 1135 | | - * @see FileBackend::getFileStat() |
| 1136 | | - * @return bool |
| 1137 | | - */ |
| 1138 | | - final public function getFileStat( array $params ) { |
| 1139 | | - wfProfileIn( __METHOD__ ); |
| 1140 | | - $path = self::normalizeStoragePath( $params['src'] ); |
| 1141 | | - if ( $path === null ) { |
| 1142 | | - wfProfileOut( __METHOD__ ); |
| 1143 | | - return false; // invalid storage path |
| 1144 | | - } |
| 1145 | | - $latest = !empty( $params['latest'] ); |
| 1146 | | - if ( isset( $this->cache[$path]['stat'] ) ) { |
| 1147 | | - // If we want the latest data, check that this cached |
| 1148 | | - // value was in fact fetched with the latest available data. |
| 1149 | | - if ( !$latest || $this->cache[$path]['stat']['latest'] ) { |
| 1150 | | - wfProfileOut( __METHOD__ ); |
| 1151 | | - return $this->cache[$path]['stat']; |
| 1152 | | - } |
| 1153 | | - } |
| 1154 | | - wfProfileIn( __METHOD__ . '-miss' ); |
| 1155 | | - $stat = $this->doGetFileStat( $params ); |
| 1156 | | - wfProfileOut( __METHOD__ . '-miss' ); |
| 1157 | | - if ( is_array( $stat ) ) { // don't cache negatives |
| 1158 | | - $this->trimCache(); // limit memory |
| 1159 | | - $this->cache[$path]['stat'] = $stat; |
| 1160 | | - $this->cache[$path]['stat']['latest'] = $latest; |
| 1161 | | - } |
| 1162 | | - wfProfileOut( __METHOD__ ); |
| 1163 | | - return $stat; |
| 1164 | | - } |
| 1165 | | - |
| 1166 | | - /** |
| 1167 | | - * @see FileBackendStore::getFileStat() |
| 1168 | | - */ |
| 1169 | | - abstract protected function doGetFileStat( array $params ); |
| 1170 | | - |
| 1171 | | - /** |
| 1172 | | - * @see FileBackend::getFileContents() |
| 1173 | | - * @return bool|string |
| 1174 | | - */ |
| 1175 | | - public function getFileContents( array $params ) { |
| 1176 | | - wfProfileIn( __METHOD__ ); |
| 1177 | | - $tmpFile = $this->getLocalReference( $params ); |
| 1178 | | - if ( !$tmpFile ) { |
| 1179 | | - wfProfileOut( __METHOD__ ); |
| 1180 | | - return false; |
| 1181 | | - } |
| 1182 | | - wfSuppressWarnings(); |
| 1183 | | - $data = file_get_contents( $tmpFile->getPath() ); |
| 1184 | | - wfRestoreWarnings(); |
| 1185 | | - wfProfileOut( __METHOD__ ); |
| 1186 | | - return $data; |
| 1187 | | - } |
| 1188 | | - |
| 1189 | | - /** |
| 1190 | | - * @see FileBackend::getFileSha1Base36() |
| 1191 | | - * @return bool|string |
| 1192 | | - */ |
| 1193 | | - final public function getFileSha1Base36( array $params ) { |
| 1194 | | - wfProfileIn( __METHOD__ ); |
| 1195 | | - $path = $params['src']; |
| 1196 | | - if ( isset( $this->cache[$path]['sha1'] ) ) { |
| 1197 | | - wfProfileOut( __METHOD__ ); |
| 1198 | | - return $this->cache[$path]['sha1']; |
| 1199 | | - } |
| 1200 | | - wfProfileIn( __METHOD__ . '-miss' ); |
| 1201 | | - $hash = $this->doGetFileSha1Base36( $params ); |
| 1202 | | - wfProfileOut( __METHOD__ . '-miss' ); |
| 1203 | | - if ( $hash ) { // don't cache negatives |
| 1204 | | - $this->trimCache(); // limit memory |
| 1205 | | - $this->cache[$path]['sha1'] = $hash; |
| 1206 | | - } |
| 1207 | | - wfProfileOut( __METHOD__ ); |
| 1208 | | - return $hash; |
| 1209 | | - } |
| 1210 | | - |
| 1211 | | - /** |
| 1212 | | - * @see FileBackendStore::getFileSha1Base36() |
| 1213 | | - * @return bool |
| 1214 | | - */ |
| 1215 | | - protected function doGetFileSha1Base36( array $params ) { |
| 1216 | | - $fsFile = $this->getLocalReference( $params ); |
| 1217 | | - if ( !$fsFile ) { |
| 1218 | | - return false; |
| 1219 | | - } else { |
| 1220 | | - return $fsFile->getSha1Base36(); |
| 1221 | | - } |
| 1222 | | - } |
| 1223 | | - |
| 1224 | | - /** |
| 1225 | | - * @see FileBackend::getFileProps() |
| 1226 | | - * @return Array |
| 1227 | | - */ |
| 1228 | | - final public function getFileProps( array $params ) { |
| 1229 | | - wfProfileIn( __METHOD__ ); |
| 1230 | | - $fsFile = $this->getLocalReference( $params ); |
| 1231 | | - $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); |
| 1232 | | - wfProfileOut( __METHOD__ ); |
| 1233 | | - return $props; |
| 1234 | | - } |
| 1235 | | - |
| 1236 | | - /** |
| 1237 | | - * @see FileBackend::getLocalReference() |
| 1238 | | - * @return TempFSFile|null |
| 1239 | | - */ |
| 1240 | | - public function getLocalReference( array $params ) { |
| 1241 | | - wfProfileIn( __METHOD__ ); |
| 1242 | | - $path = $params['src']; |
| 1243 | | - if ( isset( $this->expensiveCache[$path]['localRef'] ) ) { |
| 1244 | | - wfProfileOut( __METHOD__ ); |
| 1245 | | - return $this->expensiveCache[$path]['localRef']; |
| 1246 | | - } |
| 1247 | | - $tmpFile = $this->getLocalCopy( $params ); |
| 1248 | | - if ( $tmpFile ) { // don't cache negatives |
| 1249 | | - $this->trimExpensiveCache(); // limit memory |
| 1250 | | - $this->expensiveCache[$path]['localRef'] = $tmpFile; |
| 1251 | | - } |
| 1252 | | - wfProfileOut( __METHOD__ ); |
| 1253 | | - return $tmpFile; |
| 1254 | | - } |
| 1255 | | - |
| 1256 | | - /** |
| 1257 | | - * @see FileBackend::streamFile() |
| 1258 | | - * @return Status |
| 1259 | | - */ |
| 1260 | | - final public function streamFile( array $params ) { |
| 1261 | | - wfProfileIn( __METHOD__ ); |
| 1262 | | - $status = Status::newGood(); |
| 1263 | | - |
| 1264 | | - $info = $this->getFileStat( $params ); |
| 1265 | | - if ( !$info ) { // let StreamFile handle the 404 |
| 1266 | | - $status->fatal( 'backend-fail-notexists', $params['src'] ); |
| 1267 | | - } |
| 1268 | | - |
| 1269 | | - // Set output buffer and HTTP headers for stream |
| 1270 | | - $extraHeaders = isset( $params['headers'] ) ? $params['headers'] : array(); |
| 1271 | | - $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders ); |
| 1272 | | - if ( $res == StreamFile::NOT_MODIFIED ) { |
| 1273 | | - // do nothing; client cache is up to date |
| 1274 | | - } elseif ( $res == StreamFile::READY_STREAM ) { |
| 1275 | | - wfProfileIn( __METHOD__ . '-send' ); |
| 1276 | | - $status = $this->doStreamFile( $params ); |
| 1277 | | - wfProfileOut( __METHOD__ . '-send' ); |
| 1278 | | - } else { |
| 1279 | | - $status->fatal( 'backend-fail-stream', $params['src'] ); |
| 1280 | | - } |
| 1281 | | - |
| 1282 | | - wfProfileOut( __METHOD__ ); |
| 1283 | | - return $status; |
| 1284 | | - } |
| 1285 | | - |
| 1286 | | - /** |
| 1287 | | - * @see FileBackendStore::streamFile() |
| 1288 | | - * @return Status |
| 1289 | | - */ |
| 1290 | | - protected function doStreamFile( array $params ) { |
| 1291 | | - $status = Status::newGood(); |
| 1292 | | - |
| 1293 | | - $fsFile = $this->getLocalReference( $params ); |
| 1294 | | - if ( !$fsFile ) { |
| 1295 | | - $status->fatal( 'backend-fail-stream', $params['src'] ); |
| 1296 | | - } elseif ( !readfile( $fsFile->getPath() ) ) { |
| 1297 | | - $status->fatal( 'backend-fail-stream', $params['src'] ); |
| 1298 | | - } |
| 1299 | | - |
| 1300 | | - return $status; |
| 1301 | | - } |
| 1302 | | - |
| 1303 | | - /** |
| 1304 | | - * @copydoc FileBackend::getFileList() |
| 1305 | | - * @return Array|null|Traversable |
| 1306 | | - */ |
| 1307 | | - final public function getFileList( array $params ) { |
| 1308 | | - list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); |
| 1309 | | - if ( $dir === null ) { // invalid storage path |
| 1310 | | - return null; |
| 1311 | | - } |
| 1312 | | - if ( $shard !== null ) { |
| 1313 | | - // File listing is confined to a single container/shard |
| 1314 | | - return $this->getFileListInternal( $fullCont, $dir, $params ); |
| 1315 | | - } else { |
| 1316 | | - wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); |
| 1317 | | - // File listing spans multiple containers/shards |
| 1318 | | - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); |
| 1319 | | - return new FileBackendStoreShardListIterator( $this, |
| 1320 | | - $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); |
| 1321 | | - } |
| 1322 | | - } |
| 1323 | | - |
| 1324 | | - /** |
| 1325 | | - * Do not call this function from places outside FileBackend |
| 1326 | | - * |
| 1327 | | - * @see FileBackendStore::getFileList() |
| 1328 | | - * |
| 1329 | | - * @param $container string Resolved container name |
| 1330 | | - * @param $dir string Resolved path relative to container |
| 1331 | | - * @param $params Array |
| 1332 | | - * @return Traversable|Array|null |
| 1333 | | - */ |
| 1334 | | - abstract public function getFileListInternal( $container, $dir, array $params ); |
| 1335 | | - |
| 1336 | | - /** |
| 1337 | | - * Get the list of supported operations and their corresponding FileOp classes. |
| 1338 | | - * |
| 1339 | | - * @return Array |
| 1340 | | - */ |
| 1341 | | - protected function supportedOperations() { |
| 1342 | | - return array( |
| 1343 | | - 'store' => 'StoreFileOp', |
| 1344 | | - 'copy' => 'CopyFileOp', |
| 1345 | | - 'move' => 'MoveFileOp', |
| 1346 | | - 'delete' => 'DeleteFileOp', |
| 1347 | | - 'create' => 'CreateFileOp', |
| 1348 | | - 'null' => 'NullFileOp' |
| 1349 | | - ); |
| 1350 | | - } |
| 1351 | | - |
| 1352 | | - /** |
| 1353 | | - * Return a list of FileOp objects from a list of operations. |
| 1354 | | - * Do not call this function from places outside FileBackend. |
| 1355 | | - * |
| 1356 | | - * The result must have the same number of items as the input. |
| 1357 | | - * An exception is thrown if an unsupported operation is requested. |
| 1358 | | - * |
| 1359 | | - * @param $ops Array Same format as doOperations() |
| 1360 | | - * @return Array List of FileOp objects |
| 1361 | | - * @throws MWException |
| 1362 | | - */ |
| 1363 | | - final public function getOperations( array $ops ) { |
| 1364 | | - $supportedOps = $this->supportedOperations(); |
| 1365 | | - |
| 1366 | | - $performOps = array(); // array of FileOp objects |
| 1367 | | - // Build up ordered array of FileOps... |
| 1368 | | - foreach ( $ops as $operation ) { |
| 1369 | | - $opName = $operation['op']; |
| 1370 | | - if ( isset( $supportedOps[$opName] ) ) { |
| 1371 | | - $class = $supportedOps[$opName]; |
| 1372 | | - // Get params for this operation |
| 1373 | | - $params = $operation; |
| 1374 | | - // Append the FileOp class |
| 1375 | | - $performOps[] = new $class( $this, $params ); |
| 1376 | | - } else { |
| 1377 | | - throw new MWException( "Operation `$opName` is not supported." ); |
| 1378 | | - } |
| 1379 | | - } |
| 1380 | | - |
| 1381 | | - return $performOps; |
| 1382 | | - } |
| 1383 | | - |
| 1384 | | - /** |
| 1385 | | - * @see FileBackend::doOperationsInternal() |
| 1386 | | - * @return Status |
| 1387 | | - */ |
| 1388 | | - protected function doOperationsInternal( array $ops, array $opts ) { |
| 1389 | | - wfProfileIn( __METHOD__ ); |
| 1390 | | - $status = Status::newGood(); |
| 1391 | | - |
| 1392 | | - // Build up a list of FileOps... |
| 1393 | | - $performOps = $this->getOperations( $ops ); |
| 1394 | | - |
| 1395 | | - // Acquire any locks as needed... |
| 1396 | | - if ( empty( $opts['nonLocking'] ) ) { |
| 1397 | | - // Build up a list of files to lock... |
| 1398 | | - $filesLockEx = $filesLockSh = array(); |
| 1399 | | - foreach ( $performOps as $fileOp ) { |
| 1400 | | - $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() ); |
| 1401 | | - $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() ); |
| 1402 | | - } |
| 1403 | | - // Optimization: if doing an EX lock anyway, don't also set an SH one |
| 1404 | | - $filesLockSh = array_diff( $filesLockSh, $filesLockEx ); |
| 1405 | | - // Get a shared lock on the parent directory of each path changed |
| 1406 | | - $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) ); |
| 1407 | | - // Try to lock those files for the scope of this function... |
| 1408 | | - $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status ); |
| 1409 | | - $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); |
| 1410 | | - if ( !$status->isOK() ) { |
| 1411 | | - wfProfileOut( __METHOD__ ); |
| 1412 | | - return $status; // abort |
| 1413 | | - } |
| 1414 | | - } |
| 1415 | | - |
| 1416 | | - // Clear any cache entries (after locks acquired) |
| 1417 | | - $this->clearCache(); |
| 1418 | | - |
| 1419 | | - // Actually attempt the operation batch... |
| 1420 | | - $subStatus = FileOp::attemptBatch( $performOps, $opts ); |
| 1421 | | - |
| 1422 | | - // Merge errors into status fields |
| 1423 | | - $status->merge( $subStatus ); |
| 1424 | | - $status->success = $subStatus->success; // not done in merge() |
| 1425 | | - |
| 1426 | | - wfProfileOut( __METHOD__ ); |
| 1427 | | - return $status; |
| 1428 | | - } |
| 1429 | | - |
| 1430 | | - /** |
| 1431 | | - * @see FileBackend::clearCache() |
| 1432 | | - */ |
| 1433 | | - final public function clearCache( array $paths = null ) { |
| 1434 | | - if ( is_array( $paths ) ) { |
| 1435 | | - $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); |
| 1436 | | - $paths = array_filter( $paths, 'strlen' ); // remove nulls |
| 1437 | | - } |
| 1438 | | - if ( $paths === null ) { |
| 1439 | | - $this->cache = array(); |
| 1440 | | - $this->expensiveCache = array(); |
| 1441 | | - } else { |
| 1442 | | - foreach ( $paths as $path ) { |
| 1443 | | - unset( $this->cache[$path] ); |
| 1444 | | - unset( $this->expensiveCache[$path] ); |
| 1445 | | - } |
| 1446 | | - } |
| 1447 | | - $this->doClearCache( $paths ); |
| 1448 | | - } |
| 1449 | | - |
| 1450 | | - /** |
| 1451 | | - * Clears any additional stat caches for storage paths |
| 1452 | | - * |
| 1453 | | - * @see FileBackend::clearCache() |
| 1454 | | - * |
| 1455 | | - * @param $paths Array Storage paths (optional) |
| 1456 | | - * @return void |
| 1457 | | - */ |
| 1458 | | - protected function doClearCache( array $paths = null ) {} |
| 1459 | | - |
| 1460 | | - /** |
| 1461 | | - * Prune the inexpensive cache if it is too big to add an item |
| 1462 | | - * |
| 1463 | | - * @return void |
| 1464 | | - */ |
| 1465 | | - protected function trimCache() { |
| 1466 | | - if ( count( $this->cache ) >= $this->maxCacheSize ) { |
| 1467 | | - reset( $this->cache ); |
| 1468 | | - unset( $this->cache[key( $this->cache )] ); |
| 1469 | | - } |
| 1470 | | - } |
| 1471 | | - |
| 1472 | | - /** |
| 1473 | | - * Prune the expensive cache if it is too big to add an item |
| 1474 | | - * |
| 1475 | | - * @return void |
| 1476 | | - */ |
| 1477 | | - protected function trimExpensiveCache() { |
| 1478 | | - if ( count( $this->expensiveCache ) >= $this->maxExpensiveCacheSize ) { |
| 1479 | | - reset( $this->expensiveCache ); |
| 1480 | | - unset( $this->expensiveCache[key( $this->expensiveCache )] ); |
| 1481 | | - } |
| 1482 | | - } |
| 1483 | | - |
| 1484 | | - /** |
| 1485 | | - * Check if a container name is valid. |
| 1486 | | - * This checks for for length and illegal characters. |
| 1487 | | - * |
| 1488 | | - * @param $container string |
| 1489 | | - * @return bool |
| 1490 | | - */ |
| 1491 | | - final protected static function isValidContainerName( $container ) { |
| 1492 | | - // This accounts for Swift and S3 restrictions while leaving room |
| 1493 | | - // for things like '.xxx' (hex shard chars) or '.seg' (segments). |
| 1494 | | - // This disallows directory separators or traversal characters. |
| 1495 | | - // Note that matching strings URL encode to the same string; |
| 1496 | | - // in Swift, the length restriction is *after* URL encoding. |
| 1497 | | - return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container ); |
| 1498 | | - } |
| 1499 | | - |
| 1500 | | - /** |
| 1501 | | - * Splits a storage path into an internal container name, |
| 1502 | | - * an internal relative file name, and a container shard suffix. |
| 1503 | | - * Any shard suffix is already appended to the internal container name. |
| 1504 | | - * This also checks that the storage path is valid and within this backend. |
| 1505 | | - * |
| 1506 | | - * If the container is sharded but a suffix could not be determined, |
| 1507 | | - * this means that the path can only refer to a directory and can only |
| 1508 | | - * be scanned by looking in all the container shards. |
| 1509 | | - * |
| 1510 | | - * @param $storagePath string |
| 1511 | | - * @return Array (container, path, container suffix) or (null, null, null) if invalid |
| 1512 | | - */ |
| 1513 | | - final protected function resolveStoragePath( $storagePath ) { |
| 1514 | | - list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath ); |
| 1515 | | - if ( $backend === $this->name ) { // must be for this backend |
| 1516 | | - $relPath = self::normalizeContainerPath( $relPath ); |
| 1517 | | - if ( $relPath !== null ) { |
| 1518 | | - // Get shard for the normalized path if this container is sharded |
| 1519 | | - $cShard = $this->getContainerShard( $container, $relPath ); |
| 1520 | | - // Validate and sanitize the relative path (backend-specific) |
| 1521 | | - $relPath = $this->resolveContainerPath( $container, $relPath ); |
| 1522 | | - if ( $relPath !== null ) { |
| 1523 | | - // Prepend any wiki ID prefix to the container name |
| 1524 | | - $container = $this->fullContainerName( $container ); |
| 1525 | | - if ( self::isValidContainerName( $container ) ) { |
| 1526 | | - // Validate and sanitize the container name (backend-specific) |
| 1527 | | - $container = $this->resolveContainerName( "{$container}{$cShard}" ); |
| 1528 | | - if ( $container !== null ) { |
| 1529 | | - return array( $container, $relPath, $cShard ); |
| 1530 | | - } |
| 1531 | | - } |
| 1532 | | - } |
| 1533 | | - } |
| 1534 | | - } |
| 1535 | | - return array( null, null, null ); |
| 1536 | | - } |
| 1537 | | - |
| 1538 | | - /** |
| 1539 | | - * Like resolveStoragePath() except null values are returned if |
| 1540 | | - * the container is sharded and the shard could not be determined. |
| 1541 | | - * |
| 1542 | | - * @see FileBackendStore::resolveStoragePath() |
| 1543 | | - * |
| 1544 | | - * @param $storagePath string |
| 1545 | | - * @return Array (container, path) or (null, null) if invalid |
| 1546 | | - */ |
| 1547 | | - final protected function resolveStoragePathReal( $storagePath ) { |
| 1548 | | - list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); |
| 1549 | | - if ( $cShard !== null ) { |
| 1550 | | - return array( $container, $relPath ); |
| 1551 | | - } |
| 1552 | | - return array( null, null ); |
| 1553 | | - } |
| 1554 | | - |
| 1555 | | - /** |
| 1556 | | - * Get the container name shard suffix for a given path. |
| 1557 | | - * Any empty suffix means the container is not sharded. |
| 1558 | | - * |
| 1559 | | - * @param $container string Container name |
| 1560 | | - * @param $relStoragePath string Storage path relative to the container |
| 1561 | | - * @return string|null Returns null if shard could not be determined |
| 1562 | | - */ |
| 1563 | | - final protected function getContainerShard( $container, $relPath ) { |
| 1564 | | - list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container ); |
| 1565 | | - if ( $levels == 1 || $levels == 2 ) { |
| 1566 | | - // Hash characters are either base 16 or 36 |
| 1567 | | - $char = ( $base == 36 ) ? '[0-9a-z]' : '[0-9a-f]'; |
| 1568 | | - // Get a regex that represents the shard portion of paths. |
| 1569 | | - // The concatenation of the captures gives us the shard. |
| 1570 | | - if ( $levels === 1 ) { // 16 or 36 shards per container |
| 1571 | | - $hashDirRegex = '(' . $char . ')'; |
| 1572 | | - } else { // 256 or 1296 shards per container |
| 1573 | | - if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc") |
| 1574 | | - $hashDirRegex = $char . '/(' . $char . '{2})'; |
| 1575 | | - } else { // short hash dir format (e.g. "a/b/c") |
| 1576 | | - $hashDirRegex = '(' . $char . ')/(' . $char . ')'; |
| 1577 | | - } |
| 1578 | | - } |
| 1579 | | - // Allow certain directories to be above the hash dirs so as |
| 1580 | | - // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab"). |
| 1581 | | - // They must be 2+ chars to avoid any hash directory ambiguity. |
| 1582 | | - $m = array(); |
| 1583 | | - if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) { |
| 1584 | | - return '.' . implode( '', array_slice( $m, 1 ) ); |
| 1585 | | - } |
| 1586 | | - return null; // failed to match |
| 1587 | | - } |
| 1588 | | - return ''; // no sharding |
| 1589 | | - } |
| 1590 | | - |
| 1591 | | - /** |
| 1592 | | - * Get the sharding config for a container. |
| 1593 | | - * If greater than 0, then all file storage paths within |
| 1594 | | - * the container are required to be hashed accordingly. |
| 1595 | | - * |
| 1596 | | - * @param $container string |
| 1597 | | - * @return Array (integer levels, integer base, repeat flag) or (0, 0, false) |
| 1598 | | - */ |
| 1599 | | - final protected function getContainerHashLevels( $container ) { |
| 1600 | | - if ( isset( $this->shardViaHashLevels[$container] ) ) { |
| 1601 | | - $config = $this->shardViaHashLevels[$container]; |
| 1602 | | - $hashLevels = (int)$config['levels']; |
| 1603 | | - if ( $hashLevels == 1 || $hashLevels == 2 ) { |
| 1604 | | - $hashBase = (int)$config['base']; |
| 1605 | | - if ( $hashBase == 16 || $hashBase == 36 ) { |
| 1606 | | - return array( $hashLevels, $hashBase, $config['repeat'] ); |
| 1607 | | - } |
| 1608 | | - } |
| 1609 | | - } |
| 1610 | | - return array( 0, 0, false ); // no sharding |
| 1611 | | - } |
| 1612 | | - |
| 1613 | | - /** |
| 1614 | | - * Get a list of full container shard suffixes for a container |
| 1615 | | - * |
| 1616 | | - * @param $container string |
| 1617 | | - * @return Array |
| 1618 | | - */ |
| 1619 | | - final protected function getContainerSuffixes( $container ) { |
| 1620 | | - $shards = array(); |
| 1621 | | - list( $digits, $base ) = $this->getContainerHashLevels( $container ); |
| 1622 | | - if ( $digits > 0 ) { |
| 1623 | | - $numShards = pow( $base, $digits ); |
| 1624 | | - for ( $index = 0; $index < $numShards; $index++ ) { |
| 1625 | | - $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits ); |
| 1626 | | - } |
| 1627 | | - } |
| 1628 | | - return $shards; |
| 1629 | | - } |
| 1630 | | - |
| 1631 | | - /** |
| 1632 | | - * Get the full container name, including the wiki ID prefix |
| 1633 | | - * |
| 1634 | | - * @param $container string |
| 1635 | | - * @return string |
| 1636 | | - */ |
| 1637 | | - final protected function fullContainerName( $container ) { |
| 1638 | | - if ( $this->wikiId != '' ) { |
| 1639 | | - return "{$this->wikiId}-$container"; |
| 1640 | | - } else { |
| 1641 | | - return $container; |
| 1642 | | - } |
| 1643 | | - } |
| 1644 | | - |
| 1645 | | - /** |
| 1646 | | - * Resolve a container name, checking if it's allowed by the backend. |
| 1647 | | - * This is intended for internal use, such as encoding illegal chars. |
| 1648 | | - * Subclasses can override this to be more restrictive. |
| 1649 | | - * |
| 1650 | | - * @param $container string |
| 1651 | | - * @return string|null |
| 1652 | | - */ |
| 1653 | | - protected function resolveContainerName( $container ) { |
| 1654 | | - return $container; |
| 1655 | | - } |
| 1656 | | - |
| 1657 | | - /** |
| 1658 | | - * Resolve a relative storage path, checking if it's allowed by the backend. |
| 1659 | | - * This is intended for internal use, such as encoding illegal chars or perhaps |
| 1660 | | - * getting absolute paths (e.g. FS based backends). Note that the relative path |
| 1661 | | - * may be the empty string (e.g. the path is simply to the container). |
| 1662 | | - * |
| 1663 | | - * @param $container string Container name |
| 1664 | | - * @param $relStoragePath string Storage path relative to the container |
| 1665 | | - * @return string|null Path or null if not valid |
| 1666 | | - */ |
| 1667 | | - protected function resolveContainerPath( $container, $relStoragePath ) { |
| 1668 | | - return $relStoragePath; |
| 1669 | | - } |
| 1670 | | -} |
| 1671 | | - |
| 1672 | | -/** |
| 1673 | | - * FileBackendStore helper function to handle file listings that span container shards. |
| 1674 | | - * Do not use this class from places outside of FileBackendStore. |
| 1675 | | - * |
| 1676 | | - * @ingroup FileBackend |
| 1677 | | - */ |
| 1678 | | -class FileBackendStoreShardListIterator implements Iterator { |
| 1679 | | - /* @var FileBackendStore */ |
| 1680 | | - protected $backend; |
| 1681 | | - /* @var Array */ |
| 1682 | | - protected $params; |
| 1683 | | - /* @var Array */ |
| 1684 | | - protected $shardSuffixes; |
| 1685 | | - protected $container; // string |
| 1686 | | - protected $directory; // string |
| 1687 | | - |
| 1688 | | - /* @var Traversable */ |
| 1689 | | - protected $iter; |
| 1690 | | - protected $curShard = 0; // integer |
| 1691 | | - protected $pos = 0; // integer |
| 1692 | | - |
| 1693 | | - /** |
| 1694 | | - * @param $backend FileBackendStore |
| 1695 | | - * @param $container string Full storage container name |
| 1696 | | - * @param $dir string Storage directory relative to container |
| 1697 | | - * @param $suffixes Array List of container shard suffixes |
| 1698 | | - * @param $params Array |
| 1699 | | - */ |
| 1700 | | - public function __construct( |
| 1701 | | - FileBackendStore $backend, $container, $dir, array $suffixes, array $params |
| 1702 | | - ) { |
| 1703 | | - $this->backend = $backend; |
| 1704 | | - $this->container = $container; |
| 1705 | | - $this->directory = $dir; |
| 1706 | | - $this->shardSuffixes = $suffixes; |
| 1707 | | - $this->params = $params; |
| 1708 | | - } |
| 1709 | | - |
| 1710 | | - public function current() { |
| 1711 | | - if ( is_array( $this->iter ) ) { |
| 1712 | | - return current( $this->iter ); |
| 1713 | | - } else { |
| 1714 | | - return $this->iter->current(); |
| 1715 | | - } |
| 1716 | | - } |
| 1717 | | - |
| 1718 | | - public function key() { |
| 1719 | | - return $this->pos; |
| 1720 | | - } |
| 1721 | | - |
| 1722 | | - public function next() { |
| 1723 | | - ++$this->pos; |
| 1724 | | - if ( is_array( $this->iter ) ) { |
| 1725 | | - next( $this->iter ); |
| 1726 | | - } else { |
| 1727 | | - $this->iter->next(); |
| 1728 | | - } |
| 1729 | | - // Find the next non-empty shard if no elements are left |
| 1730 | | - $this->nextShardIteratorIfNotValid(); |
| 1731 | | - } |
| 1732 | | - |
| 1733 | | - /** |
| 1734 | | - * If the iterator for this container shard is out of items, |
| 1735 | | - * then move on to the next container that has items. |
| 1736 | | - * If there are none, then it advances to the last container. |
| 1737 | | - */ |
| 1738 | | - protected function nextShardIteratorIfNotValid() { |
| 1739 | | - while ( !$this->valid() ) { |
| 1740 | | - if ( ++$this->curShard >= count( $this->shardSuffixes ) ) { |
| 1741 | | - break; // no more container shards |
| 1742 | | - } |
| 1743 | | - $this->setIteratorFromCurrentShard(); |
| 1744 | | - } |
| 1745 | | - } |
| 1746 | | - |
| 1747 | | - protected function setIteratorFromCurrentShard() { |
| 1748 | | - $suffix = $this->shardSuffixes[$this->curShard]; |
| 1749 | | - $this->iter = $this->backend->getFileListInternal( |
| 1750 | | - "{$this->container}{$suffix}", $this->directory, $this->params ); |
| 1751 | | - } |
| 1752 | | - |
| 1753 | | - public function rewind() { |
| 1754 | | - $this->pos = 0; |
| 1755 | | - $this->curShard = 0; |
| 1756 | | - $this->setIteratorFromCurrentShard(); |
| 1757 | | - // Find the next non-empty shard if this one has no elements |
| 1758 | | - $this->nextShardIteratorIfNotValid(); |
| 1759 | | - } |
| 1760 | | - |
| 1761 | | - public function valid() { |
| 1762 | | - if ( $this->iter == null ) { |
| 1763 | | - return false; // some failure? |
| 1764 | | - } elseif ( is_array( $this->iter ) ) { |
| 1765 | | - return ( current( $this->iter ) !== false ); // no paths can have this value |
| 1766 | | - } else { |
| 1767 | | - return $this->iter->valid(); |
| 1768 | | - } |
| 1769 | | - } |
| 1770 | | -} |
| Index: trunk/phase3/includes/AutoLoader.php |
| — | — | @@ -500,9 +500,9 @@ |
| 501 | 501 | # includes/filerepo/backend |
| 502 | 502 | 'FileBackendGroup' => 'includes/filerepo/backend/FileBackendGroup.php', |
| 503 | 503 | 'FileBackend' => 'includes/filerepo/backend/FileBackend.php', |
| 504 | | - 'FileBackendStore' => 'includes/filerepo/backend/FileBackend.php', |
| | 504 | + 'FileBackendStore' => 'includes/filerepo/backend/FileBackendStore.php', |
| | 505 | + 'FileBackendStoreShardListIterator' => 'includes/filerepo/backend/FileBackendStore.php', |
| 505 | 506 | 'FileBackendMultiWrite' => 'includes/filerepo/backend/FileBackendMultiWrite.php', |
| 506 | | - 'FileBackendStoreShardListIterator' => 'includes/filerepo/backend/FileBackend.php', |
| 507 | 507 | 'FSFileBackend' => 'includes/filerepo/backend/FSFileBackend.php', |
| 508 | 508 | 'FSFileBackendFileList' => 'includes/filerepo/backend/FSFileBackend.php', |
| 509 | 509 | 'SwiftFileBackend' => 'includes/filerepo/backend/SwiftFileBackend.php', |