| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php |
| — | — | @@ -5,8 +5,8 @@ |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | /** |
| 9 | | - * This class defines the methods as abstract that should be |
| 10 | | - * implemented in child classes that represent a mutli-write backend. |
| | 9 | + * This class defines a multi-write backend. Multiple backends can |
| | 10 | + * be registered to this proxy backend it will act as a single backend. |
| 11 | 11 | * |
| 12 | 12 | * The order that the backends are defined sets the priority of which |
| 13 | 13 | * backend is read from or written to first. Functions like fileExists() |
| — | — | @@ -17,10 +17,10 @@ |
| 18 | 18 | * If an operation fails on one backend it will be rolled back from the others. |
| 19 | 19 | * |
| 20 | 20 | * To avoid excess overhead, set the the highest priority backend to use |
| 21 | | - * a generic FileLockManager and the others to use NullLockManager. |
| | 21 | + * a generic FileLockManager and the others to use NullLockManager. This can |
| | 22 | + * only be done if all access to the backends is through an instance of this class. |
| 22 | 23 | */ |
| 23 | | -class FileBackendMultiWrite implements IFileBackend { |
| 24 | | - protected $name; |
| | 24 | +class FileBackendMultiWrite extends FileBackendBase { |
| 25 | 25 | /** @var Array Prioritized list of FileBackend classes */ |
| 26 | 26 | protected $fileBackends = array(); |
| 27 | 27 | |
| — | — | @@ -29,10 +29,6 @@ |
| 30 | 30 | $this->fileBackends = $config['backends']; |
| 31 | 31 | } |
| 32 | 32 | |
| 33 | | - function getName() { |
| 34 | | - return $this->name; |
| 35 | | - } |
| 36 | | - |
| 37 | 33 | function hasNativeMove() { |
| 38 | 34 | return true; // this is irrelevant |
| 39 | 35 | } |
| — | — | @@ -144,8 +140,8 @@ |
| 145 | 141 | $status = $backend->streamFile( $params ); |
| 146 | 142 | if ( $status->isOK() ) { |
| 147 | 143 | return $status; |
| 148 | | - } else { |
| 149 | | - // @TODO: check if we failed mid-stream and return out if so |
| | 144 | + } elseif ( headers_sent() ) { |
| | 145 | + return $status; // died mid-stream...so this is already fubar |
| 150 | 146 | } |
| 151 | 147 | } |
| 152 | 148 | return Status::newFatal( "Could not stream file {$params['source']}." ); |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php |
| — | — | @@ -8,28 +8,45 @@ |
| 9 | 9 | * Class for a file-system based file backend |
| 10 | 10 | */ |
| 11 | 11 | class FSFileBackend extends FileBackend { |
| | 12 | + /** @var Array Map of container names to paths */ |
| | 13 | + protected $containerPaths = array(); |
| 12 | 14 | protected $fileMode; // file permission mode |
| 13 | 15 | |
| 14 | 16 | function __construct( array $config ) { |
| 15 | 17 | $this->name = $config['name']; |
| | 18 | + $this->containerPaths = (array)$config['containers']; |
| 16 | 19 | $this->lockManager = $config['lockManger']; |
| 17 | 20 | $this->fileMode = isset( $config['fileMode'] ) |
| 18 | 21 | ? $config['fileMode'] |
| 19 | 22 | : 0644; |
| 20 | 23 | } |
| 21 | 24 | |
| | 25 | + protected function resolveContainerPath( $container, $relStoragePath ) { |
| | 26 | + // Get absolute path given the container base dir |
| | 27 | + if ( isset( $this->containerPaths[$container] ) ) { |
| | 28 | + return $this->containerPaths[$container] . "/{$relStoragePath}"; |
| | 29 | + } |
| | 30 | + return null; |
| | 31 | + } |
| | 32 | + |
| 22 | 33 | function store( array $params ) { |
| 23 | 34 | $status = Status::newGood(); |
| 24 | 35 | |
| 25 | | - if ( file_exists( $params['dest'] ) ) { |
| | 36 | + list( $c, $dest ) = $this->resolveVirtualPath( $params['dest'] ); |
| | 37 | + if ( $dest === null ) { |
| | 38 | + $status->fatal( "Invalid storage path {$params['dest']}." ); |
| | 39 | + return $status; |
| | 40 | + } |
| | 41 | + |
| | 42 | + if ( file_exists( $dest ) ) { |
| 26 | 43 | if ( isset( $params['overwriteDest'] ) ) { |
| 27 | | - $ok = unlink( $params['dest'] ); |
| | 44 | + $ok = unlink( $dest ); |
| 28 | 45 | if ( !$ok ) { |
| 29 | 46 | $status->fatal( "Could not delete destination file." ); |
| 30 | 47 | return $status; |
| 31 | 48 | } |
| 32 | 49 | } elseif ( isset( $params['overwriteSame'] ) ) { |
| 33 | | - if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| | 50 | + if ( !$this->filesAreSame( $params['source'], $dest ) ) { |
| 34 | 51 | $status->fatal( "Non-identical destination file already exists." ); |
| 35 | 52 | } |
| 36 | 53 | return $status; // do nothing; either OK or bad status |
| — | — | @@ -38,18 +55,18 @@ |
| 39 | 56 | return $status; |
| 40 | 57 | } |
| 41 | 58 | } else { |
| 42 | | - wfMakeDirParents( $params['dest'] ); |
| | 59 | + wfMakeDirParents( $dest ); |
| 43 | 60 | } |
| 44 | 61 | |
| 45 | 62 | wfSuppressWarnings(); |
| 46 | | - $ok = copy( $params['source'], $params['dest'] ); |
| | 63 | + $ok = copy( $params['source'], $dest ); |
| 47 | 64 | wfRestoreWarnings(); |
| 48 | 65 | if ( !$ok ) { |
| 49 | 66 | $status->fatal( "Could not copy file to destination." ); |
| 50 | 67 | return $status; |
| 51 | 68 | } |
| 52 | 69 | |
| 53 | | - $this->chmod( $params['dest'] ); |
| | 70 | + $this->chmod( $dest ); |
| 54 | 71 | |
| 55 | 72 | return $status; |
| 56 | 73 | } |
| — | — | @@ -61,15 +78,26 @@ |
| 62 | 79 | function move( array $params ) { |
| 63 | 80 | $status = Status::newGood(); |
| 64 | 81 | |
| 65 | | - if ( file_exists( $params['dest'] ) ) { |
| | 82 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 83 | + if ( $source === null ) { |
| | 84 | + $status->fatal( "Invalid storage path {$params['source']}." ); |
| | 85 | + return $status; |
| | 86 | + } |
| | 87 | + list( $c, $dest ) = $this->resolveVirtualPath( $params['dest'] ); |
| | 88 | + if ( $dest === null ) { |
| | 89 | + $status->fatal( "Invalid storage path {$params['dest']}." ); |
| | 90 | + return $status; |
| | 91 | + } |
| | 92 | + |
| | 93 | + if ( file_exists( $dest ) ) { |
| 66 | 94 | if ( isset( $params['overwriteDest'] ) ) { |
| 67 | | - $ok = unlink( $params['dest'] ); |
| | 95 | + $ok = unlink( $dest ); |
| 68 | 96 | if ( !$ok ) { |
| 69 | 97 | $status->fatal( "Could not delete destination file." ); |
| 70 | 98 | return $status; |
| 71 | 99 | } |
| 72 | 100 | } elseif ( isset( $params['overwriteSame'] ) ) { |
| 73 | | - if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| | 101 | + if ( !$this->filesAreSame( $source, $dest ) ) { |
| 74 | 102 | $status->fatal( "Non-identical destination file already exists." ); |
| 75 | 103 | } |
| 76 | 104 | return $status; // do nothing; either OK or bad status |
| — | — | @@ -78,11 +106,11 @@ |
| 79 | 107 | return $status; |
| 80 | 108 | } |
| 81 | 109 | } else { |
| 82 | | - wfMakeDirParents( $params['dest'] ); |
| | 110 | + wfMakeDirParents( $dest ); |
| 83 | 111 | } |
| 84 | 112 | |
| 85 | 113 | wfSuppressWarnings(); |
| 86 | | - $ok = rename( $params['source'], $params['dest'] ); |
| | 114 | + $ok = rename( $source, $dest ); |
| 87 | 115 | wfRestoreWarnings(); |
| 88 | 116 | if ( !$ok ) { |
| 89 | 117 | $status->fatal( "Could not move file to destination." ); |
| — | — | @@ -95,7 +123,13 @@ |
| 96 | 124 | function delete( array $params ) { |
| 97 | 125 | $status = Status::newGood(); |
| 98 | 126 | |
| 99 | | - if ( !file_exists( $params['source'] ) ) { |
| | 127 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 128 | + if ( $source === null ) { |
| | 129 | + $status->fatal( "Invalid storage path {$params['source']}." ); |
| | 130 | + return $status; |
| | 131 | + } |
| | 132 | + |
| | 133 | + if ( !file_exists( $source ) ) { |
| 100 | 134 | if ( !$params['ignoreMissingSource'] ) { |
| 101 | 135 | $status->fatal( "Could not delete source because it does not exist." ); |
| 102 | 136 | } |
| — | — | @@ -103,7 +137,7 @@ |
| 104 | 138 | } |
| 105 | 139 | |
| 106 | 140 | wfSuppressWarnings(); |
| 107 | | - $ok = unlink( $params['dest'] ); |
| | 141 | + $ok = unlink( $source ); |
| 108 | 142 | wfRestoreWarnings(); |
| 109 | 143 | if ( !$ok ) { |
| 110 | 144 | $status->fatal( "Could not delete source file." ); |
| — | — | @@ -116,8 +150,14 @@ |
| 117 | 151 | function concatenate( array $params ) { |
| 118 | 152 | $status = Status::newGood(); |
| 119 | 153 | |
| | 154 | + list( $c, $dest ) = $this->resolveVirtualPath( $params['dest'] ); |
| | 155 | + if ( $dest === null ) { |
| | 156 | + $status->fatal( "Invalid storage path {$params['dest']}." ); |
| | 157 | + return $status; |
| | 158 | + } |
| | 159 | + |
| 120 | 160 | // Check if the destination file exists and we can't handle that |
| 121 | | - $destExists = file_exists( $params['dest'] ); |
| | 161 | + $destExists = file_exists( $dest ); |
| 122 | 162 | if ( $destExists && !$params['overwriteDest'] && !$params['overwriteSame'] ) { |
| 123 | 163 | $status->fatal( "Destination file already exists." ); // short-circuit |
| 124 | 164 | return $status; |
| — | — | @@ -140,7 +180,12 @@ |
| 141 | 181 | $status->fatal( "Could not open temporary file $tmpPath." ); |
| 142 | 182 | return $status; |
| 143 | 183 | } |
| 144 | | - foreach ( $params['sources'] as $source ) { |
| | 184 | + foreach ( $params['sources'] as $virtualSource ) { |
| | 185 | + list( $c, $source ) = $this->resolveVirtualPath( $virtualSource ); |
| | 186 | + if ( $source === null ) { |
| | 187 | + $status->fatal( "Invalid storage path {$virtualSource}." ); |
| | 188 | + return $status; |
| | 189 | + } |
| 145 | 190 | // Load chunk into memory (it should be a small file) |
| 146 | 191 | $chunk = file_get_contents( $source ); |
| 147 | 192 | if ( $chunk === false ) { |
| — | — | @@ -165,51 +210,74 @@ |
| 166 | 211 | if ( $destExists ) { |
| 167 | 212 | if ( isset( $params['overwriteDest'] ) ) { |
| 168 | 213 | wfSuppressWarnings(); |
| 169 | | - $ok = unlink( $params['dest'] ); |
| | 214 | + $ok = unlink( $dest ); |
| 170 | 215 | wfRestoreWarnings(); |
| 171 | 216 | if ( !$ok ) { |
| 172 | 217 | $status->fatal( "Could not delete destination file." ); |
| 173 | 218 | return $status; |
| 174 | 219 | } |
| 175 | 220 | } elseif ( isset( $params['overwriteSame'] ) ) { |
| 176 | | - if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| | 221 | + if ( !$this->filesAreSame( $tmpPath, $dest ) ) { |
| 177 | 222 | $status->fatal( "Non-identical destination file already exists." ); |
| 178 | 223 | } |
| 179 | 224 | return $status; // do nothing; either OK or bad status |
| 180 | 225 | } |
| 181 | 226 | } else { |
| 182 | 227 | // Make sure destination directory exists |
| 183 | | - wfMakeDirParents( $params['dest'] ); |
| | 228 | + wfMakeDirParents( $dest ); |
| 184 | 229 | } |
| 185 | 230 | |
| 186 | 231 | // Rename the temporary file to the destination path |
| 187 | 232 | wfSuppressWarnings(); |
| 188 | | - $ok = rename( $tmpPath, $params['dest'] ); |
| | 233 | + $ok = rename( $tmpPath, $dest ); |
| 189 | 234 | wfRestoreWarnings(); |
| 190 | 235 | if ( !$ok ) { |
| 191 | 236 | $status->fatal( "Could not rename temporary file to destination." ); |
| 192 | 237 | return $status; |
| 193 | 238 | } |
| 194 | 239 | |
| 195 | | - $this->chmod( $params['dest'] ); |
| | 240 | + $this->chmod( $dest ); |
| 196 | 241 | |
| 197 | 242 | return $status; |
| 198 | 243 | } |
| 199 | 244 | |
| 200 | 245 | function fileExists( array $params ) { |
| 201 | | - return file_exists( $params['source'] ); |
| | 246 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 247 | + if ( $source === null ) { |
| | 248 | + return false; // invalid storage path |
| | 249 | + } |
| | 250 | + return file_exists( $source ); |
| 202 | 251 | } |
| 203 | 252 | |
| 204 | 253 | function getFileProps( array $params ) { |
| 205 | | - return File::getPropsFromPath( $params['source'] ); |
| | 254 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 255 | + if ( $source === null ) { |
| | 256 | + return null; // invalid storage path |
| | 257 | + } |
| | 258 | + return File::getPropsFromPath( $source ); |
| 206 | 259 | } |
| 207 | 260 | |
| | 261 | + // Not suitable for massive listings |
| | 262 | + function getFileList( array $params ) { |
| | 263 | + list( $c, $dir ) = $this->resolveVirtualPath( $params['directory'] ); |
| | 264 | + if ( $dir === null ) { // valid storage path |
| | 265 | + return new FileIterator( '' ); // empty result |
| | 266 | + } |
| | 267 | + return new FileIterator( $dir ); |
| | 268 | + } |
| | 269 | + |
| 208 | 270 | function streamFile( array $params ) { |
| 209 | 271 | $status = Status::newGood(); |
| 210 | 272 | |
| 211 | | - $ok = StreamFile::stream( $params['source'], array(), false ); |
| | 273 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 274 | + if ( $source === null ) { |
| | 275 | + $status->fatal( "Invalid storage path {$params['source']}." ); |
| | 276 | + return $status; |
| | 277 | + } |
| | 278 | + |
| | 279 | + $ok = StreamFile::stream( $source, array(), false ); |
| 212 | 280 | if ( !$ok ) { |
| 213 | | - $status->fatal( "Unable to stream file {$params['source']}." ); |
| | 281 | + $status->fatal( "Unable to stream file {$source}." ); |
| 214 | 282 | return $status; |
| 215 | 283 | } |
| 216 | 284 | |
| — | — | @@ -217,6 +285,11 @@ |
| 218 | 286 | } |
| 219 | 287 | |
| 220 | 288 | function getLocalCopy( array $params ) { |
| | 289 | + list( $c, $source ) = $this->resolveVirtualPath( $params['source'] ); |
| | 290 | + if ( $source === null ) { |
| | 291 | + return null; |
| | 292 | + } |
| | 293 | + |
| 221 | 294 | // Create a new temporary file... |
| 222 | 295 | wfSuppressWarnings(); |
| 223 | 296 | $tmpPath = tempnam( wfTempDir(), 'file_localcopy' ); |
| — | — | @@ -227,7 +300,7 @@ |
| 228 | 301 | |
| 229 | 302 | // Copy the source file over the temp file |
| 230 | 303 | wfSuppressWarnings(); |
| 231 | | - $ok = copy( $params['source'], $tmpPath ); |
| | 304 | + $ok = copy( $source, $tmpPath ); |
| 232 | 305 | wfRestoreWarnings(); |
| 233 | 306 | if ( !$ok ) { |
| 234 | 307 | return null; |
| — | — | @@ -240,8 +313,9 @@ |
| 241 | 314 | |
| 242 | 315 | /** |
| 243 | 316 | * Check if two files are identical |
| 244 | | - * @param $path1 string |
| 245 | | - * @param $path2 string |
| | 317 | + * |
| | 318 | + * @param $path1 string Absolute filesystem path |
| | 319 | + * @param $path2 string Absolute filesystem path |
| 246 | 320 | * @return bool |
| 247 | 321 | */ |
| 248 | 322 | protected function filesAreSame( $path1, $path2 ) { |
| — | — | @@ -253,7 +327,8 @@ |
| 254 | 328 | |
| 255 | 329 | /** |
| 256 | 330 | * Chmod a file, suppressing the warnings |
| 257 | | - * @param $path string The path to change |
| | 331 | + * |
| | 332 | + * @param $path string Absolute file system path |
| 258 | 333 | * @return bool Success |
| 259 | 334 | */ |
| 260 | 335 | protected function chmod( $path ) { |
| — | — | @@ -264,3 +339,130 @@ |
| 265 | 340 | return $ok; |
| 266 | 341 | } |
| 267 | 342 | } |
| | 343 | + |
| | 344 | +/** |
| | 345 | + * Simple DFS based file browsing iterator. The highest number of file handles |
| | 346 | + * open at any given time is proportional to the height of the directory tree. |
| | 347 | + */ |
| | 348 | +class FileIterator implements Iterator { |
| | 349 | + private $directory; // starting directory |
| | 350 | + |
| | 351 | + private $position = 0; |
| | 352 | + private $currentFile = false; |
| | 353 | + private $dirStack = array(); // array of (dir name, dir handle) tuples |
| | 354 | + |
| | 355 | + private $loaded = false; |
| | 356 | + |
| | 357 | + /** |
| | 358 | + * Get an iterator to list the files under $directory and its subdirectories |
| | 359 | + * @param $directory string |
| | 360 | + */ |
| | 361 | + public function __construct( $directory ) { |
| | 362 | + $this->directory = (string)$directory; |
| | 363 | + } |
| | 364 | + |
| | 365 | + private function load() { |
| | 366 | + if ( !$this->loaded ) { |
| | 367 | + $this->loaded = true; |
| | 368 | + // If we get an invalid directory then the result is an empty list |
| | 369 | + if ( is_dir( $this->directory ) ) { |
| | 370 | + $handle = opendir( $this->directory ); |
| | 371 | + if ( $handle ) { |
| | 372 | + $this->pushDirectory( $this->directory, $handle ); |
| | 373 | + $this->currentFile = $this->nextFile(); |
| | 374 | + } |
| | 375 | + } |
| | 376 | + } |
| | 377 | + } |
| | 378 | + |
| | 379 | + function rewind() { |
| | 380 | + $this->closeDirectories(); |
| | 381 | + $this->position = 0; |
| | 382 | + $this->currentFile = false; |
| | 383 | + $this->loaded = false; |
| | 384 | + } |
| | 385 | + |
| | 386 | + function current() { |
| | 387 | + $this->load(); |
| | 388 | + return $this->currentFile; |
| | 389 | + } |
| | 390 | + |
| | 391 | + function key() { |
| | 392 | + $this->load(); |
| | 393 | + return $this->position; |
| | 394 | + } |
| | 395 | + |
| | 396 | + function next() { |
| | 397 | + $this->load(); |
| | 398 | + ++$this->position; |
| | 399 | + $this->currentFile = $this->nextFile(); |
| | 400 | + } |
| | 401 | + |
| | 402 | + function valid() { |
| | 403 | + $this->load(); |
| | 404 | + return ( $this->currentFile !== false ); |
| | 405 | + } |
| | 406 | + |
| | 407 | + function nextFile() { |
| | 408 | + $set = $this->currentDirectory(); |
| | 409 | + if ( !$set ) { |
| | 410 | + return false; // nothing else to scan |
| | 411 | + } |
| | 412 | + list( $dir, $handle ) = $set; |
| | 413 | + while ( false !== ( $file = readdir( $handle ) ) ) { |
| | 414 | + // Exclude '.' and '..' as well .svn or .lock type files. |
| | 415 | + // Also excludes symlinks and the like so as to avoid cycles. |
| | 416 | + if ( $file[0] !== '.' && !is_link( $file ) ) { |
| | 417 | + // If the first thing we find is a file, then return it. |
| | 418 | + // If the first thing we find is a directory, then return |
| | 419 | + // the first file that it contains (via recursion). |
| | 420 | + if ( is_dir( "$dir/$file" ) ) { |
| | 421 | + $subHandle = opendir( "$dir/$file" ); |
| | 422 | + if ( $subHandle ) { |
| | 423 | + $this->pushDirectory( "{$dir}/{$file}", $subHandle ); |
| | 424 | + $nextFile = $this->nextFile(); |
| | 425 | + if ( $nextFile !== false ) { |
| | 426 | + return $nextFile; // found the next one! |
| | 427 | + } |
| | 428 | + } |
| | 429 | + } elseif ( is_file( "$dir/$file" ) ) { |
| | 430 | + return "$dir/$file"; // found the next one! |
| | 431 | + } |
| | 432 | + } |
| | 433 | + } |
| | 434 | + # If we didn't find anything in this directory, |
| | 435 | + # then back out and scan the other higher directories |
| | 436 | + $this->popDirectory(); |
| | 437 | + return $this->nextFile(); |
| | 438 | + } |
| | 439 | + |
| | 440 | + private function currentDirectory() { |
| | 441 | + if ( !count( $this->dirStack ) ) { |
| | 442 | + return false; |
| | 443 | + } |
| | 444 | + return $this->dirStack[count( $this->dirStack ) - 1]; |
| | 445 | + } |
| | 446 | + |
| | 447 | + private function popDirectory() { |
| | 448 | + if ( count( $this->dirStack ) ) { |
| | 449 | + list( $dir, $handle ) = array_pop( $this->dirStack ); |
| | 450 | + closedir( $handle ); |
| | 451 | + } |
| | 452 | + } |
| | 453 | + |
| | 454 | + private function pushDirectory( $dir, $handle ) { |
| | 455 | + $this->dirStack[] = array( $dir, $handle ); |
| | 456 | + } |
| | 457 | + |
| | 458 | + private function closeDirectories() { |
| | 459 | + foreach ( $this->dirStack as $set ) { |
| | 460 | + list( $dir, $handle ) = $set; |
| | 461 | + closedir( $handle ); |
| | 462 | + } |
| | 463 | + $this->dirStack = array(); |
| | 464 | + } |
| | 465 | + |
| | 466 | + private function __destruct() { |
| | 467 | + $this->closeDirectories(); |
| | 468 | + } |
| | 469 | +} |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileLockManager.php |
| — | — | @@ -91,9 +91,8 @@ |
| 92 | 92 | |
| 93 | 93 | $lockedKeys = array(); // files locked in this attempt |
| 94 | 94 | foreach ( $keys as $key ) { |
| 95 | | - $subStatus = $this->doSingleLock( $key ); |
| 96 | | - $status->merge( $subStatus ); |
| 97 | | - if ( $subStatus->isOk() ) { |
| | 95 | + $status->merge( $this->doSingleLock( $key ) ); |
| | 96 | + if ( $status->isOk() ) { |
| 98 | 97 | $lockedKeys[] = $key; |
| 99 | 98 | } else { |
| 100 | 99 | // Abort and unlock everything |
| — | — | @@ -333,7 +332,7 @@ |
| 334 | 333 | $this->doLockingSelect( $server, $keys ); // SELECT FOR UPDATE |
| 335 | 334 | $locksMade = true; // success for this fallback |
| 336 | 335 | } catch ( DBError $e ) { |
| 337 | | - // oh well; best effort |
| | 336 | + // oh well; best effort (@TODO: logging?) |
| 338 | 337 | } |
| 339 | 338 | } |
| 340 | 339 | return $locksMade; |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php |
| — | — | @@ -5,21 +5,29 @@ |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | 8 | /** |
| 9 | | - * Base class for all file backend classes. |
| 10 | | - * This class defines the methods as abstract that |
| 11 | | - * must be implemented in all file backend classes. |
| | 9 | + * Base class for all file backend classes (including multi-write backends). |
| | 10 | + * This class defines the methods as abstract that must be implemented subclasses. |
| 12 | 11 | * |
| 13 | | - * All "storage paths" and "storage directories" may be real file system |
| 14 | | - * paths or just virtual paths such as object names in Swift. |
| | 12 | + * All "storage paths" are of the format "mwstore://container/path". |
| | 13 | + * The paths use typical file system notation, though any particular backend may |
| | 14 | + * not actually be using a local filesystem. Therefore, the paths are only virtual. |
| | 15 | + * |
| | 16 | + * All functions should avoid throwing exceptions at all costs. |
| | 17 | + * As a corollary, external dependencies should be kept to a minimal. |
| 15 | 18 | */ |
| 16 | | -interface IFileBackend { |
| | 19 | +abstract class FileBackendBase { |
| | 20 | + protected $name; // unique backend name |
| | 21 | + |
| 17 | 22 | /** |
| 18 | 23 | * We may have multiple different backends of the same type. |
| 19 | 24 | * For example, we can have two Swift backends using different proxies. |
| | 25 | + * All backend instances must have unique names. |
| 20 | 26 | * |
| 21 | 27 | * @return string |
| 22 | 28 | */ |
| 23 | | - public function getName(); |
| | 29 | + final public function getName() { |
| | 30 | + return $this->name; |
| | 31 | + } |
| 24 | 32 | |
| 25 | 33 | /** |
| 26 | 34 | * This is the main entry point into the file system back end. Callers will |
| — | — | @@ -27,7 +35,7 @@ |
| 28 | 36 | * array. This class will then handle handing the operations off to the |
| 29 | 37 | * correct file store module. |
| 30 | 38 | * |
| 31 | | - * Using $ops |
| | 39 | + * Using $ops: |
| 32 | 40 | * $ops is an array of arrays. The first array holds a list of operations. |
| 33 | 41 | * The inner array contains the parameters, E.G: |
| 34 | 42 | * <code> |
| — | — | @@ -35,7 +43,7 @@ |
| 36 | 44 | * array( |
| 37 | 45 | * 'operation' => 'store', |
| 38 | 46 | * 'src' => '/tmp/uploads/picture.png', |
| 39 | | - * 'dest' => 'zone/uploadedFilename.png' |
| | 47 | + * 'dest' => 'mwstore://container/uploadedFilename.png' |
| 40 | 48 | * ) |
| 41 | 49 | * ); |
| 42 | 50 | * </code> |
| — | — | @@ -43,7 +51,7 @@ |
| 44 | 52 | * @param Array $ops Array of arrays containing N operations to execute IN ORDER |
| 45 | 53 | * @return Status |
| 46 | 54 | */ |
| 47 | | - public function doOperations( array $ops ); |
| | 55 | + abstract public function doOperations( array $ops ); |
| 48 | 56 | |
| 49 | 57 | /** |
| 50 | 58 | * Return a list of FileOp objects from a list of operations. |
| — | — | @@ -53,11 +61,11 @@ |
| 54 | 62 | * @return Array |
| 55 | 63 | * @throws MWException |
| 56 | 64 | */ |
| 57 | | - public function getOperations( array $ops ); |
| | 65 | + abstract public function getOperations( array $ops ); |
| 58 | 66 | |
| 59 | 67 | /** |
| 60 | 68 | * Store a file into the backend from a file on disk. |
| 61 | | - * Do not call this function from places other than FileOp. |
| | 69 | + * Do not call this function from places outside FileBackend and FileOp. |
| 62 | 70 | * $params include: |
| 63 | 71 | * source : source path on disk |
| 64 | 72 | * dest : destination storage path |
| — | — | @@ -67,11 +75,11 @@ |
| 68 | 76 | * @param Array $params |
| 69 | 77 | * @return Status |
| 70 | 78 | */ |
| 71 | | - public function store( array $params ); |
| | 79 | + abstract public function store( array $params ); |
| 72 | 80 | |
| 73 | 81 | /** |
| 74 | 82 | * Copy a file from one storage path to another in the backend. |
| 75 | | - * Do not call this function from places other than FileOp. |
| | 83 | + * Do not call this function from places outside FileBackend and FileOp. |
| 76 | 84 | * $params include: |
| 77 | 85 | * source : source storage path |
| 78 | 86 | * dest : destination storage path |
| — | — | @@ -81,12 +89,12 @@ |
| 82 | 90 | * @param Array $params |
| 83 | 91 | * @return Status |
| 84 | 92 | */ |
| 85 | | - public function copy( array $params ); |
| | 93 | + abstract public function copy( array $params ); |
| 86 | 94 | |
| 87 | 95 | /** |
| 88 | 96 | * Copy a file from one storage path to another in the backend. |
| 89 | 97 | * This can be left as a dummy function as long as hasMove() returns false. |
| 90 | | - * Do not call this function from places other than FileOp. |
| | 98 | + * Do not call this function from places outside FileBackend and FileOp. |
| 91 | 99 | * $params include: |
| 92 | 100 | * source : source storage path |
| 93 | 101 | * dest : destination storage path |
| — | — | @@ -96,11 +104,11 @@ |
| 97 | 105 | * @param Array $params |
| 98 | 106 | * @return Status |
| 99 | 107 | */ |
| 100 | | - public function move( array $params ); |
| | 108 | + abstract public function move( array $params ); |
| 101 | 109 | |
| 102 | 110 | /** |
| 103 | 111 | * Delete a file at the storage path. |
| 104 | | - * Do not call this function from places other than FileOp. |
| | 112 | + * Do not call this function from places outside FileBackend and FileOp. |
| 105 | 113 | * $params include: |
| 106 | 114 | * source : source storage path |
| 107 | 115 | * ignoreMissingSource : don't return an error if the file does not exist |
| — | — | @@ -108,11 +116,11 @@ |
| 109 | 117 | * @param Array $params |
| 110 | 118 | * @return Status |
| 111 | 119 | */ |
| 112 | | - public function delete( array $params ); |
| | 120 | + abstract public function delete( array $params ); |
| 113 | 121 | |
| 114 | 122 | /** |
| 115 | 123 | * Combines files from severals storage paths into a new file in the backend. |
| 116 | | - * Do not call this function from places other than FileOp. |
| | 124 | + * Do not call this function from places outside FileBackend and FileOp. |
| 117 | 125 | * $params include: |
| 118 | 126 | * source : source storage path |
| 119 | 127 | * dest : destination storage path |
| — | — | @@ -122,26 +130,30 @@ |
| 123 | 131 | * @param Array $params |
| 124 | 132 | * @return Status |
| 125 | 133 | */ |
| 126 | | - public function concatenate( array $params ); |
| | 134 | + abstract public function concatenate( array $params ); |
| 127 | 135 | |
| 128 | 136 | /** |
| 129 | | - * Whether this backend has a move() implementation. |
| 130 | | - * Do not call this function from places other than FileOp. |
| | 137 | + * Whether this backend implements move() and can handle this potential move. |
| | 138 | + * For example, moving objects accross containers may not be supported. |
| | 139 | + * Do not call this function from places outside FileBackend and FileOp. |
| | 140 | + * $params include: |
| | 141 | + * source : source storage path |
| | 142 | + * dest : destination storage path |
| 131 | 143 | * |
| 132 | 144 | * @return bool |
| 133 | 145 | */ |
| 134 | | - public function hasMove(); |
| | 146 | + abstract public function canMove( array $params ); |
| 135 | 147 | |
| 136 | 148 | /** |
| 137 | 149 | * Check if a file exits at a storage path in the backend. |
| 138 | | - * Do not call this function from places other than FileOp. |
| | 150 | + * Do not call this function from places outside FileBackend and FileOp. |
| 139 | 151 | * $params include: |
| 140 | 152 | * source : source storage path |
| 141 | 153 | * |
| 142 | 154 | * @param Array $params |
| 143 | 155 | * @return bool |
| 144 | 156 | */ |
| 145 | | - public function fileExists( array $params ); |
| | 157 | + abstract public function fileExists( array $params ); |
| 146 | 158 | |
| 147 | 159 | /** |
| 148 | 160 | * Get the properties of the file that exists at a storage path in the backend |
| — | — | @@ -151,12 +163,12 @@ |
| 152 | 164 | * @param Array $params |
| 153 | 165 | * @return Array|null Gives null if the file does not exist |
| 154 | 166 | */ |
| 155 | | - public function getFileProps( array $params ); |
| | 167 | + abstract public function getFileProps( array $params ); |
| 156 | 168 | |
| 157 | 169 | /** |
| 158 | 170 | * Stream the file that exists at a storage path in the backend. |
| 159 | 171 | * Appropriate HTTP headers (Status, Content-Type, Content-Length) |
| 160 | | - * must be sent on success, while no headers should be sent on failure. |
| | 172 | + * must be sent if streaming began, while none should be sent otherwise. |
| 161 | 173 | * Implementations should flush the output buffer before sending data. |
| 162 | 174 | * $params include: |
| 163 | 175 | * source : source storage path |
| — | — | @@ -164,9 +176,21 @@ |
| 165 | 177 | * @param Array $params |
| 166 | 178 | * @return Status |
| 167 | 179 | */ |
| 168 | | - public function streamFile( array $params ); |
| | 180 | + abstract public function streamFile( array $params ); |
| 169 | 181 | |
| 170 | 182 | /** |
| | 183 | + * Get an iterator to list out all object files under a storage directory. |
| | 184 | + * Results should be storage paths relative to the given directory. |
| | 185 | + * If the directory is of the form "mwstore://container", then all items |
| | 186 | + * in the container should be listed. If of the form "mwstore://container/dir", |
| | 187 | + * then all items under that container directory should be listed. |
| | 188 | + * $params include: |
| | 189 | + * directory : storage path directory. |
| | 190 | + * @return Iterator |
| | 191 | + */ |
| | 192 | + abstract public function getFileList( array $params ); |
| | 193 | + |
| | 194 | + /** |
| 171 | 195 | * Get a local copy on disk of the file at a storage path in the backend |
| 172 | 196 | * $params include: |
| 173 | 197 | * source : source storage path |
| — | — | @@ -174,40 +198,34 @@ |
| 175 | 199 | * @param Array $params |
| 176 | 200 | * @return TempLocalFile|null Temporary file or null on failure |
| 177 | 201 | */ |
| 178 | | - public function getLocalCopy( array $params ); |
| | 202 | + abstract public function getLocalCopy( array $params ); |
| 179 | 203 | |
| 180 | 204 | /** |
| 181 | 205 | * Lock the files at the given storage paths in the backend. |
| 182 | | - * Do not call this function from places other than FileOp. |
| | 206 | + * Do not call this function from places outside FileBackend and FileOp. |
| 183 | 207 | * |
| 184 | 208 | * @param $sources Array Source storage paths |
| 185 | 209 | * @return Status |
| 186 | 210 | */ |
| 187 | | - public function lockFiles( array $sources ); |
| | 211 | + abstract public function lockFiles( array $sources ); |
| 188 | 212 | |
| 189 | 213 | /** |
| 190 | 214 | * Unlock the files at the given storage paths in the backend. |
| 191 | | - * Do not call this function from places other than FileOp. |
| | 215 | + * Do not call this function from places outside FileBackend and FileOp. |
| 192 | 216 | * |
| 193 | 217 | * @param $sources Array Source storage paths |
| 194 | 218 | * @return Status |
| 195 | 219 | */ |
| 196 | | - public function unlockFiles( array $sources ); |
| | 220 | + abstract public function unlockFiles( array $sources ); |
| 197 | 221 | } |
| 198 | 222 | |
| 199 | 223 | /** |
| 200 | | - * This class defines the methods as abstract that should be |
| 201 | | - * implemented in child classes that represent a single-write backend. |
| | 224 | + * Base class for all single-write backends |
| 202 | 225 | */ |
| 203 | | -abstract class FileBackend implements IFileBackend { |
| 204 | | - protected $name; |
| | 226 | +abstract class FileBackend extends FileBackendBase { |
| 205 | 227 | /** @var FileLockManager */ |
| 206 | 228 | protected $lockManager; |
| 207 | 229 | |
| 208 | | - final public function getName() { |
| 209 | | - return $this->name; |
| 210 | | - } |
| 211 | | - |
| 212 | 230 | /** |
| 213 | 231 | * Build a new object from configuration. |
| 214 | 232 | * This should only be called from within FileRepo classes. |
| — | — | @@ -222,7 +240,7 @@ |
| 223 | 241 | $this->lockManager = $config['lockManger']; |
| 224 | 242 | } |
| 225 | 243 | |
| 226 | | - function hasMove() { |
| | 244 | + function canMove( array $params ) { |
| 227 | 245 | return false; // not implemented |
| 228 | 246 | } |
| 229 | 247 | |
| — | — | @@ -232,7 +250,6 @@ |
| 233 | 251 | |
| 234 | 252 | /** |
| 235 | 253 | * Get the list of supported operations and their corresponding FileOp classes. |
| 236 | | - * Subclasses should implement these using FileOp subclasses |
| 237 | 254 | * |
| 238 | 255 | * @return Array |
| 239 | 256 | */ |
| — | — | @@ -327,6 +344,40 @@ |
| 328 | 345 | $backendKey = get_class( $this ) . '-' . $this->getName(); |
| 329 | 346 | return $this->lockManager->unlock( $backendKey, $paths ); // not supported |
| 330 | 347 | } |
| | 348 | + |
| | 349 | + /** |
| | 350 | + * Split a storage path (e.g. "mwstore://container/path/to/object") |
| | 351 | + * into a container name and a full object name within that container. |
| | 352 | + * |
| | 353 | + * @param $storagePath string |
| | 354 | + * @return Array (container, object name) or (null, null) if path is invalid |
| | 355 | + */ |
| | 356 | + final protected function resolveVirtualPath( $storagePath ) { |
| | 357 | + if ( strpos( $storagePath, 'mwstore://' ) === 0 ) { |
| | 358 | + $m = explode( '/', substr( $storagePath, 10 ), 2 ); |
| | 359 | + if ( count( $m ) == 2 ) { |
| | 360 | + list( $container, $relPath ) = $m; |
| | 361 | + $relPath = $this->resolveContainerPath( $container, $relPath ); |
| | 362 | + if ( $relPath !== null ) { |
| | 363 | + return array( $container, $relPath ); // (container, path) |
| | 364 | + } |
| | 365 | + } |
| | 366 | + } |
| | 367 | + return array( null, null ); |
| | 368 | + } |
| | 369 | + |
| | 370 | + /** |
| | 371 | + * Resolve a storage path relative to a particular container. |
| | 372 | + * This is for internal use for backends, such as encoding or |
| | 373 | + * perhaps getting absolute paths (e.g. file system based backends). |
| | 374 | + * |
| | 375 | + * @param $container string |
| | 376 | + * @param $relStoragePath string |
| | 377 | + * @return string|null Null if path is not valid |
| | 378 | + */ |
| | 379 | + protected function resolveContainerPath( $container, $relStoragePath ) { |
| | 380 | + return $relStoragePath; |
| | 381 | + } |
| 331 | 382 | } |
| 332 | 383 | |
| 333 | 384 | /** |
| — | — | @@ -360,6 +411,7 @@ |
| 361 | 412 | $this->params = $params; |
| 362 | 413 | $this->state = self::STATE_NEW; |
| 363 | 414 | $this->failedAttempt = false; |
| | 415 | + $this->initialize(); |
| 364 | 416 | } |
| 365 | 417 | |
| 366 | 418 | /** |
| — | — | @@ -425,6 +477,11 @@ |
| 426 | 478 | } |
| 427 | 479 | |
| 428 | 480 | /** |
| | 481 | + * @return void |
| | 482 | + */ |
| | 483 | + protected function initialize() {} |
| | 484 | + |
| | 485 | + /** |
| 429 | 486 | * @return Status |
| 430 | 487 | */ |
| 431 | 488 | abstract protected function doAttempt(); |
| — | — | @@ -577,6 +634,13 @@ |
| 578 | 635 | * overwriteSame : override any existing file at destination |
| 579 | 636 | */ |
| 580 | 637 | class FileMoveOp extends FileStoreOp { |
| | 638 | + protected $usingMove = false; // using backend move() function? |
| | 639 | + |
| | 640 | + function initialize() { |
| | 641 | + // Use faster, native, move() if applicable |
| | 642 | + $this->usingMove = $this->backend->canMove( $this->params ); |
| | 643 | + } |
| | 644 | + |
| 581 | 645 | function doAttempt() { |
| 582 | 646 | // Create a backup copy of any file that exists at destination |
| 583 | 647 | $status = $this->backupDest(); |
| — | — | @@ -584,7 +648,7 @@ |
| 585 | 649 | return $status; |
| 586 | 650 | } |
| 587 | 651 | // Native moves: move the file into the destination |
| 588 | | - if ( $this->backend->hasMove() ) { |
| | 652 | + if ( $this->usingMove ) { |
| 589 | 653 | $status = $this->backend->move( $this->params ); |
| 590 | 654 | // Non-native moves: copy the file into the destination |
| 591 | 655 | } else { |
| — | — | @@ -595,7 +659,7 @@ |
| 596 | 660 | |
| 597 | 661 | function doRevert() { |
| 598 | 662 | // Native moves: move the file back to the source |
| 599 | | - if ( $this->backend->hasMove() ) { |
| | 663 | + if ( $this->usingMove ) { |
| 600 | 664 | $params = array( |
| 601 | 665 | 'source' => $this->params['dest'], |
| 602 | 666 | 'dest' => $this->params['source'] |
| — | — | @@ -619,7 +683,7 @@ |
| 620 | 684 | |
| 621 | 685 | function doFinish() { |
| 622 | 686 | // Native moves: nothing is at the source anymore |
| 623 | | - if ( $this->backend->hasMove() ) { |
| | 687 | + if ( $this->usingMove ) { |
| 624 | 688 | $status = Status::newGood(); |
| 625 | 689 | // Non-native moves: delete the source file |
| 626 | 690 | } else { |