| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php |
| — | — | @@ -11,13 +11,15 @@ |
| 12 | 12 | * implemented in child classes that represent a mutli-write backend. |
| 13 | 13 | * |
| 14 | 14 | * The order that the backends are defined sets the priority of which |
| 15 | | - * backend is read from or written to first. |
| | 15 | + * backend is read from or written to first. Functions like fileExists() |
| | 16 | + * and getFileProps() will return information based on the first backend |
| | 17 | + * that has the file (normally both should have it anyway). |
| 16 | 18 | * |
| 17 | 19 | * All write operations are performed on all backends. |
| 18 | 20 | * If an operation fails on one backend it will be rolled back from the others. |
| 19 | 21 | * |
| 20 | | - * Functions like fileExists() and getFileProps() will return information |
| 21 | | - * based on the first backend that has the file (normally both should have it anyway). |
| | 22 | + * To avoid excess overhead, set the the highest priority backend to use |
| | 23 | + * a generic FileLockManager and the others to use NullLockManager. |
| 22 | 24 | */ |
| 23 | 25 | class FileBackendMultiWrite implements IFileBackend { |
| 24 | 26 | protected $name; |
| — | — | @@ -46,45 +48,30 @@ |
| 47 | 49 | } |
| 48 | 50 | // Attempt each operation; abort on failure... |
| 49 | 51 | foreach ( $performOps as $index => $transaction ) { |
| 50 | | - $tStatus = $transaction->attempt(); |
| 51 | | - if ( !$tStatus->isOk() ) { |
| 52 | | - // merge $tStatus with $status |
| 53 | | - // Revert everything done so far and error out |
| 54 | | - $tStatus = $this->revertOperations( $performOps, $index ); |
| 55 | | - // merge $tStatus with $status |
| | 52 | + $subStatus = $transaction->attempt(); |
| | 53 | + $status->merge( $subStatus ); |
| | 54 | + if ( !$subStatus->isOK() ) { // operation failed? |
| | 55 | + // Revert everything done so far and abort. |
| | 56 | + // Do this by reverting each previous operation in reverse order. |
| | 57 | + $pos = $index - 1; // last one failed; no need to revert() |
| | 58 | + while ( $pos >= 0 ) { |
| | 59 | + $subStatus = $performOps[$pos]->revert(); |
| | 60 | + $status->merge( $subStatus ); |
| | 61 | + $pos--; |
| | 62 | + } |
| 56 | 63 | return $status; |
| 57 | 64 | } |
| 58 | 65 | } |
| 59 | 66 | // Finish each operation... |
| 60 | 67 | foreach ( $performOps as $index => $transaction ) { |
| 61 | | - $tStatus = $transaction->finish(); |
| 62 | | - // merge $tStatus with $status |
| | 68 | + $subStatus = $transaction->finish(); |
| | 69 | + $status->merge( $subStatus ); |
| 63 | 70 | } |
| | 71 | + // Make sure status is OK, despite any finish() fatals |
| | 72 | + $status->setResult( true ); |
| 64 | 73 | return $status; |
| 65 | 74 | } |
| 66 | 75 | |
| 67 | | - /** |
| 68 | | - * Revert a series of operations in reverse order. |
| 69 | | - * If $index is passed, then we revert all items in |
| 70 | | - * $ops from 0 to $index (inclusive). |
| 71 | | - * |
| 72 | | - * @param $ops Array List of FileOp objects |
| 73 | | - * @param $index integer |
| 74 | | - * @return Status |
| 75 | | - */ |
| 76 | | - private function revertOperations( array $ops, $index = false ) { |
| 77 | | - $status = Status::newGood(); |
| 78 | | - $pos = ( $index !== false ) |
| 79 | | - ? $index // use provided index |
| 80 | | - : $pos = count( $ops ) - 1; // last element (or -1) |
| 81 | | - while ( $pos >= 0 ) { |
| 82 | | - $tStatus = $ops[$pos]->revert(); |
| 83 | | - // merge $tStatus with $status |
| 84 | | - $pos--; |
| 85 | | - } |
| 86 | | - return $status; |
| 87 | | - } |
| 88 | | - |
| 89 | 76 | public function store( array $params ) { |
| 90 | 77 | $op = array( 'operation' => 'store' ) + $params; |
| 91 | 78 | return $this->doOperation( array( $op ) ); |
| — | — | @@ -139,26 +126,28 @@ |
| 140 | 127 | return null; |
| 141 | 128 | } |
| 142 | 129 | |
| 143 | | - public function lockFile( array $path ) { |
| | 130 | + public function lockFiles( array $paths ) { |
| 144 | 131 | $status = Status::newGood(); |
| 145 | 132 | foreach ( $this->backends as $index => $backend ) { |
| 146 | | - $lockStatus = $backend->lockFile( $path ); |
| 147 | | - // merge $lockStatus with $status |
| 148 | | - if ( !$lockStatus->isOk() ) { |
| 149 | | - for ( $i=0; $i < $index; $i++ ) { |
| 150 | | - $lockStatus = $this->unlockFile( $path ); |
| 151 | | - // merge $lockStatus with $status |
| | 133 | + $subStatus = $backend->lockFiles( $paths ); |
| | 134 | + $status->merge( $subStatus ); |
| | 135 | + if ( !$subStatus->isOk() ) { |
| | 136 | + // Lock failed: release the locks done so far each backend |
| | 137 | + for ( $i=0; $i < $index; $i++ ) { // don't do backend $index since it failed |
| | 138 | + $subStatus = $backend->unlockFiles( $paths ); |
| | 139 | + $status->merge( $subStatus ); |
| 152 | 140 | } |
| | 141 | + return $status; |
| 153 | 142 | } |
| 154 | 143 | } |
| 155 | 144 | return $status; |
| 156 | 145 | } |
| 157 | 146 | |
| 158 | | - public function unlockFile( array $path ) { |
| | 147 | + public function unlockFiles( array $paths ) { |
| 159 | 148 | $status = Status::newGood(); |
| 160 | 149 | foreach ( $this->backends as $backend ) { |
| 161 | | - $lockStatus = $backend->unlockFile( $path ); |
| 162 | | - // merge $lockStatus with $status |
| | 150 | + $subStatus = $backend->unlockFile( $paths ); |
| | 151 | + $status->merge( $subStatus ); |
| 163 | 152 | } |
| 164 | 153 | return $status; |
| 165 | 154 | } |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php |
| — | — | @@ -1,35 +1,42 @@ |
| 2 | 2 | <?php |
| 3 | 3 | /** |
| 4 | | - * Base class for all file backend classes |
| 5 | | - * |
| 6 | 4 | * @file |
| 7 | 5 | * @ingroup FileRepo |
| 8 | 6 | */ |
| 9 | 7 | |
| 10 | 8 | /** |
| 11 | | - * This class defines the methods as abstract that should be implemented in |
| 12 | | - * child classes if the target store supports the operation. |
| | 9 | + * Class for a file-system based file backend |
| 13 | 10 | */ |
| 14 | 11 | class FSFileBackend extends FileBackend { |
| 15 | | - protected $fileMode; |
| | 12 | + protected $fileMode; // file permission mode |
| 16 | 13 | |
| 17 | 14 | function __construct( array $config ) { |
| | 15 | + $this->name = $config['name']; |
| | 16 | + $this->lockManager = $config['lockManger']; |
| 18 | 17 | $this->fileMode = isset( $config['fileMode'] ) |
| 19 | 18 | ? $config['fileMode'] |
| 20 | 19 | : 0644; |
| 21 | 20 | } |
| 22 | 21 | |
| | 22 | + protected function fullStoragePath( $path ) { |
| | 23 | + if ( $this->root == '' ) { |
| | 24 | + return "/$path"; // absolute, don't use "current directory" |
| | 25 | + } else { |
| | 26 | + return "{$root}/{$path}"; |
| | 27 | + } |
| | 28 | + } |
| | 29 | + |
| 23 | 30 | public function store( array $params ) { |
| 24 | 31 | $status = Status::newGood(); |
| 25 | 32 | |
| 26 | 33 | if ( file_exists( $params['dest'] ) ) { |
| 27 | | - if ( $params['overwriteDest'] ) { |
| | 34 | + if ( isset( $params['overwriteDest'] ) ) { |
| 28 | 35 | $ok = unlink( $params['dest'] ); |
| 29 | 36 | if ( !$ok ) { |
| 30 | 37 | $status->fatal( "Could not delete destination file." ); |
| 31 | 38 | return $status; |
| 32 | 39 | } |
| 33 | | - } elseif ( $params['overwriteSame'] ) { |
| | 40 | + } elseif ( isset( $params['overwriteSame'] ) ) { |
| 34 | 41 | if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| 35 | 42 | $status->fatal( "Non-identical destination file already exists." ); |
| 36 | 43 | } |
| — | — | @@ -63,13 +70,13 @@ |
| 64 | 71 | $status = Status::newGood(); |
| 65 | 72 | |
| 66 | 73 | if ( file_exists( $params['dest'] ) ) { |
| 67 | | - if ( $params['overwriteDest'] ) { |
| | 74 | + if ( isset( $params['overwriteDest'] ) ) { |
| 68 | 75 | $ok = unlink( $params['dest'] ); |
| 69 | 76 | if ( !$ok ) { |
| 70 | 77 | $status->fatal( "Could not delete destination file." ); |
| 71 | 78 | return $status; |
| 72 | 79 | } |
| 73 | | - } elseif ( $params['overwriteSame'] ) { |
| | 80 | + } elseif ( isset( $params['overwriteSame'] ) ) { |
| 74 | 81 | if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| 75 | 82 | $status->fatal( "Non-identical destination file already exists." ); |
| 76 | 83 | } |
| — | — | @@ -164,7 +171,7 @@ |
| 165 | 172 | // Handle overwrite behavior of file destination if applicable. |
| 166 | 173 | // Note that we already checked if no overwrite params were set above. |
| 167 | 174 | if ( $destExists ) { |
| 168 | | - if ( $params['overwriteDest'] ) { |
| | 175 | + if ( isset( $params['overwriteDest'] ) ) { |
| 169 | 176 | wfSuppressWarnings(); |
| 170 | 177 | $ok = unlink( $params['dest'] ); |
| 171 | 178 | wfRestoreWarnings(); |
| — | — | @@ -172,7 +179,7 @@ |
| 173 | 180 | $status->fatal( "Could not delete destination file." ); |
| 174 | 181 | return $status; |
| 175 | 182 | } |
| 176 | | - } elseif ( $params['overwriteSame'] ) { |
| | 183 | + } elseif ( isset( $params['overwriteSame'] ) ) { |
| 177 | 184 | if ( !$this->filesAreSame( $params['source'], $params['dest'] ) ) { |
| 178 | 185 | $status->fatal( "Non-identical destination file already exists." ); |
| 179 | 186 | } |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileLockManager.php |
| — | — | @@ -73,6 +73,9 @@ |
| 74 | 74 | |
| 75 | 75 | /** |
| 76 | 76 | * Simple version of FileLockManager based on using FS lock files |
| | 77 | + * |
| | 78 | + * This should work fine for small sites running off one server. |
| | 79 | + * Do not use this with 'lockDir' set to an NFS mount. |
| 77 | 80 | */ |
| 78 | 81 | class FSFileLockManager extends FileLockManager { |
| 79 | 82 | protected $lockDir; // global dir for all servers |
| — | — | @@ -88,13 +91,15 @@ |
| 89 | 92 | |
| 90 | 93 | $lockedKeys = array(); // files locked in this attempt |
| 91 | 94 | foreach ( $keys as $key ) { |
| 92 | | - $lockStatus = $this->doSingleLock( $key ); |
| 93 | | - if ( $lockStatus->isOk() ) { |
| | 95 | + $subStatus = $this->doSingleLock( $key ); |
| | 96 | + $status->merge( $subStatus ); |
| | 97 | + if ( $subStatus->isOk() ) { |
| 94 | 98 | $lockedKeys[] = $key; |
| 95 | 99 | } else { |
| 96 | 100 | // Abort and unlock everything |
| 97 | | - $this->doUnlock( $lockedKeys ); |
| 98 | | - return $lockStatus; |
| | 101 | + $subStatus = $this->doUnlock( $lockedKeys ); |
| | 102 | + $status->merge( $subStatus ); |
| | 103 | + return $status; |
| 99 | 104 | } |
| 100 | 105 | } |
| 101 | 106 | |
| — | — | @@ -105,10 +110,8 @@ |
| 106 | 111 | $status = Status::newGood(); |
| 107 | 112 | |
| 108 | 113 | foreach ( $keys as $key ) { |
| 109 | | - $lockStatus = $this->doSingleUnlock( $key ); |
| 110 | | - if ( !$lockStatus->isOk() ) { |
| 111 | | - // append $lockStatus to $status |
| 112 | | - } |
| | 114 | + $subStatus = $this->doSingleUnlock( $key ); |
| | 115 | + $status->merge( $subStatus ); |
| 113 | 116 | } |
| 114 | 117 | |
| 115 | 118 | return $status; |
| — | — | @@ -172,16 +175,30 @@ |
| 173 | 176 | } |
| 174 | 177 | |
| 175 | 178 | /** |
| 176 | | - * Version of FileLockManager based on using per-row DB locks |
| | 179 | + * Version of FileLockManager based on using DB table locks. |
| | 180 | + * |
| | 181 | + * This is meant for multi-wiki systems that may share share some files. |
| | 182 | + * One or several lock database servers are set up having a `file_locking` |
| | 183 | + * table with one field, fl_key, the PRIMARY. The table engine should have |
| | 184 | + * row-level locking support. For performance, deadlock detection should be |
| | 185 | + * disabled and a very low lock-wait timeout should be put in server config. |
| | 186 | + * |
| | 187 | + * All lock requests for an item (identified by an abstract key string) will |
| | 188 | + * map to one bucket. Each bucket maps to a single server, and each server can |
| | 189 | + * have several or no fallback servers. Fallback servers recieve the same lock |
| | 190 | + * statements as the servers they are set as fallbacks for. This propagation is |
| | 191 | + * only best-effort; lock requests will not be blocked just because a fallback |
| | 192 | + * server cannot be contacted to recieve a copy of the lock request. |
| 177 | 193 | */ |
| 178 | 194 | class DBFileLockManager extends FileLockManager { |
| 179 | 195 | /** @var Array Map of bucket indexes to server names */ |
| 180 | 196 | protected $serverMap = array(); // (index => server name) |
| 181 | | - protected $shards; // number of severs to shard to |
| | 197 | + /** @var Array Map of servers to fallback server names */ |
| | 198 | + protected $serverFallbackMap = array(); // (server => (server1,server2,...)) |
| 182 | 199 | |
| 183 | 200 | /** @var Array List of active lock key names */ |
| 184 | 201 | protected $locksHeld = array(); // (key => 1) |
| 185 | | - /** $var Array Map active database connections (name => Database) */ |
| | 202 | + /** $var Array Map Lock-active database connections (name => Database) */ |
| 186 | 203 | protected $activeConns = array(); |
| 187 | 204 | |
| 188 | 205 | /** |
| — | — | @@ -190,27 +207,28 @@ |
| 191 | 208 | * integer keys, starting from 0, with server name strings as values. |
| 192 | 209 | * It should have no more than 16 items in the array. |
| 193 | 210 | * |
| 194 | | - * The `file_locking` table could be a MEMORY or innoDB table. |
| | 211 | + * The `file_locking` table should have row-level locking (e.g. innoDB). |
| 195 | 212 | * |
| 196 | 213 | * @param array $config |
| 197 | 214 | */ |
| 198 | 215 | function __construct( array $config ) { |
| 199 | 216 | $this->serverMap = $config['serverMap']; |
| 200 | | - $this->shards = count( $this->serverMap ); |
| | 217 | + $this->serverFallbackMap = $config['serverFallbackMap']; |
| 201 | 218 | } |
| 202 | 219 | |
| 203 | 220 | function doLock( array $keys ) { |
| 204 | 221 | $status = Status::newGood(); |
| 205 | 222 | |
| 206 | 223 | $keysToLock = array(); |
| 207 | | - // Get locks that need to be acquired... |
| | 224 | + // Get locks that need to be acquired and which server they map to... |
| 208 | 225 | foreach ( $keys as $key ) { |
| 209 | 226 | if ( isset( $this->locksHeld[$key] ) ) { |
| 210 | 227 | $status->warning( 'File already locked.' ); |
| 211 | 228 | } else { |
| 212 | 229 | $server = $this->getDBServerFromKey( $key ); |
| 213 | | - if ( $server === null ) { |
| | 230 | + if ( $server === null ) { // config error? |
| 214 | 231 | $status->fatal( "Lock server for $key is not set." ); |
| | 232 | + return $status; |
| 215 | 233 | } |
| 216 | 234 | $keysToLock[$server][] = $key; |
| 217 | 235 | } |
| — | — | @@ -218,26 +236,36 @@ |
| 219 | 237 | |
| 220 | 238 | $lockedKeys = array(); // files locked in this attempt |
| 221 | 239 | // Attempt to acquire these locks... |
| 222 | | - try { |
| 223 | | - foreach ( $keysToLock as $server => $keys ) { |
| 224 | | - $db = $this->getDB( $server ); |
| 225 | | - $db->select( 'file_locking', |
| 226 | | - '1', |
| 227 | | - array( 'fl_key' => $keys ), |
| 228 | | - __METHOD__, |
| 229 | | - array( 'FOR UPDATE' ) |
| 230 | | - ); |
| 231 | | - // Record locks as active |
| 232 | | - foreach ( $keys as $key ) { |
| 233 | | - $this->locksHeld[$key] = 1; // locked |
| | 240 | + foreach ( $keysToLock as $server => $keys ) { |
| | 241 | + // Acquire the locks for this server. Three main cases can happen: |
| | 242 | + // (a) Server is up; common case |
| | 243 | + // (b) Server is down but a fallback is up |
| | 244 | + // (c) Server is down and no fallbacks are up (or none defined) |
| | 245 | + $propagateToFallbacks = true; |
| | 246 | + try { |
| | 247 | + $this->lockingSelect( $server, $keys ); // SELECT FOR UPDATE |
| | 248 | + } catch ( DBError $e ) { |
| | 249 | + // Can we manage to lock on any of the fallback servers? |
| | 250 | + if ( !$this->lockingSelectFallbacks( $server, $keys ) ) { |
| | 251 | + // Abort and unlock everything |
| | 252 | + $status->fatal( "Could not contact the lock server." ); |
| | 253 | + $subStatus = $this->doUnlock( $lockedKeys ); |
| | 254 | + $status->merge( $subStatus ); |
| | 255 | + return $status; |
| | 256 | + } else { // recovered using fallbacks |
| | 257 | + $propagateToFallbacks = false; // done already |
| 234 | 258 | } |
| 235 | | - // Keep track of what locks where made in this attempt |
| 236 | | - $lockedKeys = array_merge( $lockedKeys, $keys ); |
| 237 | 259 | } |
| 238 | | - } catch ( DBConnectionError $e ) { |
| 239 | | - // Abort and unlock everything |
| 240 | | - $status->fatal( "Could not contact the lock database." ); |
| 241 | | - $this->doUnlock( $lockedKeys ); |
| | 260 | + // Propagate any locks to the fallback servers (best effort) |
| | 261 | + if ( $propagateToFallbacks ) { |
| | 262 | + $this->lockingSelectFallbacks( $server, $keys ); |
| | 263 | + } |
| | 264 | + // Record locks as active |
| | 265 | + foreach ( $keys as $key ) { |
| | 266 | + $this->locksHeld[$key] = 1; // locked |
| | 267 | + } |
| | 268 | + // Keep track of what locks were made in this attempt |
| | 269 | + $lockedKeys = array_merge( $lockedKeys, $keys ); |
| 242 | 270 | } |
| 243 | 271 | |
| 244 | 272 | return $status; |
| — | — | @@ -251,10 +279,10 @@ |
| 252 | 280 | unset( $this->locksHeld[$key] ); |
| 253 | 281 | // Reference count the locks held and COMMIT when zero |
| 254 | 282 | if ( !count( $this->locksHeld ) ) { |
| 255 | | - $this->commitOpenTransactions(); |
| | 283 | + $this->commitLockTransactions(); |
| 256 | 284 | } |
| 257 | 285 | } else { |
| 258 | | - // append warning to $status |
| | 286 | + $status->warning( "There is no file lock to unlock." ); |
| 259 | 287 | } |
| 260 | 288 | } |
| 261 | 289 | |
| — | — | @@ -262,43 +290,87 @@ |
| 263 | 291 | } |
| 264 | 292 | |
| 265 | 293 | /** |
| 266 | | - * Get a database connection for $server |
| | 294 | + * Get a database connection for $server and lock the rows for $keys |
| | 295 | + * |
| 267 | 296 | * @param $server string |
| 268 | | - * @return Database |
| | 297 | + * @param $keys Array |
| | 298 | + * @return void |
| 269 | 299 | */ |
| 270 | | - protected function getDB( $server ) { |
| | 300 | + protected function lockingSelect( $server, array $keys ) { |
| 271 | 301 | if ( !isset( $this->activeConns[$server] ) ) { |
| 272 | 302 | $this->activeConns[$server] = wfGetDB( DB_MASTER, array(), $server ); |
| 273 | 303 | $this->activeConns[$server]->begin(); // start transaction |
| 274 | 304 | } |
| 275 | | - return $this->activeConns[$server]; |
| | 305 | + $this->activeConns[$server]->select( |
| | 306 | + 'file_locking', |
| | 307 | + '1', |
| | 308 | + array( 'fl_key' => $keys ), |
| | 309 | + __METHOD__, |
| | 310 | + array( 'FOR UPDATE' ) |
| | 311 | + ); |
| 276 | 312 | } |
| 277 | 313 | |
| 278 | 314 | /** |
| 279 | | - * Commit all changes to active databases |
| | 315 | + * Propagate any locks to the fallback servers for $server. |
| | 316 | + * This should avoid throwing any exceptions. |
| | 317 | + * |
| | 318 | + * @param $server string |
| | 319 | + * @param $keys Array |
| | 320 | + * @return bool Locks made on at least one fallback server |
| | 321 | + */ |
| | 322 | + protected function lockingSelectFallbacks( $server, array $keys ) { |
| | 323 | + $locksMade = false; |
| | 324 | + if ( isset( $this->serverFallbackMap[$server] ) ) { |
| | 325 | + // Propagate the $server locks to each fallback for $server... |
| | 326 | + foreach ( $this->serverFallbackMap[$server] as $fallbackServer ) { |
| | 327 | + try { |
| | 328 | + $this->doLockingSelect( $fallbackServer, $keys ); // SELECT FOR UPDATE |
| | 329 | + $locksMade = true; |
| | 330 | + } catch ( DBError $e ) { |
| | 331 | + // oh well; best effort |
| | 332 | + } |
| | 333 | + } |
| | 334 | + } |
| | 335 | + return $locksMade; |
| | 336 | + } |
| | 337 | + |
| | 338 | + /** |
| | 339 | + * Commit all changes to lock-active databases. |
| | 340 | + * This should avoid throwing any exceptions. |
| | 341 | + * |
| 280 | 342 | * @return void |
| 281 | 343 | */ |
| 282 | | - protected function commitOpenTransactions() { |
| | 344 | + protected function commitLockTransactions() { |
| 283 | 345 | try { |
| 284 | 346 | foreach ( $this->activeConns as $server => $db ) { |
| 285 | 347 | $db->commit(); // finish transaction |
| 286 | 348 | unset( $this->activeConns[$server] ); |
| 287 | 349 | } |
| 288 | | - } catch ( DBConnectionError $e ) { |
| | 350 | + } catch ( DBError $e ) { |
| 289 | 351 | // oh well |
| 290 | 352 | } |
| 291 | 353 | } |
| 292 | 354 | |
| 293 | 355 | /** |
| | 356 | + * Get the bucket for lock key |
| | 357 | + * |
| | 358 | + * @param $key string |
| | 359 | + * @return int |
| | 360 | + */ |
| | 361 | + protected function getBucketFromKey( $key ) { |
| | 362 | + $hash = str_pad( md5( $key ), 32, '0', STR_PAD_LEFT ); // 32 char hash |
| | 363 | + $prefix = substr( $hash, 0, 2 ); // first 2 hex chars (8 bits) |
| | 364 | + return ( intval( base_convert( $prefix, 16, 10 ) ) % count( $this->serverMap ) ); |
| | 365 | + } |
| | 366 | + |
| | 367 | + /** |
| 294 | 368 | * Get the server name for lock key |
| | 369 | + * |
| 295 | 370 | * @param $key string |
| 296 | 371 | * @return string|null |
| 297 | 372 | */ |
| 298 | 373 | protected function getDBServerFromKey( $key ) { |
| 299 | | - $hash = str_pad( md5( $key ), 32, '0', STR_PAD_LEFT ); // 32 char hash |
| 300 | | - $prefix = substr( $hash, 0, 2 ); // first 2 hex chars (8 bits) |
| 301 | | - $bucket = intval( base_convert( $prefix, 16, 10 ) ) % $this->shards; |
| 302 | | - |
| | 374 | + $bucket = $this->getBucketFromKey( $key ); |
| 303 | 375 | if ( isset( $this->serverMap[$bucket] ) ) { |
| 304 | 376 | return $this->serverMap[$bucket]; |
| 305 | 377 | } else { |
| — | — | @@ -312,11 +384,13 @@ |
| 313 | 385 | * Simple version of FileLockManager that does nothing |
| 314 | 386 | */ |
| 315 | 387 | class NullFileLockManager extends FileLockManager { |
| 316 | | - public function doLock( array $keys ) { |
| | 388 | + function __construct( array $config ) {} |
| | 389 | + |
| | 390 | + function doLock( array $keys ) { |
| 317 | 391 | return Status::newGood(); |
| 318 | 392 | } |
| 319 | 393 | |
| 320 | | - public function doUnlock( array $keys ) { |
| | 394 | + function doUnlock( array $keys ) { |
| 321 | 395 | return Status::newGood(); |
| 322 | 396 | } |
| 323 | 397 | } |
| Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php |
| — | — | @@ -1,12 +1,11 @@ |
| 2 | 2 | <?php |
| 3 | 3 | /** |
| 4 | | - * Base class for all file backend classes |
| 5 | | - * |
| 6 | 4 | * @file |
| 7 | 5 | * @ingroup FileRepo |
| 8 | 6 | */ |
| 9 | 7 | |
| 10 | 8 | /** |
| | 9 | + * Base class for all file backend classes. |
| 11 | 10 | * This class defines the methods as abstract that |
| 12 | 11 | * must be implemented in all file backend classes. |
| 13 | 12 | * |
| — | — | @@ -86,7 +85,7 @@ |
| 87 | 86 | |
| 88 | 87 | /** |
| 89 | 88 | * Copy a file from one storage path to another in the backend. |
| 90 | | - * This can be left as a dummy function as long as hasNativeMove() returns false. |
| | 89 | + * This can be left as a dummy function as long as hasMove() returns false. |
| 91 | 90 | * Do not call this function from places other than FileOp. |
| 92 | 91 | * $params include: |
| 93 | 92 | * source : source storage path |
| — | — | @@ -131,7 +130,7 @@ |
| 132 | 131 | * |
| 133 | 132 | * @return bool |
| 134 | 133 | */ |
| 135 | | - public function hasNativeMove(); |
| | 134 | + public function hasMove(); |
| 136 | 135 | |
| 137 | 136 | /** |
| 138 | 137 | * Check if a file exits at a storage path in the backend. |
| — | — | @@ -199,6 +198,9 @@ |
| 200 | 199 | /** |
| 201 | 200 | * Build a new object from configuration. |
| 202 | 201 | * This should only be called from within FileRepo classes. |
| | 202 | + * $config includes: |
| | 203 | + * 'name' : The name of this backend |
| | 204 | + * 'lockManager' : The lock manager to use |
| 203 | 205 | * |
| 204 | 206 | * @param $config Array |
| 205 | 207 | */ |
| — | — | @@ -207,14 +209,14 @@ |
| 208 | 210 | $this->lockManager = $config['lockManger']; |
| 209 | 211 | } |
| 210 | 212 | |
| | 213 | + function hasMove() { |
| | 214 | + return false; // not implemented |
| | 215 | + } |
| | 216 | + |
| 211 | 217 | function move( array $params ) { |
| 212 | 218 | throw new MWException( "This function is not implemented." ); |
| 213 | 219 | } |
| 214 | 220 | |
| 215 | | - function hasNativeMove() { |
| 216 | | - return false; // not implemented |
| 217 | | - } |
| 218 | | - |
| 219 | 221 | /** |
| 220 | 222 | * Get the list of supported operations and their corresponding FileOp classes. |
| 221 | 223 | * Subclasses should implement these using FileOp subclasses |
| — | — | @@ -225,8 +227,8 @@ |
| 226 | 228 | return array( |
| 227 | 229 | 'store' => 'FileStoreOp', |
| 228 | 230 | 'copy' => 'FileCopyOp', |
| 229 | | - 'delete' => 'FileDeleteOp', |
| 230 | 231 | 'move' => 'FileMoveOp', |
| | 232 | + 'delete' => 'FileDeleteOp', |
| 231 | 233 | 'concatenate' => 'FileConcatenateOp' |
| 232 | 234 | ); |
| 233 | 235 | } |
| — | — | @@ -235,7 +237,7 @@ |
| 236 | 238 | $supportedOps = $this->supportedOperations(); |
| 237 | 239 | |
| 238 | 240 | $performOps = array(); // array of FileOp objects |
| 239 | | - //// Build up ordered array of FileOps... |
| | 241 | + // Build up ordered array of FileOps... |
| 240 | 242 | foreach ( $ops as $operation ) { |
| 241 | 243 | $opName = $operation['operation']; |
| 242 | 244 | if ( isset( $supportedOps[$opName] ) ) { |
| — | — | @@ -249,6 +251,7 @@ |
| 250 | 252 | throw new MWException( "Operation `$opName` is not supported." ); |
| 251 | 253 | } |
| 252 | 254 | } |
| | 255 | + |
| 253 | 256 | return $performOps; |
| 254 | 257 | } |
| 255 | 258 | |
| — | — | @@ -258,45 +261,30 @@ |
| 259 | 262 | $performOps = $this->getOperations( $ops ); |
| 260 | 263 | // Attempt each operation; abort on failure... |
| 261 | 264 | foreach ( $performOps as $index => $transaction ) { |
| 262 | | - $tStatus = $transaction->attempt(); |
| 263 | | - if ( !$tStatus->isOK() ) { |
| 264 | | - // merge $tStatus with $status |
| 265 | | - // Revert everything done so far and error out |
| 266 | | - $tStatus = $this->revertOperations( $performOps, $index ); |
| 267 | | - // merge $tStatus with $status |
| | 265 | + $subStatus = $transaction->attempt(); |
| | 266 | + $status->merge( $subStatus ); |
| | 267 | + if ( !$subStatus->isOK() ) { // operation failed? |
| | 268 | + // Revert everything done so far and abort. |
| | 269 | + // Do this by reverting each previous operation in reverse order. |
| | 270 | + $pos = $index - 1; // last one failed; no need to revert() |
| | 271 | + while ( $pos >= 0 ) { |
| | 272 | + $subStatus = $performOps[$pos]->revert(); |
| | 273 | + $status->merge( $subStatus ); |
| | 274 | + $pos--; |
| | 275 | + } |
| 268 | 276 | return $status; |
| 269 | 277 | } |
| 270 | 278 | } |
| 271 | 279 | // Finish each operation... |
| 272 | 280 | foreach ( $performOps as $index => $transaction ) { |
| 273 | | - $tStatus = $transaction->finish(); |
| 274 | | - // merge $tStatus with $status |
| | 281 | + $subStatus = $transaction->finish(); |
| | 282 | + $status->merge( $subStatus ); |
| 275 | 283 | } |
| | 284 | + // Make sure status is OK, despite any finish() fatals |
| | 285 | + $status->setResult( true ); |
| 276 | 286 | return $status; |
| 277 | 287 | } |
| 278 | 288 | |
| 279 | | - /** |
| 280 | | - * Revert a series of operations in reverse order. |
| 281 | | - * If $index is passed, then we revert all items in |
| 282 | | - * $ops from 0 to $index (inclusive). |
| 283 | | - * |
| 284 | | - * @param $ops Array List of FileOp objects |
| 285 | | - * @param $index integer |
| 286 | | - * @return Status |
| 287 | | - */ |
| 288 | | - private function revertOperations( array $ops, $index = false ) { |
| 289 | | - $status = Status::newGood(); |
| 290 | | - $pos = ( $index !== false ) |
| 291 | | - ? $index // use provided index |
| 292 | | - : $pos = count( $ops ) - 1; // last element (or -1) |
| 293 | | - while ( $pos >= 0 ) { |
| 294 | | - $tStatus = $ops[$pos]->revert(); |
| 295 | | - // merge $tStatus with $status |
| 296 | | - $pos--; |
| 297 | | - } |
| 298 | | - return $status; |
| 299 | | - } |
| 300 | | - |
| 301 | 289 | final public function lockFiles( array $paths ) { |
| 302 | 290 | // Locks should be specific to this backend location |
| 303 | 291 | $backendKey = get_class( $this ) . '-' . $this->getName(); |
| — | — | @@ -445,7 +433,7 @@ |
| 446 | 434 | |
| 447 | 435 | /** |
| 448 | 436 | * Store a file into the backend from a file on disk. |
| 449 | | - * $params include: |
| | 437 | + * Parameters must match FileBackend::store(), which include: |
| 450 | 438 | * source : source path on disk |
| 451 | 439 | * dest : destination storage path |
| 452 | 440 | * overwriteDest : do nothing and pass if an identical file exists at destination |
| — | — | @@ -470,7 +458,7 @@ |
| 471 | 459 | $params = array( 'source' => $this->params['dest'] ); |
| 472 | 460 | $status = $this->backend->delete( $params ); |
| 473 | 461 | if ( !$status->isOK() ) { |
| 474 | | - return $status; |
| | 462 | + return $status; // also can't restore any dest file |
| 475 | 463 | } |
| 476 | 464 | // Restore any file that was at the destination |
| 477 | 465 | $status = $this->restoreDest(); |
| — | — | @@ -531,7 +519,7 @@ |
| 532 | 520 | |
| 533 | 521 | /** |
| 534 | 522 | * Copy a file from one storage path to another in the backend. |
| 535 | | - * $params include: |
| | 523 | + * Parameters must match FileBackend::copy(), which include: |
| 536 | 524 | * source : source storage path |
| 537 | 525 | * dest : destination storage path |
| 538 | 526 | * overwriteDest : do nothing and pass if an identical file exists at destination |
| — | — | @@ -554,7 +542,7 @@ |
| 555 | 543 | $params = array( 'source' => $this->params['dest'] ); |
| 556 | 544 | $status = $this->backend->delete( $params ); |
| 557 | 545 | if ( !$status->isOK() ) { |
| 558 | | - return $status; |
| | 546 | + return $status; // also can't restore any dest file |
| 559 | 547 | } |
| 560 | 548 | // Restore any file that was at the destination |
| 561 | 549 | $status = $this->restoreDest(); |
| — | — | @@ -572,7 +560,7 @@ |
| 573 | 561 | |
| 574 | 562 | /** |
| 575 | 563 | * Move a file from one storage path to another in the backend. |
| 576 | | - * $params include: |
| | 564 | + * Parameters must match FileBackend::move(), which include: |
| 577 | 565 | * source : source storage path |
| 578 | 566 | * dest : destination storage path |
| 579 | 567 | * overwriteDest : do nothing and pass if an identical file exists at destination |
| — | — | @@ -586,7 +574,7 @@ |
| 587 | 575 | return $status; |
| 588 | 576 | } |
| 589 | 577 | // Native moves: move the file into the destination |
| 590 | | - if ( $this->backend->hasNativeMove() ) { |
| | 578 | + if ( $this->backend->hasMove() ) { |
| 591 | 579 | $status = $this->backend->move( $this->params ); |
| 592 | 580 | // Non-native moves: copy the file into the destination |
| 593 | 581 | } else { |
| — | — | @@ -597,21 +585,21 @@ |
| 598 | 586 | |
| 599 | 587 | function doRevert() { |
| 600 | 588 | // Native moves: move the file back to the source |
| 601 | | - if ( $this->backend->hasNativeMove() ) { |
| | 589 | + if ( $this->backend->hasMove() ) { |
| 602 | 590 | $params = array( |
| 603 | 591 | 'source' => $this->params['dest'], |
| 604 | 592 | 'dest' => $this->params['source'] |
| 605 | 593 | ); |
| 606 | 594 | $status = $this->backend->move( $params ); |
| 607 | 595 | if ( !$status->isOK() ) { |
| 608 | | - return $status; |
| | 596 | + return $status; // also can't restore any dest file |
| 609 | 597 | } |
| 610 | 598 | // Non-native moves: remove the file saved to the destination |
| 611 | 599 | } else { |
| 612 | 600 | $params = array( 'source' => $this->params['dest'] ); |
| 613 | 601 | $status = $this->backend->delete( $params ); |
| 614 | 602 | if ( !$status->isOK() ) { |
| 615 | | - return $status; |
| | 603 | + return $status; // also can't restore any dest file |
| 616 | 604 | } |
| 617 | 605 | } |
| 618 | 606 | // Restore any file that was at the destination |
| — | — | @@ -621,7 +609,7 @@ |
| 622 | 610 | |
| 623 | 611 | function doFinish() { |
| 624 | 612 | // Native moves: nothing is at the source anymore |
| 625 | | - if ( $this->backend->hasNativeMove() ) { |
| | 613 | + if ( $this->backend->hasMove() ) { |
| 626 | 614 | $status = Status::newGood(); |
| 627 | 615 | // Non-native moves: delete the source file |
| 628 | 616 | } else { |
| — | — | @@ -634,7 +622,7 @@ |
| 635 | 623 | |
| 636 | 624 | /** |
| 637 | 625 | * Combines files from severals storage paths into a new file in the backend. |
| 638 | | - * $params include: |
| | 626 | + * Parameters must match FileBackend::concatenate(), which include: |
| 639 | 627 | * sources : ordered source storage paths (e.g. chunk1,chunk2,...) |
| 640 | 628 | * dest : destination storage path |
| 641 | 629 | * overwriteDest : do nothing and pass if an identical file exists at destination |
| — | — | @@ -657,7 +645,7 @@ |
| 658 | 646 | $params = array( 'source' => $this->params['dest'] ); |
| 659 | 647 | $status = $this->backend->delete( $params ); |
| 660 | 648 | if ( !$status->isOK() ) { |
| 661 | | - return $status; |
| | 649 | + return $status; // also can't restore any dest file |
| 662 | 650 | } |
| 663 | 651 | // Restore any file that was at the destination |
| 664 | 652 | $status = $this->restoreDest(); |
| — | — | @@ -675,7 +663,7 @@ |
| 676 | 664 | |
| 677 | 665 | /** |
| 678 | 666 | * Delete a file at the storage path. |
| 679 | | - * $params include: |
| | 667 | + * Parameters must match FileBackend::delete(), which include: |
| 680 | 668 | * source : source storage path |
| 681 | 669 | * ignoreMissingSource : don't return an error if the file does not exist |
| 682 | 670 | */ |