Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php |
— | — | @@ -7,32 +7,47 @@ |
8 | 8 | /** |
9 | 9 | * This class defines a multi-write backend. Multiple backends can |
10 | 10 | * be registered to this proxy backend it will act as a single backend. |
| 11 | + * Use this only if all access to the backends is through an instance of this class. |
11 | 12 | * |
12 | 13 | * The order that the backends are defined sets the priority of which |
13 | 14 | * backend is read from or written to first. Functions like fileExists() |
14 | 15 | * and getFileProps() will return information based on the first backend |
15 | | - * that has the file (normally both should have it anyway). |
| 16 | + * that has the file (normally both should have it anyway). Functions like |
| 17 | + * getFileList() will return results from the first backend that is not |
| 18 | + * declared as non-persistent cache. This is done for consistency. |
16 | 19 | * |
17 | 20 | * All write operations are performed on all backends. |
18 | 21 | * If an operation fails on one backend it will be rolled back from the others. |
19 | | - * |
20 | | - * To avoid excess overhead, set the the highest priority backend to use |
21 | | - * a generic FileLockManager and the others to use NullLockManager. This can |
22 | | - * only be done if all access to the backends is through an instance of this class. |
23 | 22 | */ |
24 | 23 | class FileBackendMultiWrite extends FileBackendBase { |
25 | | - /** @var Array Prioritized list of FileBackend classes */ |
26 | | - protected $fileBackends = array(); |
| 24 | + /** @var Array Prioritized list of FileBackend objects */ |
| 25 | + protected $fileBackends = array(); // array of (backend index => backends) |
| 26 | + /** @var Array List of FileBackend object informations */ |
| 27 | + protected $fileBackendsInfo = array(); // array (backend index => array of settings) |
27 | 28 | |
| 29 | + /** |
| 30 | + * Construct a proxy backend that consist of several internal backends. |
| 31 | + * $config contains: |
| 32 | + * 'name' : The name of the proxy backend |
| 33 | + * 'lockManger' : FileLockManager instance |
| 34 | + * 'backends' : Array of ( backend object, settings ) pairs. |
| 35 | + * The settings per backend include: |
| 36 | + * 'isCache': The backend is non-persistent |
| 37 | + * @param $config Array |
| 38 | + */ |
28 | 39 | public function __construct( array $config ) { |
29 | 40 | $this->name = $config['name']; |
30 | | - $this->fileBackends = $config['backends']; |
| 41 | + $this->lockManager = $config['lockManger']; |
| 42 | + foreach ( $config['backends'] as $index => $info ) { |
| 43 | + list( $backend, $settings ) = $info; |
| 44 | + $this->fileBackends[$index] = $backend; |
| 45 | + // Default backend settings |
| 46 | + $defaults = array( 'isCache' => false ); |
| 47 | + // Apply custom backend settings to defaults |
| 48 | + $this->fileBackendsInfo[$index] = $info + $defaults; |
| 49 | + } |
31 | 50 | } |
32 | 51 | |
33 | | - function hasNativeMove() { |
34 | | - return true; // this is irrelevant |
35 | | - } |
36 | | - |
37 | 52 | final public function doOperations( array $ops ) { |
38 | 53 | $status = Status::newGood(); |
39 | 54 | |
— | — | @@ -98,6 +113,10 @@ |
99 | 114 | return $this->doOperation( array( $op ) ); |
100 | 115 | } |
101 | 116 | |
| 117 | + function canMove( array $params ) { |
| 118 | + return true; // this is irrelevant |
| 119 | + } |
| 120 | + |
102 | 121 | function move( array $params ) { |
103 | 122 | $op = array( 'operation' => 'move' ) + $params; |
104 | 123 | return $this->doOperation( array( $op ) ); |
— | — | @@ -157,26 +176,13 @@ |
158 | 177 | return null; |
159 | 178 | } |
160 | 179 | |
161 | | - function lockFiles( array $paths ) { |
162 | | - $status = Status::newGood(); |
| 180 | + function getFileList( array $params ) { |
163 | 181 | foreach ( $this->backends as $index => $backend ) { |
164 | | - $status->merge( $backend->lockFiles( $paths ) ); |
165 | | - if ( !$status->isOk() ) { |
166 | | - // Lock failed: release the locks done so far each backend |
167 | | - for ( $i=0; $i < $index; $i++ ) { // don't do backend $index since it failed |
168 | | - $status->merge( $backend->unlockFiles( $paths ) ); |
169 | | - } |
170 | | - return $status; |
| 182 | + // Skip cache backends (like one using memcached) |
| 183 | + if ( !$this->fileBackendsInfo[$index]['isCache'] ) { |
| 184 | + return $backend->getFileList( $params ); |
171 | 185 | } |
172 | 186 | } |
173 | | - return $status; |
| 187 | + return array(); |
174 | 188 | } |
175 | | - |
176 | | - function unlockFiles( array $paths ) { |
177 | | - $status = Status::newGood(); |
178 | | - foreach ( $this->backends as $backend ) { |
179 | | - $status->merge( $backend->unlockFile( $paths ) ); |
180 | | - } |
181 | | - return $status; |
182 | | - } |
183 | 189 | } |
Index: branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php |
— | — | @@ -75,6 +75,10 @@ |
76 | 76 | return $this->store( $params ); // both source and dest are on FS |
77 | 77 | } |
78 | 78 | |
| 79 | + function canMove( array $params ) { |
| 80 | + return true; |
| 81 | + } |
| 82 | + |
79 | 83 | function move( array $params ) { |
80 | 84 | $status = Status::newGood(); |
81 | 85 | |
— | — | @@ -257,11 +261,10 @@ |
258 | 262 | return File::getPropsFromPath( $source ); |
259 | 263 | } |
260 | 264 | |
261 | | - // Not suitable for massive listings |
262 | 265 | function getFileList( array $params ) { |
263 | 266 | list( $c, $dir ) = $this->resolveVirtualPath( $params['directory'] ); |
264 | | - if ( $dir === null ) { // valid storage path |
265 | | - return new FileIterator( '' ); // empty result |
| 267 | + if ( $dir === null ) { // invalid storage path |
| 268 | + return array(); // empty result |
266 | 269 | } |
267 | 270 | return new FileIterator( $dir ); |
268 | 271 | } |
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php |
— | — | @@ -17,8 +17,24 @@ |
18 | 18 | */ |
19 | 19 | abstract class FileBackendBase { |
20 | 20 | protected $name; // unique backend name |
| 21 | + /** @var FileLockManager */ |
| 22 | + protected $lockManager; |
21 | 23 | |
22 | 24 | /** |
| 25 | + * Build a new object from configuration. |
| 26 | + * This should only be called from within FileRepo classes. |
| 27 | + * $config includes: |
| 28 | + * 'name' : The name of this backend |
| 29 | + * 'lockManager' : The file lock manager to use |
| 30 | + * |
| 31 | + * @param $config Array |
| 32 | + */ |
| 33 | + public function __construct( array $config ) { |
| 34 | + $this->name = $config['name']; |
| 35 | + $this->lockManager = $config['lockManger']; |
| 36 | + } |
| 37 | + |
| 38 | + /** |
23 | 39 | * We may have multiple different backends of the same type. |
24 | 40 | * For example, we can have two Swift backends using different proxies. |
25 | 41 | * All backend instances must have unique names. |
— | — | @@ -40,11 +56,11 @@ |
41 | 57 | * The inner array contains the parameters, E.G: |
42 | 58 | * <code> |
43 | 59 | * $ops = array( |
44 | | - * array( |
45 | | - * 'operation' => 'store', |
46 | | - * 'src' => '/tmp/uploads/picture.png', |
47 | | - * 'dest' => 'mwstore://container/uploadedFilename.png' |
48 | | - * ) |
| 60 | + * array( |
| 61 | + * 'operation' => 'store', |
| 62 | + * 'src' => '/tmp/uploads/picture.png', |
| 63 | + * 'dest' => 'mwstore://container/uploadedFilename.png' |
| 64 | + * ) |
49 | 65 | * ); |
50 | 66 | * </code> |
51 | 67 | * |
— | — | @@ -54,23 +70,13 @@ |
55 | 71 | abstract public function doOperations( array $ops ); |
56 | 72 | |
57 | 73 | /** |
58 | | - * Return a list of FileOp objects from a list of operations. |
59 | | - * An exception is thrown if an unsupported operation is requested. |
60 | | - * |
61 | | - * @param Array $ops Same format as doOperations() |
62 | | - * @return Array |
63 | | - * @throws MWException |
64 | | - */ |
65 | | - abstract public function getOperations( array $ops ); |
66 | | - |
67 | | - /** |
68 | 74 | * Store a file into the backend from a file on disk. |
69 | 75 | * Do not call this function from places outside FileBackend and FileOp. |
70 | 76 | * $params include: |
71 | | - * source : source path on disk |
72 | | - * dest : destination storage path |
73 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
74 | | - * overwriteSame : override any existing file at destination |
| 77 | + * source : source path on disk |
| 78 | + * dest : destination storage path |
| 79 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 80 | + * overwriteSame : override any existing file at destination |
75 | 81 | * |
76 | 82 | * @param Array $params |
77 | 83 | * @return Status |
— | — | @@ -81,12 +87,12 @@ |
82 | 88 | * Copy a file from one storage path to another in the backend. |
83 | 89 | * Do not call this function from places outside FileBackend and FileOp. |
84 | 90 | * $params include: |
85 | | - * source : source storage path |
86 | | - * dest : destination storage path |
87 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
88 | | - * overwriteSame : override any existing file at destination |
| 91 | + * source : source storage path |
| 92 | + * dest : destination storage path |
| 93 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 94 | + * overwriteSame : override any existing file at destination |
89 | 95 | * |
90 | | - * @param Array $params |
| 96 | + * @param Array $params |
91 | 97 | * @return Status |
92 | 98 | */ |
93 | 99 | abstract public function copy( array $params ); |
— | — | @@ -96,12 +102,12 @@ |
97 | 103 | * This can be left as a dummy function as long as hasMove() returns false. |
98 | 104 | * Do not call this function from places outside FileBackend and FileOp. |
99 | 105 | * $params include: |
100 | | - * source : source storage path |
101 | | - * dest : destination storage path |
102 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
103 | | - * overwriteSame : override any existing file at destination |
| 106 | + * source : source storage path |
| 107 | + * dest : destination storage path |
| 108 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 109 | + * overwriteSame : override any existing file at destination |
104 | 110 | * |
105 | | - * @param Array $params |
| 111 | + * @param Array $params |
106 | 112 | * @return Status |
107 | 113 | */ |
108 | 114 | abstract public function move( array $params ); |
— | — | @@ -110,10 +116,10 @@ |
111 | 117 | * Delete a file at the storage path. |
112 | 118 | * Do not call this function from places outside FileBackend and FileOp. |
113 | 119 | * $params include: |
114 | | - * source : source storage path |
115 | | - * ignoreMissingSource : don't return an error if the file does not exist |
| 120 | + * source : source storage path |
| 121 | + * ignoreMissingSource : don't return an error if the file does not exist |
116 | 122 | * |
117 | | - * @param Array $params |
| 123 | + * @param Array $params |
118 | 124 | * @return Status |
119 | 125 | */ |
120 | 126 | abstract public function delete( array $params ); |
— | — | @@ -122,24 +128,25 @@ |
123 | 129 | * Combines files from severals storage paths into a new file in the backend. |
124 | 130 | * Do not call this function from places outside FileBackend and FileOp. |
125 | 131 | * $params include: |
126 | | - * source : source storage path |
127 | | - * dest : destination storage path |
128 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
129 | | - * overwriteSame : override any existing file at destination |
| 132 | + * source : source storage path |
| 133 | + * dest : destination storage path |
| 134 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 135 | + * overwriteSame : override any existing file at destination |
130 | 136 | * |
131 | | - * @param Array $params |
| 137 | + * @param Array $params |
132 | 138 | * @return Status |
133 | 139 | */ |
134 | 140 | abstract public function concatenate( array $params ); |
135 | 141 | |
136 | 142 | /** |
137 | | - * Whether this backend implements move() and can handle this potential move. |
| 143 | + * Whether this backend implements move() and can handle a potential move. |
138 | 144 | * For example, moving objects accross containers may not be supported. |
139 | 145 | * Do not call this function from places outside FileBackend and FileOp. |
140 | 146 | * $params include: |
141 | | - * source : source storage path |
142 | | - * dest : destination storage path |
| 147 | + * source : source storage path |
| 148 | + * dest : destination storage path |
143 | 149 | * |
| 150 | + * @param Array $params |
144 | 151 | * @return bool |
145 | 152 | */ |
146 | 153 | abstract public function canMove( array $params ); |
— | — | @@ -148,9 +155,9 @@ |
149 | 156 | * Check if a file exits at a storage path in the backend. |
150 | 157 | * Do not call this function from places outside FileBackend and FileOp. |
151 | 158 | * $params include: |
152 | | - * source : source storage path |
| 159 | + * source : source storage path |
153 | 160 | * |
154 | | - * @param Array $params |
| 161 | + * @param Array $params |
155 | 162 | * @return bool |
156 | 163 | */ |
157 | 164 | abstract public function fileExists( array $params ); |
— | — | @@ -158,9 +165,9 @@ |
159 | 166 | /** |
160 | 167 | * Get the properties of the file that exists at a storage path in the backend |
161 | 168 | * $params include: |
162 | | - * source : source storage path |
| 169 | + * source : source storage path |
163 | 170 | * |
164 | | - * @param Array $params |
| 171 | + * @param Array $params |
165 | 172 | * @return Array|null Gives null if the file does not exist |
166 | 173 | */ |
167 | 174 | abstract public function getFileProps( array $params ); |
— | — | @@ -171,9 +178,9 @@ |
172 | 179 | * must be sent if streaming began, while none should be sent otherwise. |
173 | 180 | * Implementations should flush the output buffer before sending data. |
174 | 181 | * $params include: |
175 | | - * source : source storage path |
| 182 | + * source : source storage path |
176 | 183 | * |
177 | | - * @param Array $params |
| 184 | + * @param Array $params |
178 | 185 | * @return Status |
179 | 186 | */ |
180 | 187 | abstract public function streamFile( array $params ); |
— | — | @@ -185,29 +192,33 @@ |
186 | 193 | * in the container should be listed. If of the form "mwstore://container/dir", |
187 | 194 | * then all items under that container directory should be listed. |
188 | 195 | * $params include: |
189 | | - * directory : storage path directory. |
190 | | - * @return Iterator |
| 196 | + * directory : storage path directory. |
| 197 | + * |
| 198 | + * @return Iterator|Array |
191 | 199 | */ |
192 | 200 | abstract public function getFileList( array $params ); |
193 | 201 | |
194 | 202 | /** |
195 | 203 | * Get a local copy on disk of the file at a storage path in the backend |
196 | 204 | * $params include: |
197 | | - * source : source storage path |
| 205 | + * source : source storage path |
198 | 206 | * |
199 | | - * @param Array $params |
| 207 | + * @param Array $params |
200 | 208 | * @return TempLocalFile|null Temporary file or null on failure |
201 | 209 | */ |
202 | 210 | abstract public function getLocalCopy( array $params ); |
203 | 211 | |
204 | 212 | /** |
205 | 213 | * Lock the files at the given storage paths in the backend. |
| 214 | + * This should either lock all the files or none (on failure). |
206 | 215 | * Do not call this function from places outside FileBackend and FileOp. |
207 | 216 | * |
208 | 217 | * @param $sources Array Source storage paths |
209 | 218 | * @return Status |
210 | 219 | */ |
211 | | - abstract public function lockFiles( array $sources ); |
| 220 | + final public function lockFiles( array $paths ) { |
| 221 | + return $this->lockManager->lock( $this->getLockResourcePaths( $paths ) ); |
| 222 | + } |
212 | 223 | |
213 | 224 | /** |
214 | 225 | * Unlock the files at the given storage paths in the backend. |
— | — | @@ -216,30 +227,30 @@ |
217 | 228 | * @param $sources Array Source storage paths |
218 | 229 | * @return Status |
219 | 230 | */ |
220 | | - abstract public function unlockFiles( array $sources ); |
| 231 | + final public function unlockFiles( array $paths ) { |
| 232 | + return $this->lockManager->unlock( $this->getLockResourcePaths( $paths ) ); |
| 233 | + } |
| 234 | + |
| 235 | + /** |
| 236 | + * Prefix a list of storage paths to use as resource paths to lock |
| 237 | + * |
| 238 | + * @param $paths Array |
| 239 | + * @return Array |
| 240 | + */ |
| 241 | + private function getLockResourcePaths( array $paths ) { |
| 242 | + $backendKey = get_class( $this ) . ':' . $this->getName(); |
| 243 | + $res = array(); |
| 244 | + foreach( $paths as $path ) { |
| 245 | + $res[] = "{$backendKey}:{$path}"; |
| 246 | + } |
| 247 | + return $res; |
| 248 | + } |
221 | 249 | } |
222 | 250 | |
223 | 251 | /** |
224 | 252 | * Base class for all single-write backends |
225 | 253 | */ |
226 | 254 | abstract class FileBackend extends FileBackendBase { |
227 | | - /** @var FileLockManager */ |
228 | | - protected $lockManager; |
229 | | - |
230 | | - /** |
231 | | - * Build a new object from configuration. |
232 | | - * This should only be called from within FileRepo classes. |
233 | | - * $config includes: |
234 | | - * 'name' : The name of this backend |
235 | | - * 'lockManager' : The lock manager to use |
236 | | - * |
237 | | - * @param $config Array |
238 | | - */ |
239 | | - public function __construct( array $config ) { |
240 | | - $this->name = $config['name']; |
241 | | - $this->lockManager = $config['lockManger']; |
242 | | - } |
243 | | - |
244 | 255 | function canMove( array $params ) { |
245 | 256 | return false; // not implemented |
246 | 257 | } |
— | — | @@ -263,6 +274,14 @@ |
264 | 275 | ); |
265 | 276 | } |
266 | 277 | |
| 278 | + /** |
| 279 | + * Return a list of FileOp objects from a list of operations. |
| 280 | + * An exception is thrown if an unsupported operation is requested. |
| 281 | + * |
| 282 | + * @param Array $ops Same format as doOperations() |
| 283 | + * @return Array |
| 284 | + * @throws MWException |
| 285 | + */ |
267 | 286 | final public function getOperations( array $ops ) { |
268 | 287 | $supportedOps = $this->supportedOperations(); |
269 | 288 | |
— | — | @@ -333,28 +352,7 @@ |
334 | 353 | return $status; |
335 | 354 | } |
336 | 355 | |
337 | | - final public function lockFiles( array $paths ) { |
338 | | - return $this->lockManager->lock( $this->getLockResourcePaths( $paths ) ); |
339 | | - } |
340 | | - |
341 | | - final public function unlockFiles( array $paths ) { |
342 | | - return $this->lockManager->unlock( $this->getLockResourcePaths( $paths ) ); |
343 | | - } |
344 | | - |
345 | 356 | /** |
346 | | - * Get a prefix to use for locking keys |
347 | | - * @return string |
348 | | - */ |
349 | | - private function getLockResourcePaths( array $paths ) { |
350 | | - $backendKey = get_class( $this ) . ':' . $this->getName(); |
351 | | - $res = array(); |
352 | | - foreach( $paths as $path ) { |
353 | | - $res[] = "{$backendKey}:{$path}"; |
354 | | - } |
355 | | - return $res; |
356 | | - } |
357 | | - |
358 | | - /** |
359 | 357 | * Split a storage path (e.g. "mwstore://container/path/to/object") |
360 | 358 | * into a container name and a full object name within that container. |
361 | 359 | * |
— | — | @@ -509,10 +507,10 @@ |
510 | 508 | /** |
511 | 509 | * Store a file into the backend from a file on disk. |
512 | 510 | * Parameters must match FileBackend::store(), which include: |
513 | | - * source : source path on disk |
514 | | - * dest : destination storage path |
515 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
516 | | - * overwriteSame : override any existing file at destination |
| 511 | + * source : source path on disk |
| 512 | + * dest : destination storage path |
| 513 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 514 | + * overwriteSame : override any existing file at destination |
517 | 515 | */ |
518 | 516 | class FileStoreOp extends FileOp { |
519 | 517 | /** @var TempLocalFile|null */ |
— | — | @@ -596,10 +594,10 @@ |
597 | 595 | /** |
598 | 596 | * Copy a file from one storage path to another in the backend. |
599 | 597 | * Parameters must match FileBackend::copy(), which include: |
600 | | - * source : source storage path |
601 | | - * dest : destination storage path |
602 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
603 | | - * overwriteSame : override any existing file at destination |
| 598 | + * source : source storage path |
| 599 | + * dest : destination storage path |
| 600 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 601 | + * overwriteSame : override any existing file at destination |
604 | 602 | */ |
605 | 603 | class FileCopyOp extends FileStoreOp { |
606 | 604 | function doAttempt() { |
— | — | @@ -637,10 +635,10 @@ |
638 | 636 | /** |
639 | 637 | * Move a file from one storage path to another in the backend. |
640 | 638 | * Parameters must match FileBackend::move(), which include: |
641 | | - * source : source storage path |
642 | | - * dest : destination storage path |
643 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
644 | | - * overwriteSame : override any existing file at destination |
| 639 | + * source : source storage path |
| 640 | + * dest : destination storage path |
| 641 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 642 | + * overwriteSame : override any existing file at destination |
645 | 643 | */ |
646 | 644 | class FileMoveOp extends FileStoreOp { |
647 | 645 | protected $usingMove = false; // using backend move() function? |
— | — | @@ -710,10 +708,10 @@ |
711 | 709 | /** |
712 | 710 | * Combines files from severals storage paths into a new file in the backend. |
713 | 711 | * Parameters must match FileBackend::concatenate(), which include: |
714 | | - * sources : ordered source storage paths (e.g. chunk1,chunk2,...) |
715 | | - * dest : destination storage path |
716 | | - * overwriteDest : do nothing and pass if an identical file exists at destination |
717 | | - * overwriteSame : override any existing file at destination |
| 712 | + * sources : ordered source storage paths (e.g. chunk1,chunk2,...) |
| 713 | + * dest : destination storage path |
| 714 | + * overwriteDest : do nothing and pass if an identical file exists at destination |
| 715 | + * overwriteSame : override any existing file at destination |
718 | 716 | */ |
719 | 717 | class FileConcatenateOp extends FileStoreOp { |
720 | 718 | function doAttempt() { |
— | — | @@ -751,8 +749,8 @@ |
752 | 750 | /** |
753 | 751 | * Delete a file at the storage path. |
754 | 752 | * Parameters must match FileBackend::delete(), which include: |
755 | | - * source : source storage path |
756 | | - * ignoreMissingSource : don't return an error if the file does not exist |
| 753 | + * source : source storage path |
| 754 | + * ignoreMissingSource : don't return an error if the file does not exist |
757 | 755 | */ |
758 | 756 | class FileDeleteOp extends FileOp { |
759 | 757 | function doAttempt() { |