r103889 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r103888‎ | r103889 | r103890 >
Date:09:21, 22 November 2011
Author:aaron
Status:deferred
Tags:
Comment:
* Updated FileBackendMultiWrite to define all required functions. canMove() was misnamed and getFileList() had to be added. getOperations() moved to single-backend base class.
* Made MultiWrite backend require a lock manager. This simplifies the code, is better for performance, and fixes a logic error that was in the FileBackendMultiWrite::lockFiles() function.
* Various random fixes.
* Comment whitespace cleanups.
Modified paths:
  • /branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php (modified) (history)

Diff [purge]

Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
@@ -7,32 +7,47 @@
88 /**
99 * This class defines a multi-write backend. Multiple backends can
1010 * be registered to this proxy backend it will act as a single backend.
 11+ * Use this only if all access to the backends is through an instance of this class.
1112 *
1213 * The order that the backends are defined sets the priority of which
1314 * backend is read from or written to first. Functions like fileExists()
1415 * and getFileProps() will return information based on the first backend
15 - * that has the file (normally both should have it anyway).
 16+ * that has the file (normally both should have it anyway). Functions like
 17+ * getFileList() will return results from the first backend that is not
 18+ * declared as non-persistent cache. This is done for consistency.
1619 *
1720 * All write operations are performed on all backends.
1821 * If an operation fails on one backend it will be rolled back from the others.
19 - *
20 - * To avoid excess overhead, set the the highest priority backend to use
21 - * a generic FileLockManager and the others to use NullLockManager. This can
22 - * only be done if all access to the backends is through an instance of this class.
2322 */
2423 class FileBackendMultiWrite extends FileBackendBase {
25 - /** @var Array Prioritized list of FileBackend classes */
26 - protected $fileBackends = array();
 24+ /** @var Array Prioritized list of FileBackend objects */
 25+ protected $fileBackends = array(); // array of (backend index => backends)
 26+ /** @var Array List of FileBackend object informations */
 27+ protected $fileBackendsInfo = array(); // array (backend index => array of settings)
2728
 29+ /**
 30+ * Construct a proxy backend that consist of several internal backends.
 31+ * $config contains:
 32+ * 'name' : The name of the proxy backend
 33+ * 'lockManger' : FileLockManager instance
 34+ * 'backends' : Array of ( backend object, settings ) pairs.
 35+ * The settings per backend include:
 36+ * 'isCache': The backend is non-persistent
 37+ * @param $config Array
 38+ */
2839 public function __construct( array $config ) {
2940 $this->name = $config['name'];
30 - $this->fileBackends = $config['backends'];
 41+ $this->lockManager = $config['lockManger'];
 42+ foreach ( $config['backends'] as $index => $info ) {
 43+ list( $backend, $settings ) = $info;
 44+ $this->fileBackends[$index] = $backend;
 45+ // Default backend settings
 46+ $defaults = array( 'isCache' => false );
 47+ // Apply custom backend settings to defaults
 48+ $this->fileBackendsInfo[$index] = $info + $defaults;
 49+ }
3150 }
3251
33 - function hasNativeMove() {
34 - return true; // this is irrelevant
35 - }
36 -
3752 final public function doOperations( array $ops ) {
3853 $status = Status::newGood();
3954
@@ -98,6 +113,10 @@
99114 return $this->doOperation( array( $op ) );
100115 }
101116
 117+ function canMove( array $params ) {
 118+ return true; // this is irrelevant
 119+ }
 120+
102121 function move( array $params ) {
103122 $op = array( 'operation' => 'move' ) + $params;
104123 return $this->doOperation( array( $op ) );
@@ -157,26 +176,13 @@
158177 return null;
159178 }
160179
161 - function lockFiles( array $paths ) {
162 - $status = Status::newGood();
 180+ function getFileList( array $params ) {
163181 foreach ( $this->backends as $index => $backend ) {
164 - $status->merge( $backend->lockFiles( $paths ) );
165 - if ( !$status->isOk() ) {
166 - // Lock failed: release the locks done so far each backend
167 - for ( $i=0; $i < $index; $i++ ) { // don't do backend $index since it failed
168 - $status->merge( $backend->unlockFiles( $paths ) );
169 - }
170 - return $status;
 182+ // Skip cache backends (like one using memcached)
 183+ if ( !$this->fileBackendsInfo[$index]['isCache'] ) {
 184+ return $backend->getFileList( $params );
171185 }
172186 }
173 - return $status;
 187+ return array();
174188 }
175 -
176 - function unlockFiles( array $paths ) {
177 - $status = Status::newGood();
178 - foreach ( $this->backends as $backend ) {
179 - $status->merge( $backend->unlockFile( $paths ) );
180 - }
181 - return $status;
182 - }
183189 }
Index: branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
@@ -75,6 +75,10 @@
7676 return $this->store( $params ); // both source and dest are on FS
7777 }
7878
 79+ function canMove( array $params ) {
 80+ return true;
 81+ }
 82+
7983 function move( array $params ) {
8084 $status = Status::newGood();
8185
@@ -257,11 +261,10 @@
258262 return File::getPropsFromPath( $source );
259263 }
260264
261 - // Not suitable for massive listings
262265 function getFileList( array $params ) {
263266 list( $c, $dir ) = $this->resolveVirtualPath( $params['directory'] );
264 - if ( $dir === null ) { // valid storage path
265 - return new FileIterator( '' ); // empty result
 267+ if ( $dir === null ) { // invalid storage path
 268+ return array(); // empty result
266269 }
267270 return new FileIterator( $dir );
268271 }
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
@@ -17,8 +17,24 @@
1818 */
1919 abstract class FileBackendBase {
2020 protected $name; // unique backend name
 21+ /** @var FileLockManager */
 22+ protected $lockManager;
2123
2224 /**
 25+ * Build a new object from configuration.
 26+ * This should only be called from within FileRepo classes.
 27+ * $config includes:
 28+ * 'name' : The name of this backend
 29+ * 'lockManager' : The file lock manager to use
 30+ *
 31+ * @param $config Array
 32+ */
 33+ public function __construct( array $config ) {
 34+ $this->name = $config['name'];
 35+ $this->lockManager = $config['lockManger'];
 36+ }
 37+
 38+ /**
2339 * We may have multiple different backends of the same type.
2440 * For example, we can have two Swift backends using different proxies.
2541 * All backend instances must have unique names.
@@ -40,11 +56,11 @@
4157 * The inner array contains the parameters, E.G:
4258 * <code>
4359 * $ops = array(
44 - * array(
45 - * 'operation' => 'store',
46 - * 'src' => '/tmp/uploads/picture.png',
47 - * 'dest' => 'mwstore://container/uploadedFilename.png'
48 - * )
 60+ * array(
 61+ * 'operation' => 'store',
 62+ * 'src' => '/tmp/uploads/picture.png',
 63+ * 'dest' => 'mwstore://container/uploadedFilename.png'
 64+ * )
4965 * );
5066 * </code>
5167 *
@@ -54,23 +70,13 @@
5571 abstract public function doOperations( array $ops );
5672
5773 /**
58 - * Return a list of FileOp objects from a list of operations.
59 - * An exception is thrown if an unsupported operation is requested.
60 - *
61 - * @param Array $ops Same format as doOperations()
62 - * @return Array
63 - * @throws MWException
64 - */
65 - abstract public function getOperations( array $ops );
66 -
67 - /**
6874 * Store a file into the backend from a file on disk.
6975 * Do not call this function from places outside FileBackend and FileOp.
7076 * $params include:
71 - * source : source path on disk
72 - * dest : destination storage path
73 - * overwriteDest : do nothing and pass if an identical file exists at destination
74 - * overwriteSame : override any existing file at destination
 77+ * source : source path on disk
 78+ * dest : destination storage path
 79+ * overwriteDest : do nothing and pass if an identical file exists at destination
 80+ * overwriteSame : override any existing file at destination
7581 *
7682 * @param Array $params
7783 * @return Status
@@ -81,12 +87,12 @@
8288 * Copy a file from one storage path to another in the backend.
8389 * Do not call this function from places outside FileBackend and FileOp.
8490 * $params include:
85 - * source : source storage path
86 - * dest : destination storage path
87 - * overwriteDest : do nothing and pass if an identical file exists at destination
88 - * overwriteSame : override any existing file at destination
 91+ * source : source storage path
 92+ * dest : destination storage path
 93+ * overwriteDest : do nothing and pass if an identical file exists at destination
 94+ * overwriteSame : override any existing file at destination
8995 *
90 - * @param Array $params
 96+ * @param Array $params
9197 * @return Status
9298 */
9399 abstract public function copy( array $params );
@@ -96,12 +102,12 @@
97103 * This can be left as a dummy function as long as hasMove() returns false.
98104 * Do not call this function from places outside FileBackend and FileOp.
99105 * $params include:
100 - * source : source storage path
101 - * dest : destination storage path
102 - * overwriteDest : do nothing and pass if an identical file exists at destination
103 - * overwriteSame : override any existing file at destination
 106+ * source : source storage path
 107+ * dest : destination storage path
 108+ * overwriteDest : do nothing and pass if an identical file exists at destination
 109+ * overwriteSame : override any existing file at destination
104110 *
105 - * @param Array $params
 111+ * @param Array $params
106112 * @return Status
107113 */
108114 abstract public function move( array $params );
@@ -110,10 +116,10 @@
111117 * Delete a file at the storage path.
112118 * Do not call this function from places outside FileBackend and FileOp.
113119 * $params include:
114 - * source : source storage path
115 - * ignoreMissingSource : don't return an error if the file does not exist
 120+ * source : source storage path
 121+ * ignoreMissingSource : don't return an error if the file does not exist
116122 *
117 - * @param Array $params
 123+ * @param Array $params
118124 * @return Status
119125 */
120126 abstract public function delete( array $params );
@@ -122,24 +128,25 @@
123129 * Combines files from severals storage paths into a new file in the backend.
124130 * Do not call this function from places outside FileBackend and FileOp.
125131 * $params include:
126 - * source : source storage path
127 - * dest : destination storage path
128 - * overwriteDest : do nothing and pass if an identical file exists at destination
129 - * overwriteSame : override any existing file at destination
 132+ * source : source storage path
 133+ * dest : destination storage path
 134+ * overwriteDest : do nothing and pass if an identical file exists at destination
 135+ * overwriteSame : override any existing file at destination
130136 *
131 - * @param Array $params
 137+ * @param Array $params
132138 * @return Status
133139 */
134140 abstract public function concatenate( array $params );
135141
136142 /**
137 - * Whether this backend implements move() and can handle this potential move.
 143+ * Whether this backend implements move() and can handle a potential move.
138144 * For example, moving objects accross containers may not be supported.
139145 * Do not call this function from places outside FileBackend and FileOp.
140146 * $params include:
141 - * source : source storage path
142 - * dest : destination storage path
 147+ * source : source storage path
 148+ * dest : destination storage path
143149 *
 150+ * @param Array $params
144151 * @return bool
145152 */
146153 abstract public function canMove( array $params );
@@ -148,9 +155,9 @@
149156 * Check if a file exits at a storage path in the backend.
150157 * Do not call this function from places outside FileBackend and FileOp.
151158 * $params include:
152 - * source : source storage path
 159+ * source : source storage path
153160 *
154 - * @param Array $params
 161+ * @param Array $params
155162 * @return bool
156163 */
157164 abstract public function fileExists( array $params );
@@ -158,9 +165,9 @@
159166 /**
160167 * Get the properties of the file that exists at a storage path in the backend
161168 * $params include:
162 - * source : source storage path
 169+ * source : source storage path
163170 *
164 - * @param Array $params
 171+ * @param Array $params
165172 * @return Array|null Gives null if the file does not exist
166173 */
167174 abstract public function getFileProps( array $params );
@@ -171,9 +178,9 @@
172179 * must be sent if streaming began, while none should be sent otherwise.
173180 * Implementations should flush the output buffer before sending data.
174181 * $params include:
175 - * source : source storage path
 182+ * source : source storage path
176183 *
177 - * @param Array $params
 184+ * @param Array $params
178185 * @return Status
179186 */
180187 abstract public function streamFile( array $params );
@@ -185,29 +192,33 @@
186193 * in the container should be listed. If of the form "mwstore://container/dir",
187194 * then all items under that container directory should be listed.
188195 * $params include:
189 - * directory : storage path directory.
190 - * @return Iterator
 196+ * directory : storage path directory.
 197+ *
 198+ * @return Iterator|Array
191199 */
192200 abstract public function getFileList( array $params );
193201
194202 /**
195203 * Get a local copy on disk of the file at a storage path in the backend
196204 * $params include:
197 - * source : source storage path
 205+ * source : source storage path
198206 *
199 - * @param Array $params
 207+ * @param Array $params
200208 * @return TempLocalFile|null Temporary file or null on failure
201209 */
202210 abstract public function getLocalCopy( array $params );
203211
204212 /**
205213 * Lock the files at the given storage paths in the backend.
 214+ * This should either lock all the files or none (on failure).
206215 * Do not call this function from places outside FileBackend and FileOp.
207216 *
208217 * @param $sources Array Source storage paths
209218 * @return Status
210219 */
211 - abstract public function lockFiles( array $sources );
 220+ final public function lockFiles( array $paths ) {
 221+ return $this->lockManager->lock( $this->getLockResourcePaths( $paths ) );
 222+ }
212223
213224 /**
214225 * Unlock the files at the given storage paths in the backend.
@@ -216,30 +227,30 @@
217228 * @param $sources Array Source storage paths
218229 * @return Status
219230 */
220 - abstract public function unlockFiles( array $sources );
 231+ final public function unlockFiles( array $paths ) {
 232+ return $this->lockManager->unlock( $this->getLockResourcePaths( $paths ) );
 233+ }
 234+
 235+ /**
 236+ * Prefix a list of storage paths to use as resource paths to lock
 237+ *
 238+ * @param $paths Array
 239+ * @return Array
 240+ */
 241+ private function getLockResourcePaths( array $paths ) {
 242+ $backendKey = get_class( $this ) . ':' . $this->getName();
 243+ $res = array();
 244+ foreach( $paths as $path ) {
 245+ $res[] = "{$backendKey}:{$path}";
 246+ }
 247+ return $res;
 248+ }
221249 }
222250
223251 /**
224252 * Base class for all single-write backends
225253 */
226254 abstract class FileBackend extends FileBackendBase {
227 - /** @var FileLockManager */
228 - protected $lockManager;
229 -
230 - /**
231 - * Build a new object from configuration.
232 - * This should only be called from within FileRepo classes.
233 - * $config includes:
234 - * 'name' : The name of this backend
235 - * 'lockManager' : The lock manager to use
236 - *
237 - * @param $config Array
238 - */
239 - public function __construct( array $config ) {
240 - $this->name = $config['name'];
241 - $this->lockManager = $config['lockManger'];
242 - }
243 -
244255 function canMove( array $params ) {
245256 return false; // not implemented
246257 }
@@ -263,6 +274,14 @@
264275 );
265276 }
266277
 278+ /**
 279+ * Return a list of FileOp objects from a list of operations.
 280+ * An exception is thrown if an unsupported operation is requested.
 281+ *
 282+ * @param Array $ops Same format as doOperations()
 283+ * @return Array
 284+ * @throws MWException
 285+ */
267286 final public function getOperations( array $ops ) {
268287 $supportedOps = $this->supportedOperations();
269288
@@ -333,28 +352,7 @@
334353 return $status;
335354 }
336355
337 - final public function lockFiles( array $paths ) {
338 - return $this->lockManager->lock( $this->getLockResourcePaths( $paths ) );
339 - }
340 -
341 - final public function unlockFiles( array $paths ) {
342 - return $this->lockManager->unlock( $this->getLockResourcePaths( $paths ) );
343 - }
344 -
345356 /**
346 - * Get a prefix to use for locking keys
347 - * @return string
348 - */
349 - private function getLockResourcePaths( array $paths ) {
350 - $backendKey = get_class( $this ) . ':' . $this->getName();
351 - $res = array();
352 - foreach( $paths as $path ) {
353 - $res[] = "{$backendKey}:{$path}";
354 - }
355 - return $res;
356 - }
357 -
358 - /**
359357 * Split a storage path (e.g. "mwstore://container/path/to/object")
360358 * into a container name and a full object name within that container.
361359 *
@@ -509,10 +507,10 @@
510508 /**
511509 * Store a file into the backend from a file on disk.
512510 * Parameters must match FileBackend::store(), which include:
513 - * source : source path on disk
514 - * dest : destination storage path
515 - * overwriteDest : do nothing and pass if an identical file exists at destination
516 - * overwriteSame : override any existing file at destination
 511+ * source : source path on disk
 512+ * dest : destination storage path
 513+ * overwriteDest : do nothing and pass if an identical file exists at destination
 514+ * overwriteSame : override any existing file at destination
517515 */
518516 class FileStoreOp extends FileOp {
519517 /** @var TempLocalFile|null */
@@ -596,10 +594,10 @@
597595 /**
598596 * Copy a file from one storage path to another in the backend.
599597 * Parameters must match FileBackend::copy(), which include:
600 - * source : source storage path
601 - * dest : destination storage path
602 - * overwriteDest : do nothing and pass if an identical file exists at destination
603 - * overwriteSame : override any existing file at destination
 598+ * source : source storage path
 599+ * dest : destination storage path
 600+ * overwriteDest : do nothing and pass if an identical file exists at destination
 601+ * overwriteSame : override any existing file at destination
604602 */
605603 class FileCopyOp extends FileStoreOp {
606604 function doAttempt() {
@@ -637,10 +635,10 @@
638636 /**
639637 * Move a file from one storage path to another in the backend.
640638 * Parameters must match FileBackend::move(), which include:
641 - * source : source storage path
642 - * dest : destination storage path
643 - * overwriteDest : do nothing and pass if an identical file exists at destination
644 - * overwriteSame : override any existing file at destination
 639+ * source : source storage path
 640+ * dest : destination storage path
 641+ * overwriteDest : do nothing and pass if an identical file exists at destination
 642+ * overwriteSame : override any existing file at destination
645643 */
646644 class FileMoveOp extends FileStoreOp {
647645 protected $usingMove = false; // using backend move() function?
@@ -710,10 +708,10 @@
711709 /**
712710 * Combines files from severals storage paths into a new file in the backend.
713711 * Parameters must match FileBackend::concatenate(), which include:
714 - * sources : ordered source storage paths (e.g. chunk1,chunk2,...)
715 - * dest : destination storage path
716 - * overwriteDest : do nothing and pass if an identical file exists at destination
717 - * overwriteSame : override any existing file at destination
 712+ * sources : ordered source storage paths (e.g. chunk1,chunk2,...)
 713+ * dest : destination storage path
 714+ * overwriteDest : do nothing and pass if an identical file exists at destination
 715+ * overwriteSame : override any existing file at destination
718716 */
719717 class FileConcatenateOp extends FileStoreOp {
720718 function doAttempt() {
@@ -751,8 +749,8 @@
752750 /**
753751 * Delete a file at the storage path.
754752 * Parameters must match FileBackend::delete(), which include:
755 - * source : source storage path
756 - * ignoreMissingSource : don't return an error if the file does not exist
 753+ * source : source storage path
 754+ * ignoreMissingSource : don't return an error if the file does not exist
757755 */
758756 class FileDeleteOp extends FileOp {
759757 function doAttempt() {

Status & tagging log