r104640 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r104639‎ | r104640 | r104641 >
Date:00:57, 30 November 2011
Author:aaron
Status:deferred
Tags:
Comment:
* Added File::getLocalCopyPath()
* Made MediaHandler work with temp FS files as needed
* Fixed resolveStoragePath() to handle containers right
* Various small fixes as needed (FSFile,FileOp...)
* Removed 'overwriteSame' param from concatenate() for performance
* Removed ImageHandler::getTransform() which was a copy paste of the parent
* Redid lockManager config
* Renamed FileIterator => FSFileIterator
* Renamed FileLockManager* => LockManager*
Modified paths:
  • /branches/FileBackend/phase3/includes/AutoLoader.php (modified) (history)
  • /branches/FileBackend/phase3/includes/Setup.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/FileRepo.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileLockManager.php (deleted) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileLockManagerGroup.php (deleted) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/LockManager.php (added) (history)
  • /branches/FileBackend/phase3/includes/filerepo/backend/LockManagerGroup.php (added) (history)
  • /branches/FileBackend/phase3/includes/filerepo/file/FSFile.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/file/File.php (modified) (history)
  • /branches/FileBackend/phase3/includes/filerepo/file/LocalFile.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/Bitmap.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/Bitmap_ClientOnly.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/DjVu.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/Generic.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/MediaTransformOutput.php (modified) (history)
  • /branches/FileBackend/phase3/includes/media/SVG.php (modified) (history)

Diff [purge]

Index: branches/FileBackend/phase3/includes/Setup.php
@@ -115,11 +115,11 @@
116116 $wgNamespaceAliases['Image_talk'] = NS_FILE_TALK;
117117
118118 /**
119 - * Initialise $wgFileLockManagers to include basic FS version
 119+ * Initialise $wgLockManagers to include basic FS version
120120 */
121 -$wgFileLockManagers[] = array(
 121+$wgLockManagers[] = array(
122122 'name' => 'fsLockManager',
123 - 'class' => 'FSFileLockManager',
 123+ 'class' => 'FSLockManager',
124124 'lockDirectory' => $wgUploadDirectory,
125125 );
126126
@@ -234,9 +234,7 @@
235235 return array(
236236 'name' => $backendName,
237237 'class' => 'FSFileBackend',
238 - 'lockManager' => new FSFileLockManager(
239 - array( 'lockDirectory' => "{$directory}/locks" )
240 - ),
 238+ 'lockManager' => 'fsLockManager',
241239 'containerPaths' => array(
242240 "public" => "{$directory}",
243241 "temp" => "{$directory}/temp",
@@ -520,7 +518,7 @@
521519 }
522520
523521 # Register file lock managers
524 -FileLockManagerGroup::singleton()->register( $wgFileLockManagers );
 522+LockManagerGroup::singleton()->register( $wgLockManagers );
525523 # Register file backends
526524 FileBackendGroup::singleton()->register( $wgFileBackends );
527525
Index: branches/FileBackend/phase3/includes/filerepo/file/LocalFile.php
@@ -628,7 +628,6 @@
629629 }
630630
631631 $backend = $this->repo->getBackend();
632 -
633632 $files = array( $dir );
634633 $iterator = $backend->getFileList( array( 'directory' => $dir ) );
635634 foreach ( $iterator as $file ) {
@@ -713,7 +712,6 @@
714713 }
715714 }
716715
717 -
718716 /**
719717 * Delete cached transformed files for the current version only.
720718 */
@@ -722,7 +720,7 @@
723721
724722 // Delete thumbnails
725723 $files = $this->getThumbnails();
726 -
 724+
727725 // Give media handler a chance to filter the purge list
728726 if ( !empty( $options['forRefresh'] ) ) {
729727 $handler = $this->getHandler();
@@ -730,7 +728,7 @@
731729 $handler->filterThumbnailPurgeList( $files, $options );
732730 }
733731 }
734 -
 732+
735733 $dir = array_shift( $files );
736734 $this->purgeThumbList( $dir, $files );
737735
@@ -752,16 +750,19 @@
753751 * @param $dir string base dir of the files.
754752 * @param $files array of strings: relative filenames (to $dir)
755753 */
756 - protected function purgeThumbList($dir, $files) {
 754+ protected function purgeThumbList( $dir, $files ) {
757755 wfDebug( __METHOD__ . ": " . var_export( $files, true ) . "\n" );
 756+ $backend = $this->repo->getBackend();
758757 foreach ( $files as $file ) {
759758 # Check that the base file name is part of the thumb name
760759 # This is a basic sanity check to avoid erasing unrelated directories
761760 if ( strpos( $file, $this->getName() ) !== false ) {
762 - $op = array( 'op' => 'delete', 'source' => "$dir/$file" );
763 - $this->repo->getBackend()->doOperation( $op );
 761+ $op = array( 'op' => 'delete', 'source' => "{$dir}/{$file}" );
 762+ $backend->doOperation( $op );
764763 }
765764 }
 765+ # Clear out directory if empty
 766+ $backend->clean( array( 'directory' => $dir ) );
766767 }
767768
768769 /** purgeDescription inherited */
Index: branches/FileBackend/phase3/includes/filerepo/file/File.php
@@ -73,10 +73,18 @@
7474 var $lastError, $redirected, $redirectedTitle;
7575
7676 /**
 77+ * @var TempFSFile|false
 78+ */
 79+ protected $tmpFile;
 80+
 81+ /**
7782 * @var MediaHandler
7883 */
7984 protected $handler;
8085
 86+ /**
 87+ * @var string
 88+ */
8189 protected $url, $extension, $name, $path, $hashPath, $pageCount, $transformScript;
8290
8391 /**
@@ -322,6 +330,27 @@
323331 }
324332
325333 /**
 334+ * Get a local FS copy of this file and return the path.
 335+ * Returns false on failure.
 336+ *
 337+ * @return string|false
 338+ */
 339+ public function getLocalCopyPath() {
 340+ $this->assertRepoDefined();
 341+ if ( !isset( $this->tmpFile ) ) {
 342+ $this->tmpFile = $this->repo->getLocalCopy( $this->getVirtualUrl() );
 343+ if ( !$this->tmpFile ) {
 344+ $this->tmpFile = false; // null => false; cache negative hits
 345+ }
 346+ }
 347+ if ( $this->tmpFile ) {
 348+ return $this->tmpFile->getPath();
 349+ } else {
 350+ return false;
 351+ }
 352+ }
 353+
 354+ /**
326355 * Return the width of the image. Returns false if the width is unknown
327356 * or undefined.
328357 *
Index: branches/FileBackend/phase3/includes/filerepo/file/FSFile.php
@@ -86,7 +86,7 @@
8787 # logical mime type
8888 $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext );
8989
90 - list( $info['major_mime'], $info['minor_mime'] ) = self::splitMime( $info['mime'] );
 90+ list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] );
9191 $info['media_type'] = $magic->getMediaType( $this->path, $info['mime'] );
9292
9393 # Get size in bytes
@@ -99,7 +99,7 @@
100100 $info['metadata'] = $handler->getMetadata( $tempImage, $this->path );
101101 $gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] );
102102 if ( is_array( $gis ) ) {
103 - $info = $this->extractImageSizeInfo() + $info;
 103+ $info = $this->extractImageSizeInfo( $gis ) + $info;
104104 }
105105 }
106106 $info['sha1'] = $this->sha1Base36();
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileLockManagerGroup.php
@@ -1,68 +0,0 @@
2 -<?php
3 -/**
4 - * Class to handle file lock manager registration
5 - */
6 -class FileLockManagerGroup {
7 - protected static $instance = null;
8 -
9 - /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */
10 - protected $managers = array();
11 -
12 - protected function __construct() {}
13 - protected function __clone() {}
14 -
15 - public static function singleton() {
16 - if ( self::$instance == null ) {
17 - self::$instance = new self();
18 - }
19 - return self::$instance;
20 - }
21 -
22 - /**
23 - * Register an array of file lock manager configurations
24 - *
25 - * @param $configs Array
26 - * @return void
27 - * @throws MWException
28 - */
29 - final public function register( array $configs ) {
30 - foreach ( $configs as $config ) {
31 - if ( !isset( $config['name'] ) ) {
32 - throw new MWException( "Cannot register a lock manager with no name." );
33 - }
34 - $name = $config['name'];
35 - if ( !isset( $config['class'] ) ) {
36 - throw new MWException( "Cannot register lock manager `{$name}` with no class." );
37 - }
38 - $class = $config['class'];
39 -
40 - unset( $config['class'] ); // lock manager won't need this
41 - $this->managers[$name] = array(
42 - 'class' => $class,
43 - 'config' => $config,
44 - 'instance' => null
45 - );
46 - }
47 - }
48 -
49 -
50 - /**
51 - * Get the lock manager object with a given name
52 - *
53 - * @param $name string
54 - * @return FileLockManager
55 - * @throws MWException
56 - */
57 - public function get( $name ) {
58 - if ( !isset( $this->managers[$name] ) ) {
59 - throw new MWException( "No lock manager defined with the name `$name`." );
60 - }
61 - // Lazy-load the actual backend instance
62 - if ( !isset( $this->managers[$name]['instance'] ) ) {
63 - $class = $this->managers[$name]['class'];
64 - $config = $this->managers[$name]['config'];
65 - $this->managers[$name]['instance'] = new $class( $config );
66 - }
67 - return $this->managers[$name]['instance'];
68 - }
69 -}
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileLockManager.php
@@ -1,683 +0,0 @@
2 -<?php
3 -/**
4 - * FileBackend helper class for handling file locking.
5 - * Locks on resource keys can either be shared or exclusive.
6 - *
7 - * Implemenations can keep track of what is locked in the process cache.
8 - * This can reduce hits to external resources for lock()/unlock() calls.
9 - *
10 - * Subclasses should avoid throwing exceptions at all costs.
11 - */
12 -abstract class FileLockManager {
13 - /* Lock types; stronger locks have high values */
14 - const LOCK_SH = 1; // shared lock (for reads)
15 - const LOCK_EX = 2; // exclusive lock (for writes)
16 -
17 - /**
18 - * Construct a new instance from configuration
19 - *
20 - * @param $config Array
21 - */
22 - public function __construct( array $config ) {}
23 -
24 - /**
25 - * Lock the resources at the given abstract paths
26 - *
27 - * @param $paths Array List of resource names
28 - * @param $type integer FileLockManager::LOCK_EX, FileLockManager::LOCK_SH
29 - * @return Status
30 - */
31 - final public function lock( array $paths, $type = self::LOCK_EX ) {
32 - $keys = array_unique( array_map( 'sha1', $paths ) );
33 - return $this->doLock( $keys, $type );
34 - }
35 -
36 - /**
37 - * Unlock the resources at the given abstract paths
38 - *
39 - * @param $paths Array List of storage paths
40 - * @return Status
41 - */
42 - final public function unlock( array $paths ) {
43 - $keys = array_unique( array_map( 'sha1', $paths ) );
44 - return $this->doUnlock( $keys, 0 );
45 - }
46 -
47 - /**
48 - * Lock a resource with the given key
49 - *
50 - * @param $key Array List of keys to lock (40 char hex hashes)
51 - * @param $type integer FileLockManager::LOCK_EX, FileLockManager::LOCK_SH
52 - * @return string
53 - */
54 - abstract protected function doLock( array $keys, $type );
55 -
56 - /**
57 - * Unlock a resource with the given key.
58 - * If $type is given, then only locks of that type should be cleared.
59 - *
60 - * @param $key Array List of keys to unlock (40 char hex hashes)
61 - * @param $type integer FileLockManager::LOCK_EX, FileLockManager::LOCK_SH, or 0
62 - * @return string
63 - */
64 - abstract protected function doUnlock( array $keys, $type );
65 -}
66 -
67 -/**
68 - * Simple version of FileLockManager based on using FS lock files
69 - *
70 - * This should work fine for small sites running off one server.
71 - * Do not use this with 'lockDir' set to an NFS mount unless the
72 - * NFS client is at least version 2.6.12. Otherwise, the BSD flock()
73 - * locks will be ignored; see http://nfs.sourceforge.net/#section_d.
74 - */
75 -class FSFileLockManager extends FileLockManager {
76 - protected $lockDir; // global dir for all servers
77 -
78 - /** @var Array Map of (locked key => lock type => count) */
79 - protected $locksHeld = array();
80 - /** @var Array Map of (locked key => lock type => lock file handle) */
81 - protected $handles = array();
82 -
83 - function __construct( array $config ) {
84 - $this->lockDir = $config['lockDirectory'];
85 - }
86 -
87 - protected function doLock( array $keys, $type ) {
88 - $status = Status::newGood();
89 -
90 - $lockedKeys = array(); // files locked in this attempt
91 - foreach ( $keys as $key ) {
92 - $subStatus = $this->doSingleLock( $key, $type );
93 - $status->merge( $subStatus );
94 - if ( $status->isOK() ) {
95 - // Don't append to $lockedKeys if $key is already locked.
96 - // We do NOT want to unlock the key if we have to rollback.
97 - if ( $subStatus->isGood() ) { // no warnings/fatals?
98 - $lockedKeys[] = $key;
99 - }
100 - } else {
101 - // Abort and unlock everything
102 - $status->merge( $this->doUnlock( $lockedKeys, $type ) );
103 - return $status;
104 - }
105 - }
106 -
107 - return $status;
108 - }
109 -
110 - protected function doUnlock( array $keys, $type ) {
111 - $status = Status::newGood();
112 -
113 - foreach ( $keys as $key ) {
114 - $status->merge( $this->doSingleUnlock( $key, $type ) );
115 - }
116 -
117 - return $status;
118 - }
119 -
120 - /**
121 - * Lock a single resource key
122 - *
123 - * @param $key string
124 - * @param $type integer
125 - * @return Status
126 - */
127 - protected function doSingleLock( $key, $type ) {
128 - $status = Status::newGood();
129 -
130 - if ( isset( $this->locksHeld[$key][$type] ) ) {
131 - ++$this->locksHeld[$key][$type];
132 - } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
133 - $this->locksHeld[$key][$type] = 1;
134 - } else {
135 - wfSuppressWarnings();
136 - $handle = fopen( $this->getLockPath( $key ), 'c' );
137 - wfRestoreWarnings();
138 - if ( !$handle ) { // lock dir missing?
139 - wfMkdirParents( $this->lockDir );
140 - wfSuppressWarnings();
141 - $handle = fopen( $this->getLockPath( $key ), 'c' ); // try again
142 - wfRestoreWarnings();
143 - }
144 - if ( $handle ) {
145 - // Either a shared or exclusive lock
146 - $lock = ( $type == self::LOCK_SH ) ? LOCK_SH : LOCK_EX;
147 - if ( flock( $handle, $lock | LOCK_NB ) ) {
148 - // Record this lock as active
149 - $this->locksHeld[$key][$type] = 1;
150 - $this->handles[$key][$type] = $handle;
151 - } else {
152 - fclose( $handle );
153 - $status->fatal( 'lockmanager-fail-acquirelock', $key );
154 - }
155 - } else {
156 - $status->fatal( 'lockmanager-fail-openlock', $key );
157 - }
158 - }
159 -
160 - return $status;
161 - }
162 -
163 - /**
164 - * Unlock a single resource key
165 - *
166 - * @param $key string
167 - * @param $type integer
168 - * @return Status
169 - */
170 - protected function doSingleUnlock( $key, $type ) {
171 - $status = Status::newGood();
172 -
173 - if ( !isset( $this->locksHeld[$key] ) ) {
174 - $status->warning( 'lockmanager-notlocked', $key );
175 - } elseif ( $type && !isset( $this->locksHeld[$key][$type] ) ) {
176 - $status->warning( 'lockmanager-notlocked', $key );
177 - } else {
178 - foreach ( $this->locksHeld[$key] as $lockType => $count ) {
179 - if ( $type && $lockType != $type ) {
180 - continue; // only unlock locks of type $type
181 - }
182 - --$this->locksHeld[$key][$lockType];
183 - if ( $this->locksHeld[$key][$lockType] <= 0 ) {
184 - unset( $this->locksHeld[$key][$lockType] );
185 - // If a LOCK_SH comes in while we have a LOCK_EX, we don't
186 - // actually add a handler, so check for handler existence.
187 - if ( isset( $this->handles[$key][$lockType] ) ) {
188 - wfSuppressWarnings();
189 - if ( !flock( $this->handles[$key][$lockType], LOCK_UN ) ) {
190 - $status->fatal( 'lockmanager-fail-releaselock', $key );
191 - }
192 - if ( !fclose( $this->handles[$key][$lockType] ) ) {
193 - $status->warning( 'lockmanager-fail-closelock', $key );
194 - }
195 - wfRestoreWarnings();
196 - unset( $this->handles[$key][$lockType] );
197 - }
198 - }
199 - }
200 - if ( !count( $this->locksHeld[$key] ) ) {
201 - wfSuppressWarnings();
202 - # No locks are held for the lock file anymore
203 - if ( !unlink( $this->getLockPath( $key ) ) ) {
204 - $status->warning( 'lockmanager-fail-deletelock', $key );
205 - }
206 - wfRestoreWarnings();
207 - unset( $this->locksHeld[$key] );
208 - unset( $this->handles[$key] );
209 - }
210 - }
211 -
212 - return $status;
213 - }
214 -
215 - /**
216 - * Get the path to the lock file for a key
217 - * @param $key string
218 - * @return string
219 - */
220 - protected function getLockPath( $key ) {
221 - return "{$this->lockDir}/{$key}.lock";
222 - }
223 -
224 - function __destruct() {
225 - // Make sure remaining files get cleared for sanity
226 - foreach ( $this->handles as $key => $locks ) {
227 - foreach ( $locks as $type => $handle ) {
228 - flock( $handle, LOCK_UN ); // PHP 5.3 will not do this automatically
229 - fclose( $handle );
230 - }
231 - unlink( $this->getLockPath( $key ) );
232 - }
233 - }
234 -}
235 -
236 -/**
237 - * Version of FileLockManager based on using DB table locks.
238 - * This is meant for multi-wiki systems that may share share files.
239 - *
240 - * All lock requests for a resource, identified by a hash string, will
241 - * map to one bucket. Each bucket maps to one or several peer DB servers,
242 - * each having a `file_locks` table with row-level locking.
243 - *
244 - * A majority of peer servers must agree for a lock to be acquired.
245 - * As long as one peer server is up, lock requests will not be blocked
246 - * just because another peer server cannot be contacted. A global status
247 - * cache can be setup to track servers that recently missed queries; such
248 - * servers will not be trusted for obtaining locks.
249 - *
250 - * For performance, deadlock detection should be disabled and a small
251 - * lock-wait timeout should be set via server config. In innoDB, this can
252 - * done via the innodb_deadlock_detect and innodb_lock_wait_timeout settings.
253 - */
254 -class DBFileLockManager extends FileLockManager {
255 - /** @var Array Map of bucket indexes to peer sets */
256 - protected $dbsByBucket; // (bucket index => (ldb1, ldb2, ...))
257 - /** @var BagOStuff */
258 - protected $statusCache;
259 -
260 - protected $trustCache; // boolean
261 - protected $webTimeout; // integer number of seconds
262 - protected $cliTimeout; // integer number of seconds
263 - protected $safeDelay; // integer number of seconds
264 -
265 - /** @var Array Map of (locked key => lock type => count) */
266 - protected $locksHeld = array();
267 - /** $var Array Map Lock-active database connections (server name => Database) */
268 - protected $activeConns = array();
269 -
270 - /**
271 - * Construct a new instance from configuration.
272 - * $config paramaters include:
273 - * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, with
274 - * a list of DB names (peers) as values. Each list should have
275 - * an odd number of items and each DB should have its own server.
276 - * 'webTimeout' : Lock timeout (seconds) for non-CLI scripts. [optional]
277 - * This tells the DB server how long to wait before assuming
278 - * connection failure and releasing all the locks for a session.
279 - * 'cliTimeout' : Lock timeout (seconds) for CLI scripts. [optional]
280 - * This tells the DB server how long to wait before assuming
281 - * connection failure and releasing all the locks for a session.
282 - * 'safeDelay' : Seconds to mistrust a DB after restart/query loss. [optional]
283 - * This should reflect the highest max_execution_time that PHP
284 - * scripts might use on a wiki. Locks are lost on server restart.
285 - * 'cache' : $wgMemc (if set to a global memcached instance). [optional]
286 - * This tracks peer servers that couldn't be queried recently.
287 - * 'trustCache' : Assume cache knows all servers missing queries recently. [optional]
288 - *
289 - * @param Array $config
290 - */
291 - function __construct( array $config ) {
292 - // Sanitize dbsByBucket config to prevent PHP errors
293 - $this->dbsByBucket = array_filter( $config['dbsByBucket'], 'is_array' );
294 - $this->dbsByBucket = array_values( $this->dbsByBucket ); // consecutive
295 -
296 - if ( isset( $config['webTimeout'] ) ) {
297 - $this->webTimeout = $config['webTimeout'];
298 - } else {
299 - $met = ini_get( 'max_execution_time' );
300 - $this->webTimeout = $met ? $met : 60; // use some same amount if 0
301 - }
302 - $this->cliTimeout = isset( $config['cliTimeout'] )
303 - ? $config['cliTimeout']
304 - : 60; // some sane amount
305 - $this->safeDelay = isset( $config['safeDelay'] )
306 - ? $config['safeDelay']
307 - : max( $this->cliTimeout, $this->webTimeout ); // cover worst case
308 -
309 - if ( isset( $config['cache'] ) && $config['cache'] instanceof BagOStuff ) {
310 - $this->statusCache = $config['cache'];
311 - $this->trustCache = ( !empty( $config['trustCache'] ) && $this->safeDelay > 0 );
312 - } else {
313 - $this->statusCache = null;
314 - $this->trustCache = false;
315 - }
316 - }
317 -
318 - protected function doLock( array $keys, $type ) {
319 - $status = Status::newGood();
320 -
321 - $keysToLock = array();
322 - // Get locks that need to be acquired (buckets => locks)...
323 - foreach ( $keys as $key ) {
324 - if ( isset( $this->locksHeld[$key][$type] ) ) {
325 - ++$this->locksHeld[$key][$type];
326 - } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
327 - $this->locksHeld[$key][$type] = 1;
328 - } else {
329 - $bucket = $this->getBucketFromKey( $key );
330 - $keysToLock[$bucket][] = $key;
331 - }
332 - }
333 -
334 - $lockedKeys = array(); // files locked in this attempt
335 - // Attempt to acquire these locks...
336 - foreach ( $keysToLock as $bucket => $keys ) {
337 - // Acquire the locks for this server. Three main cases can happen:
338 - // (a) First server is up; common case
339 - // (b) First server is down but a peer is up
340 - // (c) First server is down and no peer are up (or none defined)
341 - $res = $this->doLockingQueryAll( $bucket, $keys, $type );
342 - if ( $res === 'cantacquire' ) {
343 - // Resources already locked by another process.
344 - // Abort and unlock everything we just locked.
345 - $status->fatal( 'lockmanager-fail-acquirelocks' );
346 - $status->merge( $this->doUnlock( $lockedKeys, $type ) );
347 - return $status;
348 - } elseif ( $res !== true ) {
349 - // Couldn't contact any servers for this bucket.
350 - // Abort and unlock everything we just locked.
351 - $status->fatal( 'lockmanager-fail-db-bucket', $bucket );
352 - $status->merge( $this->doUnlock( $lockedKeys, $type ) );
353 - return $status;
354 - }
355 - // Record these locks as active
356 - foreach ( $keys as $key ) {
357 - $this->locksHeld[$key][$type] = 1; // locked
358 - }
359 - // Keep track of what locks were made in this attempt
360 - $lockedKeys = array_merge( $lockedKeys, $keys );
361 - }
362 -
363 - return $status;
364 - }
365 -
366 - protected function doUnlock( array $keys, $type ) {
367 - $status = Status::newGood();
368 -
369 - foreach ( $keys as $key ) {
370 - if ( !isset( $this->locksHeld[$key] ) ) {
371 - $status->warning( 'lockmanager-notlocked', $key );
372 - } elseif ( $type && !isset( $this->locksHeld[$key][$type] ) ) {
373 - $status->warning( 'lockmanager-notlocked', $key );
374 - } else {
375 - foreach ( $this->locksHeld[$key] as $lockType => $count ) {
376 - if ( $type && $lockType != $type ) {
377 - continue; // only unlock locks of type $type
378 - }
379 - --$this->locksHeld[$key][$lockType];
380 - if ( $this->locksHeld[$key][$lockType] <= 0 ) {
381 - unset( $this->locksHeld[$key][$lockType] );
382 - }
383 - }
384 - if ( !count( $this->locksHeld[$key] ) ) {
385 - unset( $this->locksHeld[$key] ); // no SH or EX locks left for key
386 - }
387 - }
388 - }
389 -
390 - // Reference count the locks held and COMMIT when zero
391 - if ( !count( $this->locksHeld ) ) {
392 - $status->merge( $this->finishLockTransactions() );
393 - }
394 -
395 - return $status;
396 - }
397 -
398 - /**
399 - * Get a DB connection to a lock server and acquire locks on $keys.
400 - *
401 - * @param $server string
402 - * @param $keys Array
403 - * @param $type integer FileLockManager::LOCK_EX or FileLockManager::LOCK_SH
404 - * @return void
405 - */
406 - protected function doLockingQuery( $server, array $keys, $type ) {
407 - if ( !isset( $this->activeConns[$server] ) ) {
408 - $this->activeConns[$server] = wfGetDB( DB_MASTER, array(), $server );
409 - $this->activeConns[$server]->begin(); // start transaction
410 - # If the connection drops, try to avoid letting the DB rollback
411 - # and release the locks before the file operations are finished.
412 - # This won't handle the case of server reboots however.
413 - $options = array();
414 - if ( php_sapi_name() == 'cli' ) { // maintenance scripts
415 - if ( $this->cliTimeout > 0 ) {
416 - $options['connTimeout'] = $this->cliTimeout;
417 - }
418 - } else { // web requests
419 - if ( $this->webTimeout > 0 ) {
420 - $options['connTimeout'] = $this->webTimeout;
421 - }
422 - }
423 - $this->activeConns[$server]->setSessionOptions( $options );
424 - }
425 - $db = $this->activeConns[$server];
426 - # Try to get the locks...this should be the last query of this function
427 - if ( $type == self::LOCK_SH ) { // reader locks
428 - $db->select( 'file_locks', '1',
429 - array( 'fl_key' => $keys ),
430 - __METHOD__,
431 - array( 'LOCK IN SHARE MODE' ) // single-row gap locks
432 - );
433 - } else { // writer locks
434 - $data = array();
435 - foreach ( $keys as $key ) {
436 - $data[] = array( 'fl_key' => $key );
437 - }
438 - $db->insert( 'file_locks', $data, __METHOD__ );
439 - }
440 - }
441 -
442 - /**
443 - * Attempt to acquire locks with the peers for a bucket.
444 - * This should avoid throwing any exceptions.
445 - *
446 - * @param $bucket integer
447 - * @param $keys Array List of resource keys to lock
448 - * @param $type integer FileLockManager::LOCK_EX or FileLockManager::LOCK_SH
449 - * @return bool|string One of (true, 'cantacquire', 'dberrors')
450 - */
451 - protected function doLockingQueryAll( $bucket, array $keys, $type ) {
452 - $yesVotes = 0; // locks made on trustable servers
453 - $votesLeft = count( $this->dbsByBucket[$bucket] ); // remaining servers
454 - $quorum = floor( $votesLeft/2 + 1 ); // simple majority
455 - // Get votes for each server, in order, until we have enough...
456 - foreach ( $this->dbsByBucket[$bucket] as $index => $server ) {
457 - if ( $this->trustCache && $votesLeft < $quorum ) {
458 - // We are now hitting servers that are not normally
459 - // hit, meaning that some of the first ones are down.
460 - // Delay using these later servers until it's safe.
461 - if ( !$this->cacheCheckSkipped( $bucket, $index ) ) {
462 - return 'dberrors';
463 - }
464 - }
465 - // Check that server is not *known* to be down
466 - if ( $this->cacheCheckFailures( $server ) ) {
467 - try {
468 - // Attempt to acquire the lock on this server
469 - $this->doLockingQuery( $server, $keys, $type );
470 - // Check that server has no signs of lock loss
471 - if ( $this->checkUptime( $server ) ) {
472 - ++$yesVotes; // success for this peer
473 - if ( $yesVotes >= $quorum ) {
474 - if ( $this->trustCache ) {
475 - // We didn't bother with the servers after this one
476 - $this->cacheRecordSkipped( $bucket, $index + 1 );
477 - }
478 - return true; // lock obtained
479 - }
480 - }
481 - } catch ( DBError $e ) {
482 - if ( $this->lastErrorIndicatesLocked( $server ) ) {
483 - return 'cantacquire'; // vetoed; resource locked
484 - } else { // can't connect?
485 - $this->cacheRecordFailure( $server );
486 - }
487 - }
488 - }
489 - $votesLeft--;
490 - $votesNeeded = $quorum - $yesVotes;
491 - if ( $votesNeeded > $votesLeft && !$this->trustCache ) {
492 - // In "trust cache" mode we don't have to meet the quorum
493 - break; // short-circuit
494 - }
495 - }
496 - // At this point, we must not have meet the quorum
497 - if ( $yesVotes > 0 && $this->trustCache ) {
498 - return true; // we are trusting the cache; may comprimise correctness
499 - }
500 - return 'dberrors'; // not enough votes to ensure correctness
501 - }
502 -
503 - /**
504 - * Commit all changes to lock-active databases.
505 - * This should avoid throwing any exceptions.
506 - *
507 - * @return Status
508 - */
509 - protected function finishLockTransactions() {
510 - $status = Status::newGood();
511 - foreach ( $this->activeConns as $server => $db ) {
512 - try {
513 - $db->rollback(); // finish transaction and kill any rows
514 - } catch ( DBError $e ) {
515 - $status->fatal( 'lockmanager-fail-db-release', $server );
516 - // oh well; best effort
517 - }
518 - }
519 - $this->activeConns = array();
520 - return $status;
521 - }
522 -
523 - /**
524 - * Check if the last DB error for $server indicates
525 - * that a requested resource was locked by another process.
526 - * This should avoid throwing any exceptions.
527 - *
528 - * @param $server string
529 - * @return bool
530 - */
531 - protected function lastErrorIndicatesLocked( $server ) {
532 - if ( isset( $this->activeConns[$server] ) ) { // sanity
533 - $db = $this->activeConns[$server];
534 - return ( $db->wasDeadlock() || $db->wasLockTimeout() );
535 - }
536 - return false;
537 - }
538 -
539 - /**
540 - * Checks if the DB server did not recently restart.
541 - * This curtails the problem of locks falling off when servers restart.
542 - *
543 - * @param $server string
544 - * @return bool
545 - */
546 - protected function checkUptime( $server ) {
547 - if ( isset( $this->activeConns[$server] ) ) { // sanity
548 - if ( $this->safeDelay > 0 ) {
549 - $db = $this->activeConns[$server];
550 - if ( $db->getServerUptime() < $this->safeDelay ) {
551 - return false;
552 - }
553 - }
554 - return true;
555 - }
556 - return false;
557 - }
558 -
559 - /**
560 - * Check that DB servers starting at $index for a
561 - * bucket were not recently skipped in obtaining a lock.
562 - *
563 - * @param $bucket
564 - * @param $index
565 - * @return bool
566 - */
567 - protected function cacheCheckSkipped( $bucket, $index ) {
568 - if ( $this->statusCache && $this->safeDelay > 0 ) {
569 - $key = $this->getSkipsKey( $bucket, $index );
570 - $skips = $this->statusCache->get( $key );
571 - return !$skips;
572 - }
573 - return true;
574 - }
575 -
576 - /**
577 - * Record that DB servers starting at $index for a
578 - * bucket were just skipped in obtaining a lock (quorum met).
579 - *
580 - * @param $bucket
581 - * @param $index
582 - * @return bool Success
583 - */
584 - protected function cacheRecordSkipped( $bucket, $index ) {
585 - if ( $this->statusCache && $this->safeDelay > 0 ) {
586 - $key = $this->getSkipsKey( $bucket, $index );
587 - $skips = $this->statusCache->get( $key );
588 - if ( $skips ) {
589 - return $this->statusCache->incr( $key );
590 - } else {
591 - return $this->statusCache->add( $key, 1, $this->safeDelay );
592 - }
593 - }
594 - return true;
595 - }
596 -
597 - /**
598 - * Get a cache key for recent query skips for a bucket
599 - *
600 - * @param $bucket
601 - * @param $index
602 - * @return string
603 - */
604 - protected function getSkipsKey( $bucket, $index ) {
605 - return "lockmanager:queryskips:$bucket:$index";
606 - }
607 -
608 - /**
609 - * Checks if the DB server has not recently had connection/query errors.
610 - * When in "trust cache" mode, this curtails the problem of peers occasionally
611 - * missing locks. Otherwise, it just avoids wasting time on connection attempts.
612 - *
613 - * @param $server string
614 - * @return bool
615 - */
616 - protected function cacheCheckFailures( $server ) {
617 - if ( $this->statusCache && $this->safeDelay > 0 ) {
618 - $key = $this->getMissKey( $server );
619 - $misses = $this->statusCache->get( $key );
620 - return !$misses;
621 - }
622 - return true;
623 - }
624 -
625 - /**
626 - * Log a lock request failure to the cache.
627 - *
628 - * Worst case scenario is that a resource lock was only
629 - * on one peer and then that peer is restarted or goes down.
630 - * Clients trying to get locks need to know if a server is down.
631 - *
632 - * @param $server string
633 - * @return bool Success
634 - */
635 - protected function cacheRecordFailure( $server ) {
636 - if ( $this->statusCache && $this->safeDelay > 0 ) {
637 - $key = $this->getMissKey( $server );
638 - $misses = $this->statusCache->get( $key );
639 - if ( $misses ) {
640 - return $this->statusCache->incr( $key );
641 - } else {
642 - return $this->statusCache->add( $key, 1, $this->safeDelay );
643 - }
644 - }
645 - return true;
646 - }
647 -
648 - /**
649 - * Get a cache key for recent query misses for a DB server
650 - *
651 - * @param $server string
652 - * @return string
653 - */
654 - protected function getMissKey( $server ) {
655 - return "lockmanager:querymisses:$server";
656 - }
657 -
658 - /**
659 - * Get the bucket for lock key.
660 - * This should avoid throwing any exceptions.
661 - *
662 - * @param $key string (40 char hex key)
663 - * @return integer
664 - */
665 - protected function getBucketFromKey( $key ) {
666 - $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits)
667 - return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->dbsByBucket );
668 - }
669 -}
670 -
671 -/**
672 - * Simple version of FileLockManager that does nothing
673 - */
674 -class NullFileLockManager extends FileLockManager {
675 - function __construct( array $config ) {}
676 -
677 - protected function doLock( array $keys, $type ) {
678 - return Status::newGood();
679 - }
680 -
681 - protected function doUnlock( array $keys, $type ) {
682 - return Status::newGood();
683 - }
684 -}
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackendMultiWrite.php
@@ -40,7 +40,7 @@
4141 * Construct a proxy backend that consist of several internal backends.
4242 * $config contains:
4343 * 'name' : The name of the proxy backend
44 - * 'lockManger' : FileLockManager instance
 44+ * 'lockManger' : LockManager instance
4545 * 'backends' : Array of (backend object, settings) pairs.
4646 * The settings per backend include:
4747 * 'isCache' : The backend is non-persistent
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileOp.php
@@ -187,7 +187,7 @@
188188 return $status; // do nothing; either OK or bad status
189189 }
190190 } else {
191 - $status->fatal( 'backend-fail-alreadyexists', $params['dest'] );
 191+ $status->fatal( 'backend-fail-alreadyexists', $this->params['dest'] );
192192 return $status;
193193 }
194194
@@ -566,7 +566,7 @@
567567
568568 protected function doFinish() {
569569 // Delete the source file
570 - $status = $this->fileBackend->delete( $this->params );
 570+ $status = $this->backend->delete( $this->params );
571571 return $status;
572572 }
573573
Index: branches/FileBackend/phase3/includes/filerepo/backend/FSFileBackend.php
@@ -42,14 +42,13 @@
4343 $status->fatal( 'backend-fail-invalidpath', $params['dest'] );
4444 return $status;
4545 }
46 -
47 - if ( file_exists( $dest ) ) {
 46+ if ( is_file( $dest ) ) {
4847 if ( isset( $params['overwriteDest'] ) ) {
4948 wfSuppressWarnings();
5049 $ok = unlink( $dest );
5150 wfRestoreWarnings();
5251 if ( !$ok ) {
53 - $status->fatal( 'backend-fail-delete', $param['dest'] );
 52+ $status->fatal( 'backend-fail-delete', $params['dest'] );
5453 return $status;
5554 }
5655 } else {
@@ -57,7 +56,7 @@
5857 return $status;
5958 }
6059 } else {
61 - if ( !wfMkdirParents( $dest ) ) {
 60+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
6261 $status->fatal( 'directorycreateerror', $param['dest'] );
6362 return $status;
6463 }
@@ -115,7 +114,7 @@
116115 return $status;
117116 }
118117 } else {
119 - if ( !wfMkdirParents( $dest ) ) {
 118+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
120119 $status->fatal( 'directorycreateerror', $param['dest'] );
121120 return $status;
122121 }
@@ -142,7 +141,7 @@
143142 }
144143
145144 if ( !file_exists( $source ) ) {
146 - if ( !$params['ignoreMissingSource'] ) {
 145+ if ( empty( $params['ignoreMissingSource'] ) ) {
147146 $status->fatal( 'backend-fail-delete', $params['source'] );
148147 }
149148 return $status; // do nothing; either OK or bad status
@@ -170,14 +169,14 @@
171170
172171 // Check if the destination file exists and we can't handle that
173172 $destExists = file_exists( $dest );
174 - if ( $destExists && !$params['overwriteDest'] && !$params['overwriteSame'] ) {
 173+ if ( $destExists && empty( $params['overwriteDest'] ) ) {
175174 $status->fatal( 'backend-fail-alreadyexists', $params['dest'] );
176175 return $status;
177176 }
178177
179178 // Create a new temporary file...
180179 wfSuppressWarnings();
181 - $tmpPath = tempnam( wfTempDir(), 'file_concatenate' );
 180+ $tmpPath = tempnam( wfTempDir(), 'concatenate' );
182181 wfRestoreWarnings();
183182 if ( $tmpPath === false ) {
184183 $status->fatal( 'backend-fail-createtemp' );
@@ -220,26 +219,19 @@
221220 // Handle overwrite behavior of file destination if applicable.
222221 // Note that we already checked if no overwrite params were set above.
223222 if ( $destExists ) {
224 - if ( isset( $params['overwriteDest'] ) ) {
225 - // Windows does not support moving over existing files
226 - if ( wfIsWindows() ) {
227 - wfSuppressWarnings();
228 - $ok = unlink( $dest );
229 - wfRestoreWarnings();
230 - if ( !$ok ) {
231 - $status->fatal( 'backend-fail-delete', $params['dest'] );
232 - return $status;
233 - }
 223+ // Windows does not support moving over existing files
 224+ if ( wfIsWindows() ) {
 225+ wfSuppressWarnings();
 226+ $ok = unlink( $dest );
 227+ wfRestoreWarnings();
 228+ if ( !$ok ) {
 229+ $status->fatal( 'backend-fail-delete', $params['dest'] );
 230+ return $status;
234231 }
235 - } elseif ( isset( $params['overwriteSame'] ) ) {
236 - if ( !$this->filesAreSame( $tmpPath, $dest ) ) {
237 - $status->fatal( 'backend-fail-notsame', $params['dest'] );
238 - }
239 - return $status; // do nothing; either OK or bad status
240232 }
241233 } else {
242234 // Make sure destination directory exists
243 - if ( !wfMkdirParents( $dest ) ) {
 235+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
244236 $status->fatal( 'directorycreateerror', $param['dest'] );
245237 return $status;
246238 }
@@ -282,7 +274,7 @@
283275 return $status;
284276 }
285277 } else {
286 - if ( !wfMkdirParents( $dest ) ) {
 278+ if ( !wfMkdirParents( dirname( $dest ) ) ) {
287279 $status->fatal( 'directorycreateerror', $param['dest'] );
288280 return $status;
289281 }
@@ -364,7 +356,7 @@
365357 }
366358 wfSuppressWarnings();
367359 if ( is_dir( $dir ) ) {
368 - rmdir( $dir ); // Might have already gone away, spews errors if we don't.
 360+ rmdir( $dir ); // remove directory if empty
369361 }
370362 wfRestoreWarnings();
371363 return $status;
@@ -413,7 +405,7 @@
414406 if ( $dir === null ) { // invalid storage path
415407 return array(); // empty result
416408 }
417 - return new FileIterator( $dir );
 409+ return new FSFileIterator( $dir );
418410 }
419411
420412 function streamFile( array $params ) {
@@ -445,7 +437,7 @@
446438 $ext = strtolower( $i ? substr( $source, $i + 1 ) : '' );
447439 // Create a new temporary file...
448440 wfSuppressWarnings();
449 - $initialTmpPath = tempnam( wfTempDir(), 'file_localcopy' );
 441+ $initialTmpPath = tempnam( wfTempDir(), 'localcopy' );
450442 wfRestoreWarnings();
451443 if ( $initialTmpPath === false ) {
452444 return null;
@@ -470,23 +462,6 @@
471463 }
472464
473465 /**
474 - * Check if two files are identical
475 - *
476 - * @param $path1 string Absolute filesystem path
477 - * @param $path2 string Absolute filesystem path
478 - * @return bool
479 - */
480 - protected function filesAreSame( $path1, $path2 ) {
481 - wfSuppressWarnings();
482 - $same = ( // check size first since it's faster
483 - filesize( $path1 ) === filesize( $path2 ) &&
484 - sha1_file( $path1 ) === sha1_file( $path2 )
485 - );
486 - wfRestoreWarnings();
487 - return $same;
488 - }
489 -
490 - /**
491466 * Chmod a file, suppressing the warnings
492467 *
493468 * @param $path string Absolute file system path
@@ -505,8 +480,9 @@
506481 * Semi-DFS based file browsing iterator. The highest number of file handles
507482 * open at any given time is proportional to the height of the directory tree.
508483 */
509 -class FileIterator implements Iterator {
 484+class FSFileIterator implements Iterator {
510485 private $directory; // starting directory
 486+ private $itemStart = 0;
511487
512488 private $position = 0;
513489 private $currentFile = false;
@@ -524,6 +500,8 @@
525501 $this->directory = substr( $this->directory, 0, -1 );
526502 }
527503 $this->directory = realpath( $directory );
 504+ // Remove internal base directory and trailing slash from results
 505+ $this->itemStart = strlen( $this->directory ) + 1;
528506 }
529507
530508 private function load() {
@@ -549,7 +527,8 @@
550528
551529 function current() {
552530 $this->load();
553 - return $this->currentFile;
 531+ // Remove internal base directory and trailing slash from results
 532+ return substr( $this->currentFile, $this->itemStart );
554533 }
555534
556535 function key() {
Index: branches/FileBackend/phase3/includes/filerepo/backend/LockManagerGroup.php
@@ -0,0 +1,66 @@
 2+<?php
 3+/**
 4+ * Class to handle file lock manager registration
 5+ */
 6+class LockManagerGroup {
 7+ protected static $instance = null;
 8+
 9+ /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */
 10+ protected $managers = array();
 11+
 12+ protected function __construct() {}
 13+ protected function __clone() {}
 14+
 15+ public static function singleton() {
 16+ if ( self::$instance == null ) {
 17+ self::$instance = new self();
 18+ }
 19+ return self::$instance;
 20+ }
 21+
 22+ /**
 23+ * Register an array of file lock manager configurations
 24+ *
 25+ * @param $configs Array
 26+ * @return void
 27+ * @throws MWException
 28+ */
 29+ public function register( array $configs ) {
 30+ foreach ( $configs as $config ) {
 31+ if ( !isset( $config['name'] ) ) {
 32+ throw new MWException( "Cannot register a lock manager with no name." );
 33+ }
 34+ $name = $config['name'];
 35+ if ( !isset( $config['class'] ) ) {
 36+ throw new MWException( "Cannot register lock manager `{$name}` with no class." );
 37+ }
 38+ $class = $config['class'];
 39+ unset( $config['class'] ); // lock manager won't need this
 40+ $this->managers[$name] = array(
 41+ 'class' => $class,
 42+ 'config' => $config,
 43+ 'instance' => null
 44+ );
 45+ }
 46+ }
 47+
 48+ /**
 49+ * Get the lock manager object with a given name
 50+ *
 51+ * @param $name string
 52+ * @return LockManager
 53+ * @throws MWException
 54+ */
 55+ public function get( $name ) {
 56+ if ( !isset( $this->managers[$name] ) ) {
 57+ throw new MWException( "No lock manager defined with the name `$name`." );
 58+ }
 59+ // Lazy-load the actual backend instance
 60+ if ( !isset( $this->managers[$name]['instance'] ) ) {
 61+ $class = $this->managers[$name]['class'];
 62+ $config = $this->managers[$name]['config'];
 63+ $this->managers[$name]['instance'] = new $class( $config );
 64+ }
 65+ return $this->managers[$name]['instance'];
 66+ }
 67+}
Property changes on: branches/FileBackend/phase3/includes/filerepo/backend/LockManagerGroup.php
___________________________________________________________________
Added: svn:eol-style
168 + native
Index: branches/FileBackend/phase3/includes/filerepo/backend/LockManager.php
@@ -0,0 +1,683 @@
 2+<?php
 3+/**
 4+ * FileBackend helper class for handling file locking.
 5+ * Locks on resource keys can either be shared or exclusive.
 6+ *
 7+ * Implemenations can keep track of what is locked in the process cache.
 8+ * This can reduce hits to external resources for lock()/unlock() calls.
 9+ *
 10+ * Subclasses should avoid throwing exceptions at all costs.
 11+ */
 12+abstract class LockManager {
 13+ /* Lock types; stronger locks have high values */
 14+ const LOCK_SH = 1; // shared lock (for reads)
 15+ const LOCK_EX = 2; // exclusive lock (for writes)
 16+
 17+ /**
 18+ * Construct a new instance from configuration
 19+ *
 20+ * @param $config Array
 21+ */
 22+ public function __construct( array $config ) {}
 23+
 24+ /**
 25+ * Lock the resources at the given abstract paths
 26+ *
 27+ * @param $paths Array List of resource names
 28+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH
 29+ * @return Status
 30+ */
 31+ final public function lock( array $paths, $type = self::LOCK_EX ) {
 32+ $keys = array_unique( array_map( 'sha1', $paths ) );
 33+ return $this->doLock( $keys, $type );
 34+ }
 35+
 36+ /**
 37+ * Unlock the resources at the given abstract paths
 38+ *
 39+ * @param $paths Array List of storage paths
 40+ * @return Status
 41+ */
 42+ final public function unlock( array $paths ) {
 43+ $keys = array_unique( array_map( 'sha1', $paths ) );
 44+ return $this->doUnlock( $keys, 0 );
 45+ }
 46+
 47+ /**
 48+ * Lock a resource with the given key
 49+ *
 50+ * @param $key Array List of keys to lock (40 char hex hashes)
 51+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH
 52+ * @return string
 53+ */
 54+ abstract protected function doLock( array $keys, $type );
 55+
 56+ /**
 57+ * Unlock a resource with the given key.
 58+ * If $type is given, then only locks of that type should be cleared.
 59+ *
 60+ * @param $key Array List of keys to unlock (40 char hex hashes)
 61+ * @param $type integer LockManager::LOCK_EX, LockManager::LOCK_SH, or 0
 62+ * @return string
 63+ */
 64+ abstract protected function doUnlock( array $keys, $type );
 65+}
 66+
 67+/**
 68+ * Simple version of LockManager based on using FS lock files
 69+ *
 70+ * This should work fine for small sites running off one server.
 71+ * Do not use this with 'lockDir' set to an NFS mount unless the
 72+ * NFS client is at least version 2.6.12. Otherwise, the BSD flock()
 73+ * locks will be ignored; see http://nfs.sourceforge.net/#section_d.
 74+ */
 75+class FSLockManager extends LockManager {
 76+ protected $lockDir; // global dir for all servers
 77+
 78+ /** @var Array Map of (locked key => lock type => count) */
 79+ protected $locksHeld = array();
 80+ /** @var Array Map of (locked key => lock type => lock file handle) */
 81+ protected $handles = array();
 82+
 83+ function __construct( array $config ) {
 84+ $this->lockDir = $config['lockDirectory'];
 85+ }
 86+
 87+ protected function doLock( array $keys, $type ) {
 88+ $status = Status::newGood();
 89+
 90+ $lockedKeys = array(); // files locked in this attempt
 91+ foreach ( $keys as $key ) {
 92+ $subStatus = $this->doSingleLock( $key, $type );
 93+ $status->merge( $subStatus );
 94+ if ( $status->isOK() ) {
 95+ // Don't append to $lockedKeys if $key is already locked.
 96+ // We do NOT want to unlock the key if we have to rollback.
 97+ if ( $subStatus->isGood() ) { // no warnings/fatals?
 98+ $lockedKeys[] = $key;
 99+ }
 100+ } else {
 101+ // Abort and unlock everything
 102+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
 103+ return $status;
 104+ }
 105+ }
 106+
 107+ return $status;
 108+ }
 109+
 110+ protected function doUnlock( array $keys, $type ) {
 111+ $status = Status::newGood();
 112+
 113+ foreach ( $keys as $key ) {
 114+ $status->merge( $this->doSingleUnlock( $key, $type ) );
 115+ }
 116+
 117+ return $status;
 118+ }
 119+
 120+ /**
 121+ * Lock a single resource key
 122+ *
 123+ * @param $key string
 124+ * @param $type integer
 125+ * @return Status
 126+ */
 127+ protected function doSingleLock( $key, $type ) {
 128+ $status = Status::newGood();
 129+
 130+ if ( isset( $this->locksHeld[$key][$type] ) ) {
 131+ ++$this->locksHeld[$key][$type];
 132+ } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
 133+ $this->locksHeld[$key][$type] = 1;
 134+ } else {
 135+ wfSuppressWarnings();
 136+ $handle = fopen( $this->getLockPath( $key ), 'c' );
 137+ wfRestoreWarnings();
 138+ if ( !$handle ) { // lock dir missing?
 139+ wfMkdirParents( $this->lockDir );
 140+ wfSuppressWarnings();
 141+ $handle = fopen( $this->getLockPath( $key ), 'c' ); // try again
 142+ wfRestoreWarnings();
 143+ }
 144+ if ( $handle ) {
 145+ // Either a shared or exclusive lock
 146+ $lock = ( $type == self::LOCK_SH ) ? LOCK_SH : LOCK_EX;
 147+ if ( flock( $handle, $lock | LOCK_NB ) ) {
 148+ // Record this lock as active
 149+ $this->locksHeld[$key][$type] = 1;
 150+ $this->handles[$key][$type] = $handle;
 151+ } else {
 152+ fclose( $handle );
 153+ $status->fatal( 'lockmanager-fail-acquirelock', $key );
 154+ }
 155+ } else {
 156+ $status->fatal( 'lockmanager-fail-openlock', $key );
 157+ }
 158+ }
 159+
 160+ return $status;
 161+ }
 162+
 163+ /**
 164+ * Unlock a single resource key
 165+ *
 166+ * @param $key string
 167+ * @param $type integer
 168+ * @return Status
 169+ */
 170+ protected function doSingleUnlock( $key, $type ) {
 171+ $status = Status::newGood();
 172+
 173+ if ( !isset( $this->locksHeld[$key] ) ) {
 174+ $status->warning( 'lockmanager-notlocked', $key );
 175+ } elseif ( $type && !isset( $this->locksHeld[$key][$type] ) ) {
 176+ $status->warning( 'lockmanager-notlocked', $key );
 177+ } else {
 178+ foreach ( $this->locksHeld[$key] as $lockType => $count ) {
 179+ if ( $type && $lockType != $type ) {
 180+ continue; // only unlock locks of type $type
 181+ }
 182+ --$this->locksHeld[$key][$lockType];
 183+ if ( $this->locksHeld[$key][$lockType] <= 0 ) {
 184+ unset( $this->locksHeld[$key][$lockType] );
 185+ // If a LOCK_SH comes in while we have a LOCK_EX, we don't
 186+ // actually add a handler, so check for handler existence.
 187+ if ( isset( $this->handles[$key][$lockType] ) ) {
 188+ wfSuppressWarnings();
 189+ if ( !flock( $this->handles[$key][$lockType], LOCK_UN ) ) {
 190+ $status->fatal( 'lockmanager-fail-releaselock', $key );
 191+ }
 192+ if ( !fclose( $this->handles[$key][$lockType] ) ) {
 193+ $status->warning( 'lockmanager-fail-closelock', $key );
 194+ }
 195+ wfRestoreWarnings();
 196+ unset( $this->handles[$key][$lockType] );
 197+ }
 198+ }
 199+ }
 200+ if ( !count( $this->locksHeld[$key] ) ) {
 201+ wfSuppressWarnings();
 202+ # No locks are held for the lock file anymore
 203+ if ( !unlink( $this->getLockPath( $key ) ) ) {
 204+ $status->warning( 'lockmanager-fail-deletelock', $key );
 205+ }
 206+ wfRestoreWarnings();
 207+ unset( $this->locksHeld[$key] );
 208+ unset( $this->handles[$key] );
 209+ }
 210+ }
 211+
 212+ return $status;
 213+ }
 214+
 215+ /**
 216+ * Get the path to the lock file for a key
 217+ * @param $key string
 218+ * @return string
 219+ */
 220+ protected function getLockPath( $key ) {
 221+ return "{$this->lockDir}/{$key}.lock";
 222+ }
 223+
 224+ function __destruct() {
 225+ // Make sure remaining files get cleared for sanity
 226+ foreach ( $this->handles as $key => $locks ) {
 227+ foreach ( $locks as $type => $handle ) {
 228+ flock( $handle, LOCK_UN ); // PHP 5.3 will not do this automatically
 229+ fclose( $handle );
 230+ }
 231+ unlink( $this->getLockPath( $key ) );
 232+ }
 233+ }
 234+}
 235+
 236+/**
 237+ * Version of LockManager based on using DB table locks.
 238+ * This is meant for multi-wiki systems that may share share files.
 239+ *
 240+ * All lock requests for a resource, identified by a hash string, will
 241+ * map to one bucket. Each bucket maps to one or several peer DB servers,
 242+ * each having a `file_locks` table with row-level locking.
 243+ *
 244+ * A majority of peer servers must agree for a lock to be acquired.
 245+ * As long as one peer server is up, lock requests will not be blocked
 246+ * just because another peer server cannot be contacted. A global status
 247+ * cache can be setup to track servers that recently missed queries; such
 248+ * servers will not be trusted for obtaining locks.
 249+ *
 250+ * For performance, deadlock detection should be disabled and a small
 251+ * lock-wait timeout should be set via server config. In innoDB, this can
 252+ * done via the innodb_deadlock_detect and innodb_lock_wait_timeout settings.
 253+ */
 254+class DBLockManager extends LockManager {
 255+ /** @var Array Map of bucket indexes to peer sets */
 256+ protected $dbsByBucket; // (bucket index => (ldb1, ldb2, ...))
 257+ /** @var BagOStuff */
 258+ protected $statusCache;
 259+
 260+ protected $trustCache; // boolean
 261+ protected $webTimeout; // integer number of seconds
 262+ protected $cliTimeout; // integer number of seconds
 263+ protected $safeDelay; // integer number of seconds
 264+
 265+ /** @var Array Map of (locked key => lock type => count) */
 266+ protected $locksHeld = array();
 267+ /** $var Array Map Lock-active database connections (server name => Database) */
 268+ protected $activeConns = array();
 269+
 270+ /**
 271+ * Construct a new instance from configuration.
 272+ * $config paramaters include:
 273+ * 'dbsByBucket' : Array of 1-16 consecutive integer keys, starting from 0, with
 274+ * a list of DB names (peers) as values. Each list should have
 275+ * an odd number of items and each DB should have its own server.
 276+ * 'webTimeout' : Lock timeout (seconds) for non-CLI scripts. [optional]
 277+ * This tells the DB server how long to wait before assuming
 278+ * connection failure and releasing all the locks for a session.
 279+ * 'cliTimeout' : Lock timeout (seconds) for CLI scripts. [optional]
 280+ * This tells the DB server how long to wait before assuming
 281+ * connection failure and releasing all the locks for a session.
 282+ * 'safeDelay' : Seconds to mistrust a DB after restart/query loss. [optional]
 283+ * This should reflect the highest max_execution_time that PHP
 284+ * scripts might use on a wiki. Locks are lost on server restart.
 285+ * 'cache' : $wgMemc (if set to a global memcached instance). [optional]
 286+ * This tracks peer servers that couldn't be queried recently.
 287+ * 'trustCache' : Assume cache knows all servers missing queries recently. [optional]
 288+ *
 289+ * @param Array $config
 290+ */
 291+ function __construct( array $config ) {
 292+ // Sanitize dbsByBucket config to prevent PHP errors
 293+ $this->dbsByBucket = array_filter( $config['dbsByBucket'], 'is_array' );
 294+ $this->dbsByBucket = array_values( $this->dbsByBucket ); // consecutive
 295+
 296+ if ( isset( $config['webTimeout'] ) ) {
 297+ $this->webTimeout = $config['webTimeout'];
 298+ } else {
 299+ $met = ini_get( 'max_execution_time' );
 300+ $this->webTimeout = $met ? $met : 60; // use some same amount if 0
 301+ }
 302+ $this->cliTimeout = isset( $config['cliTimeout'] )
 303+ ? $config['cliTimeout']
 304+ : 60; // some sane amount
 305+ $this->safeDelay = isset( $config['safeDelay'] )
 306+ ? $config['safeDelay']
 307+ : max( $this->cliTimeout, $this->webTimeout ); // cover worst case
 308+
 309+ if ( isset( $config['cache'] ) && $config['cache'] instanceof BagOStuff ) {
 310+ $this->statusCache = $config['cache'];
 311+ $this->trustCache = ( !empty( $config['trustCache'] ) && $this->safeDelay > 0 );
 312+ } else {
 313+ $this->statusCache = null;
 314+ $this->trustCache = false;
 315+ }
 316+ }
 317+
 318+ protected function doLock( array $keys, $type ) {
 319+ $status = Status::newGood();
 320+
 321+ $keysToLock = array();
 322+ // Get locks that need to be acquired (buckets => locks)...
 323+ foreach ( $keys as $key ) {
 324+ if ( isset( $this->locksHeld[$key][$type] ) ) {
 325+ ++$this->locksHeld[$key][$type];
 326+ } elseif ( isset( $this->locksHeld[$key][self::LOCK_EX] ) ) {
 327+ $this->locksHeld[$key][$type] = 1;
 328+ } else {
 329+ $bucket = $this->getBucketFromKey( $key );
 330+ $keysToLock[$bucket][] = $key;
 331+ }
 332+ }
 333+
 334+ $lockedKeys = array(); // files locked in this attempt
 335+ // Attempt to acquire these locks...
 336+ foreach ( $keysToLock as $bucket => $keys ) {
 337+ // Acquire the locks for this server. Three main cases can happen:
 338+ // (a) First server is up; common case
 339+ // (b) First server is down but a peer is up
 340+ // (c) First server is down and no peer are up (or none defined)
 341+ $res = $this->doLockingQueryAll( $bucket, $keys, $type );
 342+ if ( $res === 'cantacquire' ) {
 343+ // Resources already locked by another process.
 344+ // Abort and unlock everything we just locked.
 345+ $status->fatal( 'lockmanager-fail-acquirelocks' );
 346+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
 347+ return $status;
 348+ } elseif ( $res !== true ) {
 349+ // Couldn't contact any servers for this bucket.
 350+ // Abort and unlock everything we just locked.
 351+ $status->fatal( 'lockmanager-fail-db-bucket', $bucket );
 352+ $status->merge( $this->doUnlock( $lockedKeys, $type ) );
 353+ return $status;
 354+ }
 355+ // Record these locks as active
 356+ foreach ( $keys as $key ) {
 357+ $this->locksHeld[$key][$type] = 1; // locked
 358+ }
 359+ // Keep track of what locks were made in this attempt
 360+ $lockedKeys = array_merge( $lockedKeys, $keys );
 361+ }
 362+
 363+ return $status;
 364+ }
 365+
 366+ protected function doUnlock( array $keys, $type ) {
 367+ $status = Status::newGood();
 368+
 369+ foreach ( $keys as $key ) {
 370+ if ( !isset( $this->locksHeld[$key] ) ) {
 371+ $status->warning( 'lockmanager-notlocked', $key );
 372+ } elseif ( $type && !isset( $this->locksHeld[$key][$type] ) ) {
 373+ $status->warning( 'lockmanager-notlocked', $key );
 374+ } else {
 375+ foreach ( $this->locksHeld[$key] as $lockType => $count ) {
 376+ if ( $type && $lockType != $type ) {
 377+ continue; // only unlock locks of type $type
 378+ }
 379+ --$this->locksHeld[$key][$lockType];
 380+ if ( $this->locksHeld[$key][$lockType] <= 0 ) {
 381+ unset( $this->locksHeld[$key][$lockType] );
 382+ }
 383+ }
 384+ if ( !count( $this->locksHeld[$key] ) ) {
 385+ unset( $this->locksHeld[$key] ); // no SH or EX locks left for key
 386+ }
 387+ }
 388+ }
 389+
 390+ // Reference count the locks held and COMMIT when zero
 391+ if ( !count( $this->locksHeld ) ) {
 392+ $status->merge( $this->finishLockTransactions() );
 393+ }
 394+
 395+ return $status;
 396+ }
 397+
 398+ /**
 399+ * Get a DB connection to a lock server and acquire locks on $keys.
 400+ *
 401+ * @param $server string
 402+ * @param $keys Array
 403+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
 404+ * @return void
 405+ */
 406+ protected function doLockingQuery( $server, array $keys, $type ) {
 407+ if ( !isset( $this->activeConns[$server] ) ) {
 408+ $this->activeConns[$server] = wfGetDB( DB_MASTER, array(), $server );
 409+ $this->activeConns[$server]->begin(); // start transaction
 410+ # If the connection drops, try to avoid letting the DB rollback
 411+ # and release the locks before the file operations are finished.
 412+ # This won't handle the case of server reboots however.
 413+ $options = array();
 414+ if ( php_sapi_name() == 'cli' ) { // maintenance scripts
 415+ if ( $this->cliTimeout > 0 ) {
 416+ $options['connTimeout'] = $this->cliTimeout;
 417+ }
 418+ } else { // web requests
 419+ if ( $this->webTimeout > 0 ) {
 420+ $options['connTimeout'] = $this->webTimeout;
 421+ }
 422+ }
 423+ $this->activeConns[$server]->setSessionOptions( $options );
 424+ }
 425+ $db = $this->activeConns[$server];
 426+ # Try to get the locks...this should be the last query of this function
 427+ if ( $type == self::LOCK_SH ) { // reader locks
 428+ $db->select( 'file_locks', '1',
 429+ array( 'fl_key' => $keys ),
 430+ __METHOD__,
 431+ array( 'LOCK IN SHARE MODE' ) // single-row gap locks
 432+ );
 433+ } else { // writer locks
 434+ $data = array();
 435+ foreach ( $keys as $key ) {
 436+ $data[] = array( 'fl_key' => $key );
 437+ }
 438+ $db->insert( 'file_locks', $data, __METHOD__ );
 439+ }
 440+ }
 441+
 442+ /**
 443+ * Attempt to acquire locks with the peers for a bucket.
 444+ * This should avoid throwing any exceptions.
 445+ *
 446+ * @param $bucket integer
 447+ * @param $keys Array List of resource keys to lock
 448+ * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH
 449+ * @return bool|string One of (true, 'cantacquire', 'dberrors')
 450+ */
 451+ protected function doLockingQueryAll( $bucket, array $keys, $type ) {
 452+ $yesVotes = 0; // locks made on trustable servers
 453+ $votesLeft = count( $this->dbsByBucket[$bucket] ); // remaining servers
 454+ $quorum = floor( $votesLeft/2 + 1 ); // simple majority
 455+ // Get votes for each server, in order, until we have enough...
 456+ foreach ( $this->dbsByBucket[$bucket] as $index => $server ) {
 457+ if ( $this->trustCache && $votesLeft < $quorum ) {
 458+ // We are now hitting servers that are not normally
 459+ // hit, meaning that some of the first ones are down.
 460+ // Delay using these later servers until it's safe.
 461+ if ( !$this->cacheCheckSkipped( $bucket, $index ) ) {
 462+ return 'dberrors';
 463+ }
 464+ }
 465+ // Check that server is not *known* to be down
 466+ if ( $this->cacheCheckFailures( $server ) ) {
 467+ try {
 468+ // Attempt to acquire the lock on this server
 469+ $this->doLockingQuery( $server, $keys, $type );
 470+ // Check that server has no signs of lock loss
 471+ if ( $this->checkUptime( $server ) ) {
 472+ ++$yesVotes; // success for this peer
 473+ if ( $yesVotes >= $quorum ) {
 474+ if ( $this->trustCache ) {
 475+ // We didn't bother with the servers after this one
 476+ $this->cacheRecordSkipped( $bucket, $index + 1 );
 477+ }
 478+ return true; // lock obtained
 479+ }
 480+ }
 481+ } catch ( DBError $e ) {
 482+ if ( $this->lastErrorIndicatesLocked( $server ) ) {
 483+ return 'cantacquire'; // vetoed; resource locked
 484+ } else { // can't connect?
 485+ $this->cacheRecordFailure( $server );
 486+ }
 487+ }
 488+ }
 489+ $votesLeft--;
 490+ $votesNeeded = $quorum - $yesVotes;
 491+ if ( $votesNeeded > $votesLeft && !$this->trustCache ) {
 492+ // In "trust cache" mode we don't have to meet the quorum
 493+ break; // short-circuit
 494+ }
 495+ }
 496+ // At this point, we must not have meet the quorum
 497+ if ( $yesVotes > 0 && $this->trustCache ) {
 498+ return true; // we are trusting the cache; may comprimise correctness
 499+ }
 500+ return 'dberrors'; // not enough votes to ensure correctness
 501+ }
 502+
 503+ /**
 504+ * Commit all changes to lock-active databases.
 505+ * This should avoid throwing any exceptions.
 506+ *
 507+ * @return Status
 508+ */
 509+ protected function finishLockTransactions() {
 510+ $status = Status::newGood();
 511+ foreach ( $this->activeConns as $server => $db ) {
 512+ try {
 513+ $db->rollback(); // finish transaction and kill any rows
 514+ } catch ( DBError $e ) {
 515+ $status->fatal( 'lockmanager-fail-db-release', $server );
 516+ // oh well; best effort
 517+ }
 518+ }
 519+ $this->activeConns = array();
 520+ return $status;
 521+ }
 522+
 523+ /**
 524+ * Check if the last DB error for $server indicates
 525+ * that a requested resource was locked by another process.
 526+ * This should avoid throwing any exceptions.
 527+ *
 528+ * @param $server string
 529+ * @return bool
 530+ */
 531+ protected function lastErrorIndicatesLocked( $server ) {
 532+ if ( isset( $this->activeConns[$server] ) ) { // sanity
 533+ $db = $this->activeConns[$server];
 534+ return ( $db->wasDeadlock() || $db->wasLockTimeout() );
 535+ }
 536+ return false;
 537+ }
 538+
 539+ /**
 540+ * Checks if the DB server did not recently restart.
 541+ * This curtails the problem of locks falling off when servers restart.
 542+ *
 543+ * @param $server string
 544+ * @return bool
 545+ */
 546+ protected function checkUptime( $server ) {
 547+ if ( isset( $this->activeConns[$server] ) ) { // sanity
 548+ if ( $this->safeDelay > 0 ) {
 549+ $db = $this->activeConns[$server];
 550+ if ( $db->getServerUptime() < $this->safeDelay ) {
 551+ return false;
 552+ }
 553+ }
 554+ return true;
 555+ }
 556+ return false;
 557+ }
 558+
 559+ /**
 560+ * Check that DB servers starting at $index for a
 561+ * bucket were not recently skipped in obtaining a lock.
 562+ *
 563+ * @param $bucket
 564+ * @param $index
 565+ * @return bool
 566+ */
 567+ protected function cacheCheckSkipped( $bucket, $index ) {
 568+ if ( $this->statusCache && $this->safeDelay > 0 ) {
 569+ $key = $this->getSkipsKey( $bucket, $index );
 570+ $skips = $this->statusCache->get( $key );
 571+ return !$skips;
 572+ }
 573+ return true;
 574+ }
 575+
 576+ /**
 577+ * Record that DB servers starting at $index for a
 578+ * bucket were just skipped in obtaining a lock (quorum met).
 579+ *
 580+ * @param $bucket
 581+ * @param $index
 582+ * @return bool Success
 583+ */
 584+ protected function cacheRecordSkipped( $bucket, $index ) {
 585+ if ( $this->statusCache && $this->safeDelay > 0 ) {
 586+ $key = $this->getSkipsKey( $bucket, $index );
 587+ $skips = $this->statusCache->get( $key );
 588+ if ( $skips ) {
 589+ return $this->statusCache->incr( $key );
 590+ } else {
 591+ return $this->statusCache->add( $key, 1, $this->safeDelay );
 592+ }
 593+ }
 594+ return true;
 595+ }
 596+
 597+ /**
 598+ * Get a cache key for recent query skips for a bucket
 599+ *
 600+ * @param $bucket
 601+ * @param $index
 602+ * @return string
 603+ */
 604+ protected function getSkipsKey( $bucket, $index ) {
 605+ return "lockmanager:queryskips:$bucket:$index";
 606+ }
 607+
 608+ /**
 609+ * Checks if the DB server has not recently had connection/query errors.
 610+ * When in "trust cache" mode, this curtails the problem of peers occasionally
 611+ * missing locks. Otherwise, it just avoids wasting time on connection attempts.
 612+ *
 613+ * @param $server string
 614+ * @return bool
 615+ */
 616+ protected function cacheCheckFailures( $server ) {
 617+ if ( $this->statusCache && $this->safeDelay > 0 ) {
 618+ $key = $this->getMissKey( $server );
 619+ $misses = $this->statusCache->get( $key );
 620+ return !$misses;
 621+ }
 622+ return true;
 623+ }
 624+
 625+ /**
 626+ * Log a lock request failure to the cache.
 627+ *
 628+ * Worst case scenario is that a resource lock was only
 629+ * on one peer and then that peer is restarted or goes down.
 630+ * Clients trying to get locks need to know if a server is down.
 631+ *
 632+ * @param $server string
 633+ * @return bool Success
 634+ */
 635+ protected function cacheRecordFailure( $server ) {
 636+ if ( $this->statusCache && $this->safeDelay > 0 ) {
 637+ $key = $this->getMissKey( $server );
 638+ $misses = $this->statusCache->get( $key );
 639+ if ( $misses ) {
 640+ return $this->statusCache->incr( $key );
 641+ } else {
 642+ return $this->statusCache->add( $key, 1, $this->safeDelay );
 643+ }
 644+ }
 645+ return true;
 646+ }
 647+
 648+ /**
 649+ * Get a cache key for recent query misses for a DB server
 650+ *
 651+ * @param $server string
 652+ * @return string
 653+ */
 654+ protected function getMissKey( $server ) {
 655+ return "lockmanager:querymisses:$server";
 656+ }
 657+
 658+ /**
 659+ * Get the bucket for lock key.
 660+ * This should avoid throwing any exceptions.
 661+ *
 662+ * @param $key string (40 char hex key)
 663+ * @return integer
 664+ */
 665+ protected function getBucketFromKey( $key ) {
 666+ $prefix = substr( $key, 0, 2 ); // first 2 hex chars (8 bits)
 667+ return intval( base_convert( $prefix, 16, 10 ) ) % count( $this->dbsByBucket );
 668+ }
 669+}
 670+
 671+/**
 672+ * Simple version of LockManager that does nothing
 673+ */
 674+class NullLockManager extends LockManager {
 675+ function __construct( array $config ) {}
 676+
 677+ protected function doLock( array $keys, $type ) {
 678+ return Status::newGood();
 679+ }
 680+
 681+ protected function doUnlock( array $keys, $type ) {
 682+ return Status::newGood();
 683+ }
 684+}
Property changes on: branches/FileBackend/phase3/includes/filerepo/backend/LockManager.php
___________________________________________________________________
Added: svn:eol-style
1685 + native
Index: branches/FileBackend/phase3/includes/filerepo/backend/FileBackend.php
@@ -24,7 +24,7 @@
2525 abstract class FileBackendBase {
2626 protected $name; // unique backend name
2727 protected $wikiId; // unique wiki name
28 - /** @var FileLockManager */
 28+ /** @var LockManager */
2929 protected $lockManager;
3030
3131 /**
@@ -42,7 +42,7 @@
4343 $this->wikiId = isset( $config['wikiId'] )
4444 ? $config['wikiId']
4545 : wfWikiID();
46 - $this->lockManager = $config['lockManager'];
 46+ $this->lockManager = LockManagerGroup::singleton()->get( $config['lockManager'] );
4747 }
4848
4949 /**
@@ -309,7 +309,6 @@
310310 * source : source storage path
311311 * dest : destination storage path
312312 * overwriteDest : do nothing and pass if an identical file exists at destination
313 - * overwriteSame : override any existing file at destination
314313 *
315314 * @param Array $params
316315 * @return Status
@@ -540,9 +539,9 @@
541540 if ( $parts[0] !== null ) { // either all null or all not null
542541 list( $backend, $container, $relPath ) = $parts;
543542 if ( $backend === $this->name ) { // sanity
544 - $container = $this->fullContainerName( $container );
545543 $relPath = $this->resolveContainerPath( $container, $relPath );
546544 if ( $relPath !== null ) {
 545+ $container = $this->fullContainerName( $container );
547546 return array( $container, $relPath ); // (container, path)
548547 }
549548 }
Index: branches/FileBackend/phase3/includes/filerepo/FileRepo.php
@@ -1076,6 +1076,17 @@
10771077 }
10781078
10791079 /**
 1080+ * Get a local FS copy of a file with a given virtual URL/storage path.
 1081+ * Returns null on failure.
 1082+ *
 1083+ * @return TempFSFile|null
 1084+ */
 1085+ public function getLocalCopy( $virtualUrl ) {
 1086+ $path = $this->resolveToStoragePath( $virtualUrl );
 1087+ return $this->backend->getLocalCopy( array( 'source' => $path ) );
 1088+ }
 1089+
 1090+ /**
10801091 * Get properties of a file with a given virtual URL/storage path.
10811092 * Properties should ultimately be obtained via FSFile::getProps().
10821093 *
Index: branches/FileBackend/phase3/includes/media/MediaTransformOutput.php
@@ -7,7 +7,7 @@
88 */
99
1010 /**
11 - * Base class for the output of MediaHandler::doTransform() and File::transform().
 11+ * Base class for the output of MediaHandler::doFSTransform() and File::transform().
1212 *
1313 * @ingroup Media
1414 */
Index: branches/FileBackend/phase3/includes/media/SVG.php
@@ -85,7 +85,7 @@
8686 * @param int $flags
8787 * @return bool|MediaTransformError|ThumbnailImage|TransformParameterError
8888 */
89 - function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
 89+ function doFSTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
9090 if ( !$this->normaliseParams( $image, $params ) ) {
9191 return new TransformParameterError( $params );
9292 }
@@ -93,7 +93,7 @@
9494 $clientHeight = $params['height'];
9595 $physicalWidth = $params['physicalWidth'];
9696 $physicalHeight = $params['physicalHeight'];
97 - $srcPath = $image->getPath();
 97+ $srcPath = $image->getLocalCopyPath();
9898
9999 if ( $flags & self::TRANSFORM_LATER ) {
100100 return new ThumbnailImage( $image, $dstUrl, $clientWidth, $clientHeight, $dstPath );
Index: branches/FileBackend/phase3/includes/media/DjVu.php
@@ -113,7 +113,7 @@
114114 * @param int $flags
115115 * @return MediaTransformError|ThumbnailImage|TransformParameterError
116116 */
117 - function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
 117+ function doFSTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
118118 global $wgDjvuRenderer, $wgDjvuPostProcessor;
119119
120120 // Fetch XML and check it, to give a more informative error message than the one which
@@ -131,7 +131,7 @@
132132 }
133133 $width = $params['width'];
134134 $height = $params['height'];
135 - $srcPath = $image->getPath();
 135+ $srcPath = $image->getLocalCopyPath();
136136 $page = $params['page'];
137137 if ( $page > $this->pageCount( $image ) ) {
138138 return new MediaTransformError( 'thumbnail_error', $width, $height, wfMsg( 'djvu_page_error' ) );
Index: branches/FileBackend/phase3/includes/media/Bitmap.php
@@ -106,7 +106,7 @@
107107 * @param int $flags
108108 * @return MediaTransformError|ThumbnailImage|TransformParameterError
109109 */
110 - function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
 110+ function doFSTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
111111 if ( !$this->normaliseParams( $image, $params ) ) {
112112 return new TransformParameterError( $params );
113113 }
@@ -126,7 +126,7 @@
127127 'srcWidth' => $image->getWidth(),
128128 'srcHeight' => $image->getHeight(),
129129 'mimeType' => $image->getMimeType(),
130 - 'srcPath' => $image->getPath(),
 130+ 'srcPath' => $image->getLocalCopyPath(),
131131 'dstPath' => $dstPath,
132132 'dstUrl' => $dstUrl,
133133 );
Index: branches/FileBackend/phase3/includes/media/Bitmap_ClientOnly.php
@@ -33,11 +33,11 @@
3434 * @param int $flags
3535 * @return ThumbnailImage|TransformParameterError
3636 */
37 - function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
 37+ function doFSTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
3838 if ( !$this->normaliseParams( $image, $params ) ) {
3939 return new TransformParameterError( $params );
4040 }
4141 return new ThumbnailImage( $image, $image->getURL(), $params['width'],
42 - $params['height'], $image->getPath() );
 42+ $params['height'], $image->getLocalCopyPath() );
4343 }
4444 }
Index: branches/FileBackend/phase3/includes/media/Generic.php
@@ -187,7 +187,7 @@
188188 * @param $dstUrl String: Destination URL to use in output HTML
189189 * @param $params Array: Arbitrary set of parameters validated by $this->validateParam()
190190 */
191 - function getTransform( $image, $dstPath, $dstUrl, $params ) {
 191+ final function getTransform( $image, $dstPath, $dstUrl, $params ) {
192192 return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER );
193193 }
194194
@@ -203,9 +203,47 @@
204204 *
205205 * @return MediaTransformOutput
206206 */
207 - abstract function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 );
 207+ final function doTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 ) {
 208+ if ( FileBackend::isStoragePath( $dstPath ) ) {
 209+ // Create output on FS
 210+ $tmpDest = tempnam( wfTempDir(), 'transform' ) . '.' . $image->getExtension();
 211+ if ( $tmpDest === false ) {
 212+ throw new MWException( "Could not create temporary thumbnail file." );
 213+ }
 214+ $out = $this->doFSTransform( $image, $tmpDest, $dstUrl, $params, $flags );
 215+ // Copy any thumbnail from FS $tmpDest to $dstpath.
 216+ // Note: no file is created if it's to be rendered client-side.
 217+ if ( !$out->isError() && is_file( $tmpDest ) ) {
 218+ $op = array( 'op' => 'store',
 219+ 'source' => $tmpDest, 'dest' => $dstPath, 'overwriteDest' => true );
 220+ if ( !$image->getRepo()->getBackend()->doOperation( $op )->isOK() ) {
 221+ throw new MWException( "Could not copy thumbnail to $dstPath." );
 222+ }
 223+ }
 224+ wfSuppressWarnings();
 225+ unlink( $tmpDest );
 226+ wfRestoreWarnings();
 227+ } else {
 228+ $out = $this->doFSTransform( $image, $dstPath, $dstUrl, $params, $flags );
 229+ }
 230+ return $out;
 231+ }
208232
209233 /**
 234+ * Get a MediaTransformOutput object representing the transformed output. Does the
 235+ * transform unless $flags contains self::TRANSFORM_LATER.
 236+ *
 237+ * @param $image File: the image object
 238+ * @param $dstPath String: filesystem destination path
 239+ * @param $dstUrl String: destination URL to use in output HTML
 240+ * @param $params Array: arbitrary set of parameters validated by $this->validateParam()
 241+ * @param $flags Integer: a bitfield, may contain self::TRANSFORM_LATER
 242+ *
 243+ * @return MediaTransformOutput
 244+ */
 245+ abstract function doFSTransform( $image, $dstPath, $dstUrl, $params, $flags = 0 );
 246+
 247+ /**
210248 * Get the thumbnail extension and MIME type for a given source MIME type
211249 * @return array thumbnail extension and MIME type
212250 */
@@ -260,7 +298,7 @@
261299 * @param $image File
262300 */
263301 function getPageDimensions( $image, $page ) {
264 - $gis = $this->getImageSize( $image, $image->getPath() );
 302+ $gis = $this->getImageSize( $image, $image->getLocalCopyPath() );
265303 return array(
266304 'width' => $gis[0],
267305 'height' => $gis[1]
@@ -643,13 +681,6 @@
644682 }
645683
646684 /**
647 - * Get a transform output object without actually doing the transform
648 - */
649 - function getTransform( $image, $dstPath, $dstUrl, $params ) {
650 - return $this->doTransform( $image, $dstPath, $dstUrl, $params, self::TRANSFORM_LATER );
651 - }
652 -
653 - /**
654685 * Validate thumbnail parameters and fill in the correct height
655686 *
656687 * @param $width Integer: specified width (input/output)
Index: branches/FileBackend/phase3/includes/AutoLoader.php
@@ -490,11 +490,11 @@
491491 'FileBackend' => 'includes/filerepo/backend/FileBackend.php',
492492 'FileBackendMultiWrite' => 'includes/filerepo/backend/FileBackendMultiWrite.php',
493493 'FSFileBackend' => 'includes/filerepo/backend/FSFileBackend.php',
494 - 'FileIterator' => 'includes/filerepo/backend/FSFileBackend.php',
495 - 'FileLockManagerGroup' => 'includes/filerepo/backend/FileLockManagerGroup.php',
496 - 'FileLockManager' => 'includes/filerepo/backend/FileLockManager.php',
497 - 'FSFileLockManager' => 'includes/filerepo/backend/FileLockManager.php',
498 - 'DBFileLockManager' => 'includes/filerepo/backend/FileLockManager.php',
 494+ 'FSFileIterator' => 'includes/filerepo/backend/FSFileBackend.php',
 495+ 'LockManagerGroup' => 'includes/filerepo/backend/LockManagerGroup.php',
 496+ 'LockManager' => 'includes/filerepo/backend/LockManager.php',
 497+ 'FSLockManager' => 'includes/filerepo/backend/LockManager.php',
 498+ 'DBLockManager' => 'includes/filerepo/backend/LockManager.php',
499499 'FileOp' => 'includes/filerepo/backend/FileOp.php',
500500 'StoreFileOp' => 'includes/filerepo/backend/FileOp.php',
501501 'CopyFileOp' => 'includes/filerepo/backend/FileOp.php',

Status & tagging log