r109809 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r109808‎ | r109809 | r109810 >
Date:08:33, 23 January 2012
Author:aaron
Status:ok
Tags:filebackend 
Comment:
* Added some wfProfileIn() calls to file backend code.
* Made FileBackend::getFileProps() final.
* Added exception needed in SwiftFileBackend::getConnection().
* Various FileBackendTests fixes and additions. Optimized it a bit by keeping the backend instance in memory.
Modified paths:
  • /trunk/phase3/includes/filerepo/backend/FileBackend.php (modified) (history)
  • /trunk/phase3/includes/filerepo/backend/SwiftFileBackend.php (modified) (history)
  • /trunk/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php (modified) (history)

Diff [purge]

Index: trunk/phase3/tests/phpunit/includes/filerepo/FileBackendTest.php
@@ -6,21 +6,27 @@
77 class FileBackendTest extends MediaWikiTestCase {
88 private $backend, $multiBackend;
99 private $filesToPrune;
 10+ private static $backendToUse;
1011
1112 function setUp() {
1213 global $wgFileBackends;
1314 parent::setUp();
1415 $tmpDir = wfTempDir() . '/file-backend-test-' . time() . '-' . mt_rand();
1516 if ( $this->getCliArg( 'use-filebackend=' ) ) {
16 - $name = $this->getCliArg( 'use-filebackend=' );
17 - $useConfig = array();
18 - foreach ( $wgFileBackends as $conf ) {
19 - if ( $conf['name'] == $name ) {
20 - $useConfig = $conf;
 17+ if ( self::$backendToUse ) {
 18+ $this->singleBackend = self::$backendToUse;
 19+ } else {
 20+ $name = $this->getCliArg( 'use-filebackend=' );
 21+ $useConfig = array();
 22+ foreach ( $wgFileBackends as $conf ) {
 23+ if ( $conf['name'] == $name ) {
 24+ $useConfig = $conf;
 25+ }
2126 }
 27+ $useConfig['name'] = 'localtesting'; // swap name
 28+ self::$backendToUse = new $conf['class']( $useConfig );
 29+ $this->singleBackend = self::$backendToUse;
2230 }
23 - $useConfig['name'] = 'localtesting'; // swap name
24 - $this->singleBackend = new $conf['class']( $useConfig );
2531 } else {
2632 $this->singleBackend = new FSFileBackend( array(
2733 'name' => 'localtesting',
@@ -88,11 +94,16 @@
8995 $this->backend->prepare( array( 'dir' => dirname( $dest ) ) );
9096
9197 file_put_contents( $source, "Unit test file" );
 98+
 99+ if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 100+ $this->backend->store( $op );
 101+ }
 102+
92103 $status = $this->backend->doOperation( $op );
93104
94105 $this->assertEquals( array(), $status->errors,
95106 "Store from $source to $dest succeeded without warnings ($backendName)." );
96 - $this->assertEquals( true, $status->isOK(),
 107+ $this->assertEquals( array(), $status->errors,
97108 "Store from $source to $dest succeeded ($backendName)." );
98109 $this->assertEquals( array( 0 => true ), $status->success,
99110 "Store from $source to $dest has proper 'success' field in Status ($backendName)." );
@@ -123,14 +134,21 @@
124135 $toPath, // dest
125136 );
126137
127 - $op['overwrite'] = true;
 138+ $op2 = $op;
 139+ $op2['overwrite'] = true;
128140 $cases[] = array(
129 - $op, // operation
 141+ $op2, // operation
130142 $tmpName, // source
131143 $toPath, // dest
132144 );
133145
134 - // @TODO: test overwriteSame
 146+ $op2 = $op;
 147+ $op2['overwriteSame'] = true;
 148+ $cases[] = array(
 149+ $op2, // operation
 150+ $tmpName, // source
 151+ $toPath, // dest
 152+ );
135153
136154 return $cases;
137155 }
@@ -158,10 +176,15 @@
159177
160178 $status = $this->backend->doOperation(
161179 array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
162 - $this->assertEquals( true, $status->isOK(),
 180+ $this->assertEquals( array(), $status->errors,
163181 "Creation of file at $source succeeded ($backendName)." );
164182
 183+ if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 184+ $this->backend->copy( $op );
 185+ }
 186+
165187 $status = $this->backend->doOperation( $op );
 188+
166189 $this->assertEquals( array(), $status->errors,
167190 "Copy from $source to $dest succeeded without warnings ($backendName)." );
168191 $this->assertEquals( true, $status->isOK(),
@@ -197,13 +220,22 @@
198221 $dest, // dest
199222 );
200223
201 - $op['overwrite'] = true;
 224+ $op2 = $op;
 225+ $op2['overwrite'] = true;
202226 $cases[] = array(
203 - $op, // operation
 227+ $op2, // operation
204228 $source, // source
205229 $dest, // dest
206230 );
207231
 232+ $op2 = $op;
 233+ $op2['overwriteSame'] = true;
 234+ $cases[] = array(
 235+ $op2, // operation
 236+ $source, // source
 237+ $dest, // dest
 238+ );
 239+
208240 return $cases;
209241 }
210242
@@ -230,9 +262,13 @@
231263
232264 $status = $this->backend->doOperation(
233265 array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
234 - $this->assertEquals( true, $status->isOK(),
 266+ $this->assertEquals( array(), $status->errors,
235267 "Creation of file at $source succeeded ($backendName)." );
236268
 269+ if ( isset( $op['overwrite'] ) || isset( $op['overwriteSame'] ) ) {
 270+ $this->backend->copy( $op );
 271+ }
 272+
237273 $status = $this->backend->doOperation( $op );
238274 $this->assertEquals( array(), $status->errors,
239275 "Move from $source to $dest succeeded without warnings ($backendName)." );
@@ -271,13 +307,22 @@
272308 $dest, // dest
273309 );
274310
275 - $op['overwrite'] = true;
 311+ $op2 = $op;
 312+ $op2['overwrite'] = true;
276313 $cases[] = array(
277 - $op, // operation
 314+ $op2, // operation
278315 $source, // source
279316 $dest, // dest
280317 );
281318
 319+ $op2 = $op;
 320+ $op2['overwriteSame'] = true;
 321+ $cases[] = array(
 322+ $op2, // operation
 323+ $source, // source
 324+ $dest, // dest
 325+ );
 326+
282327 return $cases;
283328 }
284329
@@ -304,7 +349,7 @@
305350 if ( $withSource ) {
306351 $status = $this->backend->doOperation(
307352 array( 'op' => 'create', 'content' => 'blahblah', 'dst' => $source ) );
308 - $this->assertEquals( true, $status->isOK(),
 353+ $this->assertEquals( array(), $status->errors,
309354 "Creation of file at $source succeeded ($backendName)." );
310355 }
311356
@@ -388,7 +433,7 @@
389434 if ( $alreadyExists ) {
390435 $status = $this->backend->doOperation(
391436 array( 'op' => 'create', 'content' => $oldText, 'dst' => $dest ) );
392 - $this->assertEquals( true, $status->isOK(),
 437+ $this->assertEquals( array(), $status->errors,
393438 "Creation of file at $dest succeeded ($backendName)." );
394439 }
395440
@@ -452,15 +497,26 @@
453498 strlen( $dummyText )
454499 );
455500
456 - $op['overwrite'] = true;
 501+ $op2 = $op;
 502+ $op2['overwrite'] = true;
457503 $cases[] = array(
458 - $op, // operation
 504+ $op2, // operation
459505 $source, // source
460506 true, // dest already exists
461507 true, // succeeds
462508 strlen( $dummyText )
463509 );
464510
 511+ $op2 = $op;
 512+ $op2['overwriteSame'] = true;
 513+ $cases[] = array(
 514+ $op2, // operation
 515+ $source, // source
 516+ true, // dest already exists
 517+ false, // succeeds
 518+ strlen( $dummyText )
 519+ );
 520+
465521 return $cases;
466522 }
467523
@@ -498,7 +554,7 @@
499555 }
500556 $status = $this->backend->doOperations( $ops );
501557
502 - $this->assertEquals( true, $status->isOK(),
 558+ $this->assertEquals( array(), $status->errors,
503559 "Creation of source files succeeded ($backendName)." );
504560
505561 $dest = $params['dst'];
@@ -664,7 +720,7 @@
665721
666722 $status = $this->backend->doOperation(
667723 array( 'op' => 'create', 'content' => $content, 'dst' => $source ) );
668 - $this->assertEquals( true, $status->isOK(),
 724+ $this->assertEquals( array(), $status->errors,
669725 "Creation of file at $source succeeded ($backendName)." );
670726
671727 $tmpFile = $this->backend->getLocalCopy( array( 'src' => $source ) );
@@ -707,7 +763,7 @@
708764
709765 $status = $this->backend->doOperation(
710766 array( 'op' => 'create', 'content' => $content, 'dst' => $source ) );
711 - $this->assertEquals( true, $status->isOK(),
 767+ $this->assertEquals( array(), $status->errors,
712768 "Creation of file at $source succeeded ($backendName)." );
713769
714770 $tmpFile = $this->backend->getLocalReference( array( 'src' => $source ) );
@@ -996,7 +1052,7 @@
9971053 $iter = $backend->getFileList( array( 'dir' => "$base/$container" ) );
9981054 if ( $iter ) {
9991055 foreach ( $iter as $file ) {
1000 - $backend->doOperation( array( 'op' => 'delete', 'src' => "$base/$container/$file" ) );
 1056+ $backend->delete( array( 'src' => "$base/$container/$file", 'ignoreMissingSource' => 1 ) );
10011057 $tmp = $file;
10021058 while ( $tmp = FileBackend::parentStoragePath( $tmp ) ) {
10031059 $backend->clean( array( 'dir' => $tmp ) );
Index: trunk/phase3/includes/filerepo/backend/SwiftFileBackend.php
@@ -655,7 +655,7 @@
656656 */
657657 protected function getConnection() {
658658 if ( $this->conn === false ) {
659 - return false; // failed last attempt
 659+ throw new InvalidResponseException; // failed last attempt
660660 }
661661 // Session keys expire after a while, so we renew them periodically
662662 if ( $this->conn && ( time() - $this->connStarted ) > $this->authTTL ) {
Index: trunk/phase3/includes/filerepo/backend/FileBackend.php
@@ -629,12 +629,14 @@
630630 * @return Status
631631 */
632632 final public function createInternal( array $params ) {
 633+ wfProfileIn( __METHOD__ );
633634 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
634635 $status = Status::newFatal( 'backend-fail-create', $params['dst'] );
635636 } else {
636637 $status = $this->doCreateInternal( $params );
637638 $this->clearCache( array( $params['dst'] ) );
638639 }
 640+ wfProfileOut( __METHOD__ );
639641 return $status;
640642 }
641643
@@ -656,12 +658,14 @@
657659 * @return Status
658660 */
659661 final public function storeInternal( array $params ) {
 662+ wfProfileIn( __METHOD__ );
660663 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
661664 $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
662665 } else {
663666 $status = $this->doStoreInternal( $params );
664667 $this->clearCache( array( $params['dst'] ) );
665668 }
 669+ wfProfileOut( __METHOD__ );
666670 return $status;
667671 }
668672
@@ -683,8 +687,10 @@
684688 * @return Status
685689 */
686690 final public function copyInternal( array $params ) {
 691+ wfProfileIn( __METHOD__ );
687692 $status = $this->doCopyInternal( $params );
688693 $this->clearCache( array( $params['dst'] ) );
 694+ wfProfileOut( __METHOD__ );
689695 return $status;
690696 }
691697
@@ -705,8 +711,10 @@
706712 * @return Status
707713 */
708714 final public function deleteInternal( array $params ) {
 715+ wfProfileIn( __METHOD__ );
709716 $status = $this->doDeleteInternal( $params );
710717 $this->clearCache( array( $params['src'] ) );
 718+ wfProfileOut( __METHOD__ );
711719 return $status;
712720 }
713721
@@ -728,8 +736,10 @@
729737 * @return Status
730738 */
731739 final public function moveInternal( array $params ) {
 740+ wfProfileIn( __METHOD__ );
732741 $status = $this->doMoveInternal( $params );
733742 $this->clearCache( array( $params['src'], $params['dst'] ) );
 743+ wfProfileOut( __METHOD__ );
734744 return $status;
735745 }
736746
@@ -739,12 +749,11 @@
740750 protected function doMoveInternal( array $params ) {
741751 // Copy source to dest
742752 $status = $this->copyInternal( $params );
743 - if ( !$status->isOK() ) {
744 - return $status;
 753+ if ( $status->isOK() ) {
 754+ // Delete source (only fails due to races or medium going down)
 755+ $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
 756+ $status->setResult( true, $status->value ); // ignore delete() errors
745757 }
746 - // Delete source (only fails due to races or medium going down)
747 - $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
748 - $status->setResult( true, $status->value ); // ignore delete() errors
749758 return $status;
750759 }
751760
@@ -752,17 +761,17 @@
753762 * @see FileBackendBase::concatenate()
754763 */
755764 final public function concatenate( array $params ) {
 765+ wfProfileIn( __METHOD__ );
756766 $status = Status::newGood();
757767
758768 // Try to lock the source files for the scope of this function
759769 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
760 - if ( !$status->isOK() ) {
761 - return $status; // abort
 770+ if ( $status->isOK() ) {
 771+ // Actually do the concatenation
 772+ $status->merge( $this->doConcatenate( $params ) );
762773 }
763774
764 - // Actually do the concatenation
765 - $status->merge( $this->doConcatenate( $params ) );
766 -
 775+ wfProfileOut( __METHOD__ );
767776 return $status;
768777 }
769778
@@ -825,12 +834,16 @@
826835 * @see FileBackendBase::doPrepare()
827836 */
828837 final protected function doPrepare( array $params ) {
 838+ wfProfileIn( __METHOD__ );
 839+
829840 $status = Status::newGood();
830841 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
831842 if ( $dir === null ) {
832843 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 844+ wfProfileOut( __METHOD__ );
833845 return $status; // invalid storage path
834846 }
 847+
835848 if ( $shard !== null ) { // confined to a single container/shard
836849 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
837850 } else { // directory is on several shards
@@ -840,6 +853,8 @@
841854 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
842855 }
843856 }
 857+
 858+ wfProfileOut( __METHOD__ );
844859 return $status;
845860 }
846861
@@ -854,12 +869,16 @@
855870 * @see FileBackendBase::doSecure()
856871 */
857872 final protected function doSecure( array $params ) {
 873+ wfProfileIn( __METHOD__ );
858874 $status = Status::newGood();
 875+
859876 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
860877 if ( $dir === null ) {
861878 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 879+ wfProfileOut( __METHOD__ );
862880 return $status; // invalid storage path
863881 }
 882+
864883 if ( $shard !== null ) { // confined to a single container/shard
865884 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
866885 } else { // directory is on several shards
@@ -869,6 +888,8 @@
870889 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
871890 }
872891 }
 892+
 893+ wfProfileOut( __METHOD__ );
873894 return $status;
874895 }
875896
@@ -883,18 +904,24 @@
884905 * @see FileBackendBase::doClean()
885906 */
886907 final protected function doClean( array $params ) {
 908+ wfProfileIn( __METHOD__ );
887909 $status = Status::newGood();
 910+
888911 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
889912 if ( $dir === null ) {
890913 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
 914+ wfProfileOut( __METHOD__ );
891915 return $status; // invalid storage path
892916 }
 917+
893918 // Attempt to lock this directory...
894919 $filesLockEx = array( $params['dir'] );
895920 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
896921 if ( !$status->isOK() ) {
 922+ wfProfileOut( __METHOD__ );
897923 return $status; // abort
898924 }
 925+
899926 if ( $shard !== null ) { // confined to a single container/shard
900927 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
901928 } else { // directory is on several shards
@@ -904,6 +931,8 @@
905932 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
906933 }
907934 }
 935+
 936+ wfProfileOut( __METHOD__ );
908937 return $status;
909938 }
910939
@@ -918,47 +947,44 @@
919948 * @see FileBackendBase::fileExists()
920949 */
921950 final public function fileExists( array $params ) {
 951+ wfProfileIn( __METHOD__ );
922952 $stat = $this->getFileStat( $params );
923 - if ( $stat === null ) {
924 - return null; // failure
925 - }
926 - return (bool)$stat;
 953+ wfProfileOut( __METHOD__ );
 954+ return ( $stat === null ) ? null : (bool)$stat; // null => failure
927955 }
928956
929957 /**
930958 * @see FileBackendBase::getFileTimestamp()
931959 */
932960 final public function getFileTimestamp( array $params ) {
 961+ wfProfileIn( __METHOD__ );
933962 $stat = $this->getFileStat( $params );
934 - if ( $stat ) {
935 - return $stat['mtime'];
936 - } else {
937 - return false;
938 - }
 963+ wfProfileOut( __METHOD__ );
 964+ return $stat ? $stat['mtime'] : false;
939965 }
940966
941967 /**
942968 * @see FileBackendBase::getFileSize()
943969 */
944970 final public function getFileSize( array $params ) {
 971+ wfProfileIn( __METHOD__ );
945972 $stat = $this->getFileStat( $params );
946 - if ( $stat ) {
947 - return $stat['size'];
948 - } else {
949 - return false;
950 - }
 973+ wfProfileOut( __METHOD__ );
 974+ return $stat ? $stat['size'] : false;
951975 }
952976
953977 /**
954978 * @see FileBackendBase::getFileStat()
955979 */
956980 final public function getFileStat( array $params ) {
 981+ wfProfileIn( __METHOD__ );
957982 $path = $params['src'];
958983 $latest = !empty( $params['latest'] );
959984 if ( isset( $this->cache[$path]['stat'] ) ) {
960985 // If we want the latest data, check that this cached
961986 // value was in fact fetched with the latest available data.
962987 if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
 988+ wfProfileOut( __METHOD__ );
963989 return $this->cache[$path]['stat'];
964990 }
965991 }
@@ -968,6 +994,7 @@
969995 $this->cache[$path]['stat'] = $stat;
970996 $this->cache[$path]['stat']['latest'] = $latest;
971997 }
 998+ wfProfileOut( __METHOD__ );
972999 return $stat;
9731000 }
9741001
@@ -980,13 +1007,16 @@
9811008 * @see FileBackendBase::getFileContents()
9821009 */
9831010 public function getFileContents( array $params ) {
 1011+ wfProfileIn( __METHOD__ );
9841012 $tmpFile = $this->getLocalReference( $params );
9851013 if ( !$tmpFile ) {
 1014+ wfProfileOut( __METHOD__ );
9861015 return false;
9871016 }
9881017 wfSuppressWarnings();
9891018 $data = file_get_contents( $tmpFile->getPath() );
9901019 wfRestoreWarnings();
 1020+ wfProfileOut( __METHOD__ );
9911021 return $data;
9921022 }
9931023
@@ -994,8 +1024,10 @@
9951025 * @see FileBackendBase::getFileSha1Base36()
9961026 */
9971027 final public function getFileSha1Base36( array $params ) {
 1028+ wfProfileIn( __METHOD__ );
9981029 $path = $params['src'];
9991030 if ( isset( $this->cache[$path]['sha1'] ) ) {
 1031+ wfProfileOut( __METHOD__ );
10001032 return $this->cache[$path]['sha1'];
10011033 }
10021034 $hash = $this->doGetFileSha1Base36( $params );
@@ -1003,6 +1035,7 @@
10041036 $this->trimCache(); // limit memory
10051037 $this->cache[$path]['sha1'] = $hash;
10061038 }
 1039+ wfProfileOut( __METHOD__ );
10071040 return $hash;
10081041 }
10091042
@@ -1021,21 +1054,22 @@
10221055 /**
10231056 * @see FileBackendBase::getFileProps()
10241057 */
1025 - public function getFileProps( array $params ) {
 1058+ final public function getFileProps( array $params ) {
 1059+ wfProfileIn( __METHOD__ );
10261060 $fsFile = $this->getLocalReference( $params );
1027 - if ( !$fsFile ) {
1028 - return FSFile::placeholderProps();
1029 - } else {
1030 - return $fsFile->getProps();
1031 - }
 1061+ $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
 1062+ wfProfileOut( __METHOD__ );
 1063+ return $props;
10321064 }
10331065
10341066 /**
10351067 * @see FileBackendBase::getLocalReference()
10361068 */
10371069 public function getLocalReference( array $params ) {
 1070+ wfProfileIn( __METHOD__ );
10381071 $path = $params['src'];
10391072 if ( isset( $this->cache[$path]['localRef'] ) ) {
 1073+ wfProfileOut( __METHOD__ );
10401074 return $this->cache[$path]['localRef'];
10411075 }
10421076 $tmpFile = $this->getLocalCopy( $params );
@@ -1043,6 +1077,7 @@
10441078 $this->trimCache(); // limit memory
10451079 $this->cache[$path]['localRef'] = $tmpFile;
10461080 }
 1081+ wfProfileOut( __METHOD__ );
10471082 return $tmpFile;
10481083 }
10491084
@@ -1050,6 +1085,7 @@
10511086 * @see FileBackendBase::streamFile()
10521087 */
10531088 final public function streamFile( array $params ) {
 1089+ wfProfileIn( __METHOD__ );
10541090 $status = Status::newGood();
10551091
10561092 $info = $this->getFileStat( $params );
@@ -1068,6 +1104,7 @@
10691105 $status->fatal( 'backend-fail-stream', $params['src'] );
10701106 }
10711107
 1108+ wfProfileOut( __METHOD__ );
10721109 return $status;
10731110 }
10741111
@@ -1169,6 +1206,7 @@
11701207 * @see FileBackendBase::doOperationsInternal()
11711208 */
11721209 protected function doOperationsInternal( array $ops, array $opts ) {
 1210+ wfProfileIn( __METHOD__ );
11731211 $status = Status::newGood();
11741212
11751213 // Build up a list of FileOps...
@@ -1190,6 +1228,7 @@
11911229 $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
11921230 $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
11931231 if ( !$status->isOK() ) {
 1232+ wfProfileOut( __METHOD__ );
11941233 return $status; // abort
11951234 }
11961235 }
@@ -1204,6 +1243,7 @@
12051244 $status->merge( $subStatus );
12061245 $status->success = $subStatus->success; // not done in merge()
12071246
 1247+ wfProfileOut( __METHOD__ );
12081248 return $status;
12091249 }
12101250

Status & tagging log