| Index: trunk/extensions/SwiftMedia/LocalSettings.php |
| — | — | @@ -126,8 +126,6 @@ |
| 127 | 127 | # End of automatically generated settings. |
| 128 | 128 | # Add more configuration options below. |
| 129 | 129 | |
| 130 | | -if (1) { |
| 131 | | - |
| 132 | 130 | require_once( "$IP/extensions/SwiftMedia/SwiftMedia.php" ); |
| 133 | 131 | |
| 134 | 132 | $wgUploadDirectory = "$IP/images/swift"; |
| — | — | @@ -135,24 +133,24 @@ |
| 136 | 134 | $wgUploadPath = "http://alsted.wikimedia.org/images/swift"; |
| 137 | 135 | |
| 138 | 136 | $wgLocalFileRepo = array( |
| 139 | | - 'class' => 'SwiftRepo', |
| 140 | | - 'name' => 'swift', |
| 141 | | - #'directory' => 'http://alsted.wikimedia.org/images', #$wgUploadDirectory, |
| 142 | | - 'directory' => $wgUploadDirectory, |
| | 137 | + 'class' => 'SwiftRepo', |
| | 138 | + // $wgLocalFileRepo must be named 'local' for $repo->isLocal() to work |
| | 139 | + 'name' => 'local', |
| | 140 | + #'directory' => 'http://alsted.wikimedia.org/images', #$wgUploadDirectory, |
| | 141 | + 'directory' => $wgUploadDirectory, |
| 143 | 142 | 'user' => 'system:media', |
| 144 | | - 'key' => '8lksg0p', |
| | 143 | + 'key' => 'secret', |
| 145 | 144 | 'authurl' => 'http://alsted.wikimedia.org/auth/v1.0', |
| 146 | 145 | 'container' => 'images%2Fswift', |
| 147 | | - 'scriptDirUrl' => $wgScriptPath, |
| 148 | | - 'scriptExtension' => $wgScriptExtension, |
| 149 | | - 'url' => $wgUploadBaseUrl ? $wgUploadBaseUrl . $wgUploadPath : $wgUploadPath, |
| 150 | | - 'hashLevels' => $wgHashedUploadDirectory ? 2 : 0, |
| 151 | | - 'thumbScriptUrl' => $wgThumbnailScriptPath, |
| 152 | | - 'transformVia404' => !$wgGenerateThumbnailOnParse, |
| 153 | | - #'deletedDir' => $wgDeletedDirectory, |
| 154 | | - #'deletedHashLevels' => 3 |
| | 146 | + 'scriptDirUrl' => $wgScriptPath, |
| | 147 | + 'scriptExtension' => $wgScriptExtension, |
| | 148 | + 'url' => $wgUploadBaseUrl ? $wgUploadBaseUrl . $wgUploadPath : $wgUploadPath, |
| | 149 | + 'hashLevels' => $wgHashedUploadDirectory ? 2 : 0, |
| | 150 | + 'thumbScriptUrl' => $wgThumbnailScriptPath, |
| | 151 | + 'transformVia404' => !$wgGenerateThumbnailOnParse, |
| | 152 | + 'deletedDir' => $wgDeletedDirectory, |
| | 153 | + 'deletedHashLevels' => 3 |
| 155 | 154 | ); |
| 156 | | -} |
| 157 | 155 | |
| 158 | 156 | $wgDebugLogFile = "/var/www/debug/abcd"; |
| 159 | 157 | $wgShowExceptionDetails = true; |
| Index: trunk/extensions/SwiftMedia/copyover |
| — | — | @@ -4,3 +4,4 @@ |
| 5 | 5 | scp rnelson@alsted.wikimedia.org:/etc/swift/proxy-server.conf . |
| 6 | 6 | scp rnelson@ersch.wikimedia.org:/var/www/extensions/SwiftMedia/{SwiftMedia.body.php,SwiftMedia.i18n.php,SwiftMedia.php} . |
| 7 | 7 | scp rnelson@alsted.wikimedia.org:/usr/local/lib/python2.6/dist-packages/wmf/{client.py,__init__.py,rewrite.py} wmf/ |
| | 8 | +perl -pi -e "s/'key' => '.*'/'key' => 'secret'/" LocalSettings.php |
| Index: trunk/extensions/SwiftMedia/SwiftMedia.body.php |
| — | — | @@ -1171,6 +1171,7 @@ |
| 1172 | 1172 | foreach ( $result as $row ) { |
| 1173 | 1173 | $batch->addOld( $row->oi_archive_name ); |
| 1174 | 1174 | } |
| | 1175 | + //wfDebug(__METHOD__ . var_export($batch, true) . "\n"); |
| 1175 | 1176 | $status = $batch->execute(); |
| 1176 | 1177 | |
| 1177 | 1178 | if ( $status->ok ) { |
| — | — | @@ -2410,43 +2411,18 @@ |
| 2411 | 2412 | */ |
| 2412 | 2413 | function deleteBatch( $sourceDestPairs ) { |
| 2413 | 2414 | wfDebug( __METHOD__ . " deleting " . var_export($sourceDestPairs, true) . "\n"); |
| 2414 | | - $status = $this->newGood(); |
| 2415 | 2415 | |
| 2416 | 2416 | /** |
| 2417 | | - * Validate filenames and create archive directories |
| | 2417 | + * Move the files |
| 2418 | 2418 | */ |
| | 2419 | + $triplets = array(); |
| 2419 | 2420 | foreach ( $sourceDestPairs as $pair ) { |
| 2420 | 2421 | list( $srcRel, $archiveRel ) = $pair; |
| 2421 | | - if ( !$this->validateFilename( $srcRel ) ) { |
| 2422 | | - throw new MWException( __METHOD__.':Validation error in $srcRel' ); |
| 2423 | | - } |
| 2424 | | - if ( !$this->validateFilename( $archiveRel ) ) { |
| 2425 | | - throw new MWException( __METHOD__.':Validation error in $archiveRel' ); |
| 2426 | | - } |
| 2427 | | - } |
| 2428 | | - if ( !$status->ok ) { |
| 2429 | | - // Abort early |
| 2430 | | - return $status; |
| 2431 | | - } |
| 2432 | 2422 | |
| 2433 | | - /** |
| 2434 | | - * Move the files |
| 2435 | | - * We're now committed to returning an OK result, which will lead to |
| 2436 | | - * the files being moved in the DB also. |
| 2437 | | - */ |
| 2438 | | - foreach ( $sourceDestPairs as $pair ) { |
| 2439 | | - list( $srcRel, $archiveRel ) = $pair; |
| 2440 | | - $srcPath = "{$this->directory}/$srcRel"; |
| 2441 | | - $archivePath = "{$this->deletedDir}/$archiveRel"; |
| 2442 | | - $good = true; |
| 2443 | | - // FIXME we need to copy $this->container/$srcPath to $this->container%2Fdeleted/$archivePath |
| 2444 | | - // FIXME then delete $this->container/$srcPath |
| 2445 | | - if ( $good ) { |
| 2446 | | - $status->successCount++; |
| 2447 | | - } else { |
| 2448 | | - $status->failCount++; |
| 2449 | | - } |
| | 2423 | + $triplets[] = array( "mwrepo://{$this->name}/public/$srcRel", 'deleted', $archiveRel ); |
| | 2424 | + |
| 2450 | 2425 | } |
| | 2426 | + $status = $this->storeBatch( $triplets, FileRepo::OVERWRITE_SAME | FileRepo::DELETE_SOURCE ); |
| 2451 | 2427 | return $status; |
| 2452 | 2428 | } |
| 2453 | 2429 | |
| — | — | @@ -2499,14 +2475,14 @@ |
| 2500 | 2476 | } |
| 2501 | 2477 | function newFile( $title, $time = false ) { |
| 2502 | 2478 | if ( empty($title) ) { return null; } |
| 2503 | | - wfDebug( __METHOD__ . " $title, $time " . var_export($this->fileFactory, true) ."\n" ); |
| | 2479 | + //wfDebug( __METHOD__ . " $title, $time " . var_export($this->fileFactory, true) ."\n" ); |
| 2504 | 2480 | $f = parent::newFile( $title, $time ); |
| 2505 | 2481 | return $f; |
| 2506 | 2482 | } |
| 2507 | 2483 | function findFile( $title, $options = array() ) { |
| 2508 | | - wfDebug( __METHOD__ . " finding $title" . var_export($options, true) . "\n" ); |
| | 2484 | + //wfDebug( __METHOD__ . " finding $title" . var_export($options, true) . "\n" ); |
| 2509 | 2485 | $found = parent::findFile( $title, $options ); |
| 2510 | | - wfDebug( __METHOD__ . " found " . var_export($found, true) . "\n" ); |
| | 2486 | + //wfDebug( __METHOD__ . " found " . var_export($found, true) . "\n" ); |
| 2511 | 2487 | return $found; |
| 2512 | 2488 | } |
| 2513 | 2489 | |
| — | — | @@ -2540,9 +2516,6 @@ |
| 2541 | 2517 | #wfDebug( "Bytes stored in container: " . $container->bytes_used . "\n" ); |
| 2542 | 2518 | #wfDebug( "Object: " . var_export($pic, true) . "\n" ); |
| 2543 | 2519 | |
| 2544 | | - # Delete specific object |
| 2545 | | - #$container->delete_object("disco_dancing.jpg"); |
| 2546 | | - |
| 2547 | 2520 | # paranoia |
| 2548 | 2521 | $status = $this->newGood( array() ); |
| 2549 | 2522 | foreach ( $triplets as $i => $triplet ) { |
| — | — | @@ -2645,7 +2618,58 @@ |
| 2646 | 2619 | $container->delete_object($rel); |
| 2647 | 2620 | } |
| 2648 | 2621 | } |
| | 2622 | + |
| 2649 | 2623 | /** |
| | 2624 | + * Delete files in the deleted directory if they are not referenced in the |
| | 2625 | + * filearchive table. This needs to be done in the repo because it needs to |
| | 2626 | + * interleave database locks with file operations, which is potentially a |
| | 2627 | + * remote operation. |
| | 2628 | + * @return FileRepoStatus |
| | 2629 | + */ |
| | 2630 | + function cleanupDeletedBatch( $storageKeys ) { |
| | 2631 | + $auth = new CF_Authentication($this->swiftuser, $this->key, NULL, $this->authurl); |
| | 2632 | + $auth->authenticate(); |
| | 2633 | + $conn = new CF_Connection($auth); |
| | 2634 | + $cont = $this->getZoneContainer( 'deleted' ); |
| | 2635 | + $container = $conn->get_container($cont); |
| | 2636 | + |
| | 2637 | + $dbw = $this->getMasterDB(); |
| | 2638 | + $status = $this->newGood(); |
| | 2639 | + $storageKeys = array_unique($storageKeys); |
| | 2640 | + foreach ( $storageKeys as $key ) { |
| | 2641 | + $hashPath = $this->getDeletedHashPath( $key ); |
| | 2642 | + $rel = "$hashPath$key"; |
| | 2643 | + $dbw->begin(); |
| | 2644 | + $inuse = $dbw->selectField( 'filearchive', '1', |
| | 2645 | + array( 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ), |
| | 2646 | + __METHOD__, array( 'FOR UPDATE' ) ); |
| | 2647 | + if( !$inuse ) { |
| | 2648 | + $sha1 = self::getHashFromKey( $key ); |
| | 2649 | + $ext = substr( $key, strcspn( $key, '.' ) + 1 ); |
| | 2650 | + $ext = File::normalizeExtension($ext); |
| | 2651 | + $inuse = $dbw->selectField( 'oldimage', '1', |
| | 2652 | + array( 'oi_sha1' => $sha1, |
| | 2653 | + 'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ), |
| | 2654 | + $dbw->bitAnd('oi_deleted', File::DELETED_FILE) => File::DELETED_FILE ), |
| | 2655 | + __METHOD__, array( 'FOR UPDATE' ) ); |
| | 2656 | + } |
| | 2657 | + if ( !$inuse ) { |
| | 2658 | + wfDebug( __METHOD__ . ": deleting $key\n" ); |
| | 2659 | + $container->delete_object($rel); |
| | 2660 | + if ( 0 ) { |
| | 2661 | + $status->error( 'undelete-cleanup-error', $rel ); |
| | 2662 | + $status->failCount++; |
| | 2663 | + } |
| | 2664 | + } else { |
| | 2665 | + wfDebug( __METHOD__ . ": $key still in use\n" ); |
| | 2666 | + $status->successCount++; |
| | 2667 | + } |
| | 2668 | + $dbw->commit(); |
| | 2669 | + } |
| | 2670 | + return $status; |
| | 2671 | + } |
| | 2672 | + |
| | 2673 | + /** |
| 2650 | 2674 | * Makes no sense in our context -- don't let anybody call it. |
| 2651 | 2675 | */ |
| 2652 | 2676 | function getZonePath( $zone ) { |
| — | — | @@ -2912,6 +2936,7 @@ |
| 2913 | 2937 | } |
| 2914 | 2938 | } |
| 2915 | 2939 | |
| | 2940 | + |
| 2916 | 2941 | class Junkyjunk { |
| 2917 | 2942 | function __construct( $info ) { |
| 2918 | 2943 | parent::__construct( $info ); |