r96645 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r96644‎ | r96645 | r96646 >
Date:03:51, 9 September 2011
Author:tstarling
Status:ok (Comments)
Tags:todo 
Comment:
* Added a script to reduce disk space on a MySQL parser cache setup such as the one at Wikimedia at the moment, by removing all objects which expire before a given time.
* Fixed unintentional shortcut evaluation in MultiWriteBagOStuff::doWrite(). It would have caused writes to be skipped on the second cache if the first cache failed, now writes should be attempted on all caches.
Modified paths:
  • /trunk/phase3/includes/objectcache/BagOStuff.php (modified) (history)
  • /trunk/phase3/includes/objectcache/MultiWriteBagOStuff.php (modified) (history)
  • /trunk/phase3/includes/objectcache/SqlBagOStuff.php (modified) (history)
  • /trunk/phase3/maintenance/purgeParserCache.php (added) (history)

Diff [purge]

Index: trunk/phase3/maintenance/purgeParserCache.php
@@ -0,0 +1,43 @@
 2+<?php
 3+
 4+require( dirname( __FILE__ ) . '/Maintenance.php' );
 5+
 6+class PurgeParserCache extends Maintenance {
 7+ function __construct() {
 8+ parent::__construct();
 9+ $this->addDescription( "Remove old objects from the parser cache. " .
 10+ "This only works when the parser cache is in an SQL database." );
 11+ $this->addOption( 'expiredate', 'Delete objects expiring before this date.', false, true );
 12+ $this->addOption( 'age',
 13+ 'Delete objects created more than this many seconds ago, assuming $wgParserCacheExpireTime '.
 14+ 'has been consistent.',
 15+ false, true );
 16+ }
 17+
 18+ function execute() {
 19+ $inputDate = $this->getOption( 'expiredate' );
 20+ $inputAge = $this->getOption( 'age' );
 21+ if ( $inputDate !== null ) {
 22+ $date = wfTimestamp( TS_MW, strtotime( $inputDate ) );
 23+ } elseif ( $inputAge !== null ) {
 24+ global $wgParserCacheExpireTime;
 25+ $date = wfTimestamp( TS_MW, time() + $wgParserCacheExpireTime - intval( $inputAge ) );
 26+ } else {
 27+ echo "Must specify either --expiredate or --age\n";
 28+ exit( 1 );
 29+ }
 30+
 31+ $english = Language::factory( 'en' );
 32+ echo "Deleting objects expiring before " . $english->timeanddate( $date ) . "\n";
 33+
 34+ $pc = wfGetParserCacheStorage();
 35+ $success = $pc->deleteObjectsExpiringBefore( $date );
 36+ if ( !$success ) {
 37+ echo "Cannot purge this kind of parser cache.\n";
 38+ exit( 1 );
 39+ }
 40+ echo "Done\n";
 41+ }
 42+}
 43+$maintClass = 'PurgeParserCache';
 44+require_once( RUN_MAINTENANCE_IF_MAIN );
Property changes on: trunk/phase3/maintenance/purgeParserCache.php
___________________________________________________________________
Added: svn:eol-style
145 + native
Index: trunk/phase3/includes/objectcache/BagOStuff.php
@@ -91,6 +91,16 @@
9292 return array();
9393 }
9494
 95+ /**
 96+ * Delete all objects expiring before a certain date.
 97+ *
 98+ * @return true on success, false if unimplemented
 99+ */
 100+ public function deleteObjectsExpiringBefore( $date ) {
 101+ // stub
 102+ return false;
 103+ }
 104+
95105 /* *** Emulated functions *** */
96106
97107 public function add( $key, $value, $exptime = 0 ) {
Index: trunk/phase3/includes/objectcache/SqlBagOStuff.php
@@ -305,21 +305,29 @@
306306 }
307307
308308 public function expireAll() {
 309+ $this->deleteObjectsExpiringBefore( wfTimestampNow() );
 310+ }
 311+
 312+ /**
 313+ * Delete objects from the database which expire before a certain date.
 314+ */
 315+ public function deleteObjectsExpiringBefore( $timestamp ) {
309316 $db = $this->getDB();
310 - $now = $db->timestamp();
 317+ $dbTimestamp = $db->timestamp( $timestamp );
311318
312319 try {
313320 for ( $i = 0; $i < $this->shards; $i++ ) {
314321 $db->begin();
315322 $db->delete(
316323 $this->getTableByShard( $i ),
317 - array( 'exptime < ' . $db->addQuotes( $now ) ),
 324+ array( 'exptime < ' . $db->addQuotes( $dbTimestamp ) ),
318325 __METHOD__ );
319326 $db->commit();
320327 }
321328 } catch ( DBQueryError $e ) {
322329 $this->handleWriteError( $e );
323330 }
 331+ return true;
324332 }
325333
326334 public function deleteAll() {
Index: trunk/phase3/includes/objectcache/MultiWriteBagOStuff.php
@@ -89,9 +89,25 @@
9090 array_shift( $args );
9191
9292 foreach ( $this->caches as $cache ) {
93 - $ret = $ret && call_user_func_array( array( $cache, $method ), $args );
 93+ if ( !call_user_func_array( array( $cache, $method ), $args ) ) {
 94+ $ret = false;
 95+ }
9496 }
9597 return $ret;
9698 }
9799
 100+ /**
 101+ * Delete objects expiring before a certain date.
 102+ *
 103+ * Succeed if any of the child caches succeed.
 104+ */
 105+ public function deleteObjectsExpiringBefore( $date ) {
 106+ $ret = false;
 107+ foreach ( $this->caches as $cache ) {
 108+ if ( $cache->deleteObjectsExpiringBefore( $date ) ) {
 109+ $ret = true;
 110+ }
 111+ }
 112+ return $ret;
 113+ }
98114 }

Follow-up revisions

RevisionCommit summaryAuthorDate
r96849REL1_18: r96509, r96522, r96606, r96643, r96645, r96655, r96659, r96687, r967......reedy15:03, 12 September 2011
r968561.17wmf1: MFT r94212, r96373, r96386, r96389, r96420, r96645...reedy15:35, 12 September 2011

Comments

#Comment by 😂 (talk | contribs)   04:04, 9 September 2011

Looks good. One minor nitpick:

+		$this->addDescription( "Remove old objects from the parser cache. " . 
+			"This only works when the parser cache is in an SQL database." );

The convention is to set $mDescription. To be honest, I'm not entirely sure why Mark introduced it in r71749...nothing uses it in core or extensions until this rev.

#Comment by Midom (talk | contribs)   10:36, 17 September 2011

this does not account that certain objects in cache (ones having magicwords) are not affected by $wgParserCacheExpireTime - so exptime is 'now' and they'd be always purged

#Comment by Tim Starling (talk | contribs)   09:53, 8 December 2011

I think this should be converted from a fixme status to todo tag since I'm not planning on doing it right now.

Status & tagging log