r47065 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r47064‎ | r47065 | r47066 >
Date:23:18, 9 February 2009
Author:werdna
Status:deferred
Tags:
Comment:
Apply changes made live on Wikimedia cluster related to preprocessor caching to subversion. Patch worked on by Tim Starling and myself.

Changes from the original patch (r46936):
* Add versioning to the cache, so the cache can be purged.
* Only cache preprocessor output for input of over a certain amount (default of 1000 bytes).
Modified paths:
  • /trunk/phase3/includes/DefaultSettings.php (modified) (history)
  • /trunk/phase3/includes/parser/Preprocessor_DOM.php (modified) (history)
  • /trunk/phase3/includes/parser/Preprocessor_Hash.php (modified) (history)

Diff [purge]

Index: trunk/phase3/includes/parser/Preprocessor_Hash.php
——@@ -8,6 +8,8 @@
99 */
1010 class Preprocessor_Hash implements Preprocessor {
1111 var $parser;
 12+
 13+ const CACHE_VERSION = 1;
1214
1315 function __construct( $parser ) {
1416 $this->parser = $parser;
——@@ -45,14 +47,30 @@
4648 */
4749 function preprocessToObj( $text, $flags = 0 ) {
4850 wfProfileIn( __METHOD__ );
 51+
 52+
 53+ // Check cache.
 54+ global $wgMemc, $wgPreprocessorCacheThreshold;
4955
50 - global $wgMemc;
51 - $cacheKey = wfMemckey( 'preprocessor-hash', md5( $text ), $flags );
52 -
53 - if ( $obj = $wgMemc->get( $cacheKey ) ) {
54 - wfDebugLog( "Preprocessor", "Got preprocessor_hash output from cache" );
55 - wfProfileOut( __METHOD__ );
56 - return $obj;
 56+ $cacheable = strlen( $text ) > $wgPreprocessorCacheThreshold;
 57+ if ( $cacheable ) {
 58+ wfProfileIn( __METHOD__.'-cacheable' );
 59+
 60+ $cacheKey = wfMemcKey( 'preprocess-hash', md5($text), $flags );
 61+ $cacheValue = $wgMemc->get( $cacheKey );
 62+ if ( $cacheValue ) {
 63+ $version = substr( $cacheValue, 0, 8 );
 64+ if ( intval( $version ) == self::CACHE_VERSION ) {
 65+ $hash = unserialize( substr( $cacheValue, 8 ) );
 66+ // From the cache
 67+ wfDebugLog( "Preprocessor",
 68+ "Loaded preprocessor hash from memcached (key $cacheKey)" );
 69+ wfProfileOut( __METHOD__.'-cacheable' );
 70+ wfProfileOut( __METHOD__ );
 71+ return $hash;
 72+ }
 73+ }
 74+ wfProfileIn( __METHOD__.'-cache-miss' );
5775 }
5876
5977 $rules = array(
——@@ -626,10 +644,17 @@
627645 $rootNode = new PPNode_Hash_Tree( 'root' );
628646 $rootNode->firstChild = $stack->rootAccum->firstNode;
629647 $rootNode->lastChild = $stack->rootAccum->lastNode;
630 - wfProfileOut( __METHOD__ );
631648
632 - $wgMemc->set( $cacheKey, $rootNode, 86400 );
 649+ // Cache
 650+ if ($cacheable) {
 651+ $cacheValue = sprintf( "%08d", self::CACHE_VERSION ) . serialize( $rootNode );;
 652+ $wgMemc->set( $cacheKey, $cacheValue, 86400 );
 653+ wfProfileOut( __METHOD__.'-cache-miss' );
 654+ wfProfileOut( __METHOD__.'-cacheable' );
 655+ wfDebugLog( "Preprocessor", "Saved preprocessor Hash to memcached (key $cacheKey)" );
 656+ }
633657
 658+ wfProfileOut( __METHOD__ );
634659 return $rootNode;
635660 }
636661 }
Index: trunk/phase3/includes/parser/Preprocessor_DOM.php
——@@ -6,6 +6,8 @@
77 class Preprocessor_DOM implements Preprocessor {
88 var $parser, $memoryLimit;
99
 10+ const CACHE_VERSION = 1;
 11+
1012 function __construct( $parser ) {
1113 $this->parser = $parser;
1214 $mem = ini_get( 'memory_limit' );
——@@ -63,19 +65,37 @@
6466 */
6567 function preprocessToObj( $text, $flags = 0 ) {
6668 wfProfileIn( __METHOD__ );
 69+ global $wgMemc, $wgPreprocessorCacheThreshold;
6770
68 - global $wgMemc;
69 - $cacheKey = wfMemcKey( 'preprocess-xml', md5($text), $flags );
70 -
71 - if ( $xml = $wgMemc->get( $cacheKey ) ) {
72 - // From the cache
73 - wfDebugLog( "Preprocessor", "Loaded preprocessor XML from memcached (key $cacheKey)" );
74 - } else {
75 - $xml = $this->preprocessToXml( $text, $flags );
76 - $wgMemc->set( $cacheKey, $xml, 86400 );
77 - wfDebugLog( "Preprocessor", "Saved preprocessor XML to memcached (key $cacheKey)" );
 71+ $xml = false;
 72+ $cacheable = strlen( $text ) > $wgPreprocessorCacheThreshold;
 73+ if ( $cacheable ) {
 74+ wfProfileIn( __METHOD__.'-cacheable' );
 75+
 76+ $cacheKey = wfMemcKey( 'preprocess-xml', md5($text), $flags );
 77+ $cacheValue = $wgMemc->get( $cacheKey );
 78+ if ( $cacheValue ) {
 79+ $version = substr( $cacheValue, 0, 8 );
 80+ if ( intval( $version ) == self::CACHE_VERSION ) {
 81+ $xml = substr( $cacheValue, 8 );
 82+ // From the cache
 83+ wfDebugLog( "Preprocessor", "Loaded preprocessor XML from memcached (key $cacheKey)" );
 84+ }
 85+ }
7886 }
79 -
 87+ if ( $xml === false ) {
 88+ if ( $cacheable ) {
 89+ wfProfileIn( __METHOD__.'-cache-miss' );
 90+ $xml = $this->preprocessToXml( $text, $flags );
 91+ $cacheValue = sprintf( "%08d", self::CACHE_VERSION ) . $xml;
 92+ $wgMemc->set( $cacheKey, $cacheValue, 86400 );
 93+ wfProfileOut( __METHOD__.'-cache-miss' );
 94+ wfDebugLog( "Preprocessor", "Saved preprocessor XML to memcached (key $cacheKey)" );
 95+ } else {
 96+ $xml = $this->preprocessToXml( $text, $flags );
 97+ }
 98+
 99+ }
80100 wfProfileIn( __METHOD__.'-loadXML' );
81101 $dom = new DOMDocument;
82102 wfSuppressWarnings();
——@@ -91,6 +111,9 @@
92112 }
93113 $obj = new PPNode_DOM( $dom->documentElement );
94114 wfProfileOut( __METHOD__.'-loadXML' );
 115+ if ( $cacheable ) {
 116+ wfProfileOut( __METHOD__.'-cacheable' );
 117+ }
95118 wfProfileOut( __METHOD__ );
96119 return $obj;
97120 }
Index: trunk/phase3/includes/DefaultSettings.php
——@@ -3708,3 +3708,8 @@
37093709 * false = use Go button & Advanced search link
37103710 */
37113711 $wgUseTwoButtonsSearchForm = true;
 3712+
 3713+/**
 3714+ * Preprocessor caching threshold
 3715+ */
 3716+$wgPreprocessorCacheThreshold = 1000;
\ No newline at end of file

Past revisions this follows-up on

RevisionCommit summaryAuthorDate
r46936Cache preprocessor output in memcached.werdna20:27, 6 February 2009

Status & tagging log