r52024 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r52023‎ | r52024 | r52025 >
Date:07:31, 17 June 2009
Author:tstarling
Status:deferred
Tags:
Comment:
Bug 19240 (bad image list performance regression):
* Don't connect to the commons DB on cache hits, in order to determine the cache key name. Removed remnants of that bright idea from GlobalFunctions.php.
* Fixed total failure of negative caching in checkRedirect() due to memcached stripping trailing spaces from string values. Probably never worked.

Also:
* Respect hasSharedCache in foreign repos. Recently-added code was apparently ignorant of this setting.
* Renamed getMemcKey() to getSharedCacheKey() to make its function more clear, introduced getLocalCacheKey() to do the other thing. Fixed its parameters to be like wfMemcKey() and used it in more places.
* Used getLocalCacheKey() in various places instead of wfMemc(), to avoid having multiple repositories overwrite each others' caches.
* Fixed the BagOStuff bug that the FIXME in LocalRepo::checkRedirect() appears to refer to.
* Removed getMasterDB() and getSlaveDB() from FileRepo, it's incorrect to assume that a repo other than a LocalRepo has an associated database.
* Made FileRepo::invalidateImageRedirect() a stub, to match FileRepo::checkRedirect(). Moved the functionality to LocalRepo where checkRedirect() is concretely implemented.
Modified paths:
  • /trunk/phase3/RELEASE-NOTES (modified) (history)
  • /trunk/phase3/includes/BagOStuff.php (modified) (history)
  • /trunk/phase3/includes/GlobalFunctions.php (modified) (history)
  • /trunk/phase3/includes/filerepo/File.php (modified) (history)
  • /trunk/phase3/includes/filerepo/FileRepo.php (modified) (history)
  • /trunk/phase3/includes/filerepo/ForeignAPIFile.php (modified) (history)
  • /trunk/phase3/includes/filerepo/ForeignAPIRepo.php (modified) (history)
  • /trunk/phase3/includes/filerepo/ForeignDBFile.php (modified) (history)
  • /trunk/phase3/includes/filerepo/ForeignDBRepo.php (modified) (history)
  • /trunk/phase3/includes/filerepo/ForeignDBViaLBRepo.php (modified) (history)
  • /trunk/phase3/includes/filerepo/LocalFile.php (modified) (history)
  • /trunk/phase3/includes/filerepo/LocalRepo.php (modified) (history)

Diff [purge]

Index: trunk/phase3/includes/BagOStuff.php
@@ -191,7 +191,7 @@
192192 }
193193
194194 function get($key) {
195 - if(!$this->bag[$key])
 195+ if( !isset( $this->bag[$key] ) )
196196 return false;
197197 if($this->_expire($key))
198198 return false;
@@ -203,7 +203,7 @@
204204 }
205205
206206 function delete($key,$time=0) {
207 - if(!$this->bag[$key])
 207+ if( !isset( $this->bag[$key] ) )
208208 return false;
209209 unset($this->bag[$key]);
210210 return true;
Index: trunk/phase3/includes/GlobalFunctions.php
@@ -2809,16 +2809,12 @@
28102810 * Get an ASCII string identifying this wiki
28112811 * This is used as a prefix in memcached keys
28122812 */
2813 -function wfWikiID( $db = null ) {
2814 - if( $db instanceof Database ) {
2815 - return $db->getWikiID();
2816 - } else {
 2813+function wfWikiID() {
28172814 global $wgDBprefix, $wgDBname;
2818 - if ( $wgDBprefix ) {
2819 - return "$wgDBname-$wgDBprefix";
2820 - } else {
2821 - return $wgDBname;
2822 - }
 2815+ if ( $wgDBprefix ) {
 2816+ return "$wgDBname-$wgDBprefix";
 2817+ } else {
 2818+ return $wgDBname;
28232819 }
28242820 }
28252821
Index: trunk/phase3/includes/filerepo/LocalFile.php
@@ -133,11 +133,12 @@
134134 }
135135
136136 /**
137 - * Get the memcached key
 137+ * Get the memcached key for the main data for this file, or false if
 138+ * there is no access to the shared cache.
138139 */
139140 function getCacheKey() {
140141 $hashedName = md5($this->getName());
141 - return wfMemcKey( 'file', $hashedName );
 142+ return $this->repo->getSharedCacheKey( 'file', $hashedName );
142143 }
143144
144145 /**
@@ -590,8 +591,10 @@
591592 function purgeHistory() {
592593 global $wgMemc;
593594 $hashedName = md5($this->getName());
594 - $oldKey = wfMemcKey( 'oldfile', $hashedName );
595 - $wgMemc->delete( $oldKey );
 595+ $oldKey = $this->getSharedCacheKey( 'oldfile', $hashedName );
 596+ if ( $oldKey ) {
 597+ $wgMemc->delete( $oldKey );
 598+ }
596599 }
597600
598601 /**
Index: trunk/phase3/includes/filerepo/ForeignDBFile.php
@@ -19,16 +19,6 @@
2020 return $file;
2121 }
2222
23 - function getCacheKey() {
24 - if ( $this->repo->hasSharedCache() ) {
25 - $hashedName = md5($this->name);
26 - return wfForeignMemcKey( $this->repo->dbName, $this->repo->tablePrefix,
27 - 'file', $hashedName );
28 - } else {
29 - return false;
30 - }
31 - }
32 -
3323 function publish( $srcPath, $flags = 0 ) {
3424 $this->readOnlyError();
3525 }
Index: trunk/phase3/includes/filerepo/ForeignDBViaLBRepo.php
@@ -27,6 +27,21 @@
2828 return $this->hasSharedCache;
2929 }
3030
 31+ /**
 32+ * Get a key on the primary cache for this repository.
 33+ * Returns false if the repository's cache is not accessible at this site.
 34+ * The parameters are the parts of the key, as for wfMemcKey().
 35+ */
 36+ function getSharedCacheKey( /*...*/ ) {
 37+ if ( $this->hasSharedCache() ) {
 38+ $args = func_get_args();
 39+ array_unshift( $args, $this->wiki );
 40+ return implode( ':', $args );
 41+ } else {
 42+ return false;
 43+ }
 44+ }
 45+
3146 function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
3247 throw new MWException( get_class($this) . ': write operations are not supported' );
3348 }
Index: trunk/phase3/includes/filerepo/FileRepo.php
@@ -545,15 +545,19 @@
546546
547547 /**
548548 * Invalidates image redirect cache related to that image
549 - *
 549+ * Doesn't do anything for repositories that don't support image redirects.
 550+ *
 551+ * STUB
550552 * @param Title $title Title of image
551553 */
552 - function invalidateImageRedirect( $title ) {
553 - global $wgMemc;
554 - $memcKey = $this->getMemcKey( "image_redirect:" . md5( $title->getPrefixedDBkey() ) );
555 - $wgMemc->delete( $memcKey );
556 - }
 554+ function invalidateImageRedirect( $title ) {}
557555
 556+ /**
 557+ * Get an array or iterator of file objects for files that have a given
 558+ * SHA-1 content hash.
 559+ *
 560+ * STUB
 561+ */
558562 function findBySha1( $hash ) {
559563 return array();
560564 }
@@ -574,16 +578,25 @@
575579 return wfMsg( 'shared-repo' );
576580 }
577581
578 - function getSlaveDB() {
579 - return wfGetDB( DB_SLAVE );
 582+ /**
 583+ * Get a key on the primary cache for this repository.
 584+ * Returns false if the repository's cache is not accessible at this site.
 585+ * The parameters are the parts of the key, as for wfMemcKey().
 586+ *
 587+ * STUB
 588+ */
 589+ function getSharedCacheKey( /*...*/ ) {
 590+ return false;
580591 }
581592
582 - function getMasterDB() {
583 - return wfGetDB( DB_MASTER );
 593+ /**
 594+ * Get a key for this repo in the local cache domain. These cache keys are
 595+ * not shared with remote instances of the repo.
 596+ * The parameters are the parts of the key, as for wfMemcKey().
 597+ */
 598+ function getLocalCacheKey( /*...*/ ) {
 599+ $args = func_get_args();
 600+ array_unshift( $args, 'filerepo', $this->getName() );
 601+ return call_user_func_array( 'wfMemcKey', $args );
584602 }
585 -
586 - function getMemcKey( $key ) {
587 - return wfWikiID( $this->getSlaveDB() ) . ":{$key}";
588 - }
589 -
590603 }
Index: trunk/phase3/includes/filerepo/ForeignAPIRepo.php
@@ -114,7 +114,7 @@
115115 'action' => 'query' ) ) );
116116
117117 if( !isset( $this->mQueryCache[$url] ) ) {
118 - $key = wfMemcKey( 'ForeignAPIRepo', 'Metadata', md5( $url ) );
 118+ $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'Metadata', md5( $url ) );
119119 $data = $wgMemc->get( $key );
120120 if( !$data ) {
121121 $data = Http::get( $url );
@@ -176,7 +176,7 @@
177177 return $this->getThumbUrl( $name, $width, $height );
178178 }
179179
180 - $key = wfMemcKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
 180+ $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
181181 if ( $thumbUrl = $wgMemc->get($key) ) {
182182 wfDebug("Got thumb from local cache. $thumbUrl \n");
183183 return $thumbUrl;
Index: trunk/phase3/includes/filerepo/File.php
@@ -1077,7 +1077,7 @@
10781078 if ( $renderUrl ) {
10791079 if ( $this->repo->descriptionCacheExpiry > 0 ) {
10801080 wfDebug("Attempting to get the description from cache...");
1081 - $key = wfMemcKey( 'RemoteFileDescription', 'url', $wgContLang->getCode(),
 1081+ $key = $this->getLocalCacheKey( 'RemoteFileDescription', 'url', $wgContLang->getCode(),
10821082 $this->getName() );
10831083 $obj = $wgMemc->get($key);
10841084 if ($obj) {
Index: trunk/phase3/includes/filerepo/LocalRepo.php
@@ -83,17 +83,24 @@
8484 $title = Title::makeTitle( NS_FILE, $title->getText() );
8585 }
8686
87 - $memcKey = $this->getMemcKey( "image_redirect:" . md5( $title->getPrefixedDBkey() ) );
 87+ $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getPrefixedDBkey() ) );
 88+ if ( $memcKey === false ) {
 89+ $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getPrefixedDBkey() ) );
 90+ $expiry = 300; // no invalidation, 5 minutes
 91+ } else {
 92+ $expiry = 86400; // has invalidation, 1 day
 93+ }
8894 $cachedValue = $wgMemc->get( $memcKey );
89 - if( $cachedValue ) {
90 - return Title::newFromDbKey( $cachedValue );
91 - } elseif( $cachedValue == ' ' ) { # FIXME: ugly hack, but BagOStuff caching seems to be weird and return false if !cachedValue, not only if it doesn't exist
 95+ if ( $cachedValue === ' ' || $cachedValue === '' ) {
 96+ // Does not exist
9297 return false;
93 - }
 98+ } elseif ( strval( $cachedValue ) !== '' ) {
 99+ return Title::newFromText( $cachedValue );
 100+ } // else $cachedValue is false or null: cache miss
94101
95102 $id = $this->getArticleID( $title );
96103 if( !$id ) {
97 - $wgMemc->set( $memcKey, " ", 9000 );
 104+ $wgMemc->set( $memcKey, " ", $expiry );
98105 return false;
99106 }
100107 $dbr = $this->getSlaveDB();
@@ -104,12 +111,14 @@
105112 __METHOD__
106113 );
107114
108 - if( $row ) $targetTitle = Title::makeTitle( $row->rd_namespace, $row->rd_title );
109 - $wgMemc->set( $memcKey, ($row ? $targetTitle->getPrefixedDBkey() : " "), 9000 );
110 - if( !$row ) {
 115+ if( $row ) {
 116+ $targetTitle = Title::makeTitle( $row->rd_namespace, $row->rd_title );
 117+ $wgMemc->set( $memcKey, $targetTitle->getPrefixedDBkey(), $expiry );
 118+ return $targetTitle;
 119+ } else {
 120+ $wgMemc->set( $memcKey, '', $expiry );
111121 return false;
112122 }
113 - return $targetTitle;
114123 }
115124
116125
@@ -134,8 +143,10 @@
135144 return $id;
136145 }
137146
138 -
139 -
 147+ /**
 148+ * Get an array or iterator of file objects for files that have a given
 149+ * SHA-1 content hash.
 150+ */
140151 function findBySha1( $hash ) {
141152 $dbr = $this->getSlaveDB();
142153 $res = $dbr->select(
@@ -174,4 +185,42 @@
175186 $res->free();
176187 return $result;
177188 }
 189+
 190+ /**
 191+ * Get a connection to the slave DB
 192+ */
 193+ function getSlaveDB() {
 194+ return wfGetDB( DB_SLAVE );
 195+ }
 196+
 197+ /**
 198+ * Get a connection to the master DB
 199+ */
 200+ function getMasterDB() {
 201+ return wfGetDB( DB_MASTER );
 202+ }
 203+
 204+ /**
 205+ * Get a key on the primary cache for this repository.
 206+ * Returns false if the repository's cache is not accessible at this site.
 207+ * The parameters are the parts of the key, as for wfMemcKey().
 208+ */
 209+ function getSharedCacheKey( /*...*/ ) {
 210+ $args = func_get_args();
 211+ return call_user_func_array( 'wfMemcKey', $args );
 212+ }
 213+
 214+ /**
 215+ * Invalidates image redirect cache related to that image
 216+ *
 217+ * @param Title $title Title of image
 218+ */
 219+ function invalidateImageRedirect( $title ) {
 220+ global $wgMemc;
 221+ $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getPrefixedDBkey() ) );
 222+ if ( $memcKey ) {
 223+ $wgMemc->delete( $memcKey );
 224+ }
 225+ }
178226 }
 227+
Index: trunk/phase3/includes/filerepo/ForeignAPIFile.php
@@ -162,13 +162,13 @@
163163 function purgeDescriptionPage() {
164164 global $wgMemc, $wgContLang;
165165 $url = $this->repo->getDescriptionRenderUrl( $this->getName(), $wgContLang->getCode() );
166 - $key = wfMemcKey( 'RemoteFileDescription', 'url', md5($url) );
 166+ $key = $this->repo->getLocalCacheKey( 'RemoteFileDescription', 'url', md5($url) );
167167 $wgMemc->delete( $key );
168168 }
169169
170170 function purgeThumbnails() {
171171 global $wgMemc;
172 - $key = wfMemcKey( 'ForeignAPIRepo', 'ThumbUrl', $this->getName() );
 172+ $key = $this->repo->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $this->getName() );
173173 $wgMemc->delete( $key );
174174 $files = $this->getThumbnails();
175175 $dir = $this->getThumbPath( $this->getName() );
Index: trunk/phase3/includes/filerepo/ForeignDBRepo.php
@@ -44,6 +44,21 @@
4545 return $this->hasSharedCache;
4646 }
4747
 48+ /**
 49+ * Get a key on the primary cache for this repository.
 50+ * Returns false if the repository's cache is not accessible at this site.
 51+ * The parameters are the parts of the key, as for wfMemcKey().
 52+ */
 53+ function getSharedCacheKey( /*...*/ ) {
 54+ if ( $this->hasSharedCache() ) {
 55+ $args = func_get_args();
 56+ array_unshift( $args, $this->dbName, $this->tablePrefix );
 57+ return call_user_func_array( 'wfForeignMemcKey', $args );
 58+ } else {
 59+ return false;
 60+ }
 61+ }
 62+
4863 function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
4964 throw new MWException( get_class($this) . ': write operations are not supported' );
5065 }
Index: trunk/phase3/RELEASE-NOTES
@@ -185,6 +185,7 @@
186186 * (bug 18173) MediaWiki now fails when unable to determine a client IP
187187 * (bug 19170) Special:Version should follow the content language direction
188188 * (bug 19160) maintenance/purgeOldText.inc is now compatible with PostgreSQL
 189+* Fixed performance regression in "bad image list" feature
189190
190191 == API changes in 1.16 ==
191192

Follow-up revisions

RevisionCommit summaryAuthorDate
r52028Fix incorrect method call from r52024.tstarling08:18, 17 June 2009
r52033Fix another screwup from r52024, same issue as r52028.tstarling09:39, 17 June 2009

Status & tagging log