| Index: trunk/phase3/includes/Article.php |
| — | — | @@ -1677,8 +1677,8 @@ |
| 1678 | 1678 | } |
| 1679 | 1679 | |
| 1680 | 1680 | # Invalidate cache of this article and all pages using this article |
| 1681 | | - # as a template. Partly deferred. Leave templatelinks for editUpdates(). |
| 1682 | | - Article::onArticleEdit( $this->mTitle, 'skiptransclusions' ); |
| | 1681 | + # as a template. Partly deferred. |
| | 1682 | + Article::onArticleEdit( $this->mTitle ); |
| 1683 | 1683 | # Update links tables, site stats, etc. |
| 1684 | 1684 | $this->editUpdates( $text, $summary, $isminor, $now, $revisionId, $changed ); |
| 1685 | 1685 | } else { |
| — | — | @@ -2893,8 +2893,7 @@ |
| 2894 | 2894 | } |
| 2895 | 2895 | |
| 2896 | 2896 | # Update the links tables |
| 2897 | | - $u = new LinksUpdate( $this->mTitle, $editInfo->output, false ); |
| 2898 | | - $u->setRecursiveTouch( $changed ); // refresh/invalidate including pages too |
| | 2897 | + $u = new LinksUpdate( $this->mTitle, $editInfo->output ); |
| 2899 | 2898 | $u->doUpdate(); |
| 2900 | 2899 | |
| 2901 | 2900 | wfRunHooks( 'ArticleEditUpdates', array( &$this, &$editInfo, $changed ) ); |
| — | — | @@ -3288,12 +3287,11 @@ |
| 3289 | 3288 | /** |
| 3290 | 3289 | * Purge caches on page update etc |
| 3291 | 3290 | */ |
| 3292 | | - public static function onArticleEdit( $title, $transclusions = 'transclusions' ) { |
| | 3291 | + public static function onArticleEdit( $title, $flags = '' ) { |
| 3293 | 3292 | global $wgDeferredUpdateList; |
| 3294 | 3293 | |
| 3295 | 3294 | // Invalidate caches of articles which include this page |
| 3296 | | - if( $transclusions !== 'skiptransclusions' ) |
| 3297 | | - $wgDeferredUpdateList[] = new HTMLCacheUpdate( $title, 'templatelinks' ); |
| | 3295 | + $wgDeferredUpdateList[] = new HTMLCacheUpdate( $title, 'templatelinks' ); |
| 3298 | 3296 | |
| 3299 | 3297 | // Invalidate the caches of all pages which redirect here |
| 3300 | 3298 | $wgDeferredUpdateList[] = new HTMLCacheUpdate( $title, 'redirect' ); |
| Index: trunk/phase3/includes/RefreshLinksJob.php |
| — | — | @@ -82,35 +82,21 @@ |
| 83 | 83 | wfProfileOut( __METHOD__ ); |
| 84 | 84 | return false; |
| 85 | 85 | } |
| 86 | | - $start = intval($this->params['start']); |
| 87 | | - $end = intval($this->params['end']); |
| | 86 | + $titles = $this->title->getBacklinkCache()->getLinks( |
| | 87 | + 'templatelinks', $this->params['start'], $this->params['end']); |
| 88 | 88 | |
| 89 | | - $dbr = wfGetDB( DB_SLAVE ); |
| 90 | | - $res = $dbr->select( array( 'templatelinks', 'page' ), |
| 91 | | - array( 'page_namespace', 'page_title' ), |
| 92 | | - array( |
| 93 | | - 'page_id=tl_from', |
| 94 | | - "tl_from >= '$start'", |
| 95 | | - "tl_from <= '$end'", |
| 96 | | - 'tl_namespace' => $this->title->getNamespace(), |
| 97 | | - 'tl_title' => $this->title->getDBkey() |
| 98 | | - ), __METHOD__ |
| 99 | | - ); |
| 100 | | - |
| 101 | 89 | # Not suitable for page load triggered job running! |
| 102 | 90 | # Gracefully switch to refreshLinks jobs if this happens. |
| 103 | 91 | if( php_sapi_name() != 'cli' ) { |
| 104 | 92 | $jobs = array(); |
| 105 | | - while( $row = $dbr->fetchObject( $res ) ) { |
| 106 | | - $title = Title::makeTitle( $row->page_namespace, $row->page_title ); |
| | 93 | + foreach ( $titles as $title ) { |
| 107 | 94 | $jobs[] = new RefreshLinksJob( $title, '' ); |
| 108 | 95 | } |
| 109 | 96 | Job::batchInsert( $jobs ); |
| 110 | 97 | return true; |
| 111 | 98 | } |
| 112 | 99 | # Re-parse each page that transcludes this page and update their tracking links... |
| 113 | | - while( $row = $dbr->fetchObject( $res ) ) { |
| 114 | | - $title = Title::makeTitle( $row->page_namespace, $row->page_title ); |
| | 100 | + foreach ( $titles as $title ) { |
| 115 | 101 | $revision = Revision::newFromTitle( $title ); |
| 116 | 102 | if ( !$revision ) { |
| 117 | 103 | $this->error = 'refreshLinks: Article not found "' . $title->getPrefixedDBkey() . '"'; |
| Index: trunk/phase3/includes/LinksUpdate.php |
| — | — | @@ -20,8 +20,7 @@ |
| 21 | 21 | $mProperties, //!< Map of arbitrary name to value |
| 22 | 22 | $mDb, //!< Database connection reference |
| 23 | 23 | $mOptions, //!< SELECT options to be used (array) |
| 24 | | - $mRecursive, //!< Whether to queue jobs for recursive updates |
| 25 | | - $mTouchTmplLinks; //!< Whether to queue HTMLCacheUpdate jobs IF recursive |
| | 24 | + $mRecursive; //!< Whether to queue jobs for recursive updates |
| 26 | 25 | /**@}}*/ |
| 27 | 26 | |
| 28 | 27 | /** |
| — | — | @@ -72,15 +71,6 @@ |
| 73 | 72 | |
| 74 | 73 | wfRunHooks( 'LinksUpdateConstructed', array( &$this ) ); |
| 75 | 74 | } |
| 76 | | - |
| 77 | | - /** |
| 78 | | - * Invalidate HTML cache of pages that include this page? |
| 79 | | - */ |
| 80 | | - public function setRecursiveTouch( $val ) { |
| 81 | | - $this->mTouchTmplLinks = (bool)$val; |
| 82 | | - if( $val ) // Cannot invalidate without queueRecursiveJobs() |
| 83 | | - $this->mRecursive = true; |
| 84 | | - } |
| 85 | 75 | |
| 86 | 76 | /** |
| 87 | 77 | * Update link tables with outgoing links from an updated article |
| — | — | @@ -95,7 +85,6 @@ |
| 96 | 86 | $this->doIncrementalUpdate(); |
| 97 | 87 | } |
| 98 | 88 | wfRunHooks( 'LinksUpdateComplete', array( &$this ) ); |
| 99 | | - |
| 100 | 89 | } |
| 101 | 90 | |
| 102 | 91 | protected function doIncrementalUpdate() { |
| — | — | @@ -207,49 +196,17 @@ |
| 208 | 197 | global $wgUpdateRowsPerJob; |
| 209 | 198 | wfProfileIn( __METHOD__ ); |
| 210 | 199 | |
| 211 | | - $dbr = wfGetDB( DB_SLAVE ); |
| 212 | | - $res = $dbr->select( 'templatelinks', |
| 213 | | - array( 'tl_from' ), |
| 214 | | - array( |
| 215 | | - 'tl_namespace' => $this->mTitle->getNamespace(), |
| 216 | | - 'tl_title' => $this->mTitle->getDBkey() |
| 217 | | - ), __METHOD__ |
| 218 | | - ); |
| 219 | | - |
| 220 | | - $numRows = $res->numRows(); |
| 221 | | - if( !$numRows ) { |
| 222 | | - wfProfileOut( __METHOD__ ); |
| 223 | | - return; // nothing to do |
| 224 | | - } |
| 225 | | - $numBatches = ceil( $numRows / $wgUpdateRowsPerJob ); |
| 226 | | - $realBatchSize = $numRows / $numBatches; |
| 227 | | - $start = false; |
| | 200 | + $cache = $this->mTitle->getBacklinkCache(); |
| | 201 | + $batches = $cache->partition( 'templatelinks', $wgUpdateRowsPerJob ); |
| 228 | 202 | $jobs = array(); |
| 229 | | - do { |
| 230 | | - for( $i = 0; $i <= $realBatchSize - 1; $i++ ) { |
| 231 | | - $row = $res->fetchRow(); |
| 232 | | - if( $row ) { |
| 233 | | - $id = $row[0]; |
| 234 | | - } else { |
| 235 | | - $id = false; |
| 236 | | - break; |
| 237 | | - } |
| 238 | | - } |
| | 203 | + foreach ( $batches as $batch ) { |
| | 204 | + list( $start, $end ) = $batch; |
| 239 | 205 | $params = array( |
| 240 | 206 | 'start' => $start, |
| 241 | | - 'end' => ( $id !== false ? $id - 1 : false ), |
| | 207 | + 'end' => $end, |
| 242 | 208 | ); |
| 243 | 209 | $jobs[] = new RefreshLinksJob2( $this->mTitle, $params ); |
| 244 | | - # Hit page caches while we're at it if set to do so... |
| 245 | | - if( $this->mTouchTmplLinks ) { |
| 246 | | - $params['table'] = 'templatelinks'; |
| 247 | | - $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); |
| 248 | | - } |
| 249 | | - $start = $id; |
| 250 | | - } while ( $start ); |
| 251 | | - |
| 252 | | - $dbr->freeResult( $res ); |
| 253 | | - |
| | 210 | + } |
| 254 | 211 | Job::batchInsert( $jobs ); |
| 255 | 212 | |
| 256 | 213 | wfProfileOut( __METHOD__ ); |
| Index: trunk/phase3/includes/AutoLoader.php |
| — | — | @@ -19,6 +19,7 @@ |
| 20 | 20 | 'AuthPlugin' => 'includes/AuthPlugin.php', |
| 21 | 21 | 'AuthPluginUser' => 'includes/AuthPlugin.php', |
| 22 | 22 | 'Autopromote' => 'includes/Autopromote.php', |
| | 23 | + 'BacklinkCache' => 'includes/BacklinkCache.php', |
| 23 | 24 | 'BagOStuff' => 'includes/BagOStuff.php', |
| 24 | 25 | 'Block' => 'includes/Block.php', |
| 25 | 26 | 'CacheDependency' => 'includes/CacheDependency.php', |
| Index: trunk/phase3/includes/BacklinkCache.php |
| — | — | @@ -0,0 +1,234 @@ |
| | 2 | +<?php |
| | 3 | + |
| | 4 | +/** |
| | 5 | + * Class for fetching backlink lists, approximate backlink counts and partitions. |
| | 6 | + * Instances of this class should typically be fetched with $title->getBacklinkCache(). |
| | 7 | + * |
| | 8 | + * Ideally you should only get your backlinks from here when you think there is some |
| | 9 | + * advantage in caching them. Otherwise it's just a waste of memory. |
| | 10 | + */ |
| | 11 | +class BacklinkCache { |
| | 12 | + var $partitionCache = array(); |
| | 13 | + var $fullResultCache = array(); |
| | 14 | + var $title; |
| | 15 | + var $db; |
| | 16 | + |
| | 17 | + const CACHE_EXPIRY = 3600; |
| | 18 | + |
| | 19 | + /** |
| | 20 | + * Create a new BacklinkCache |
| | 21 | + */ |
| | 22 | + function __construct( $title ) { |
| | 23 | + $this->title = $title; |
| | 24 | + } |
| | 25 | + |
| | 26 | + /** |
| | 27 | + * Clear locally stored data |
| | 28 | + */ |
| | 29 | + function clear() { |
| | 30 | + $this->partitionCache = array(); |
| | 31 | + $this->fullResultCache = array(); |
| | 32 | + unset( $this->db ); |
| | 33 | + } |
| | 34 | + |
| | 35 | + /** |
| | 36 | + * Set the Database object to use |
| | 37 | + */ |
| | 38 | + public function setDB( $db ) { |
| | 39 | + $this->db = $db; |
| | 40 | + } |
| | 41 | + |
| | 42 | + protected function getDB() { |
| | 43 | + if ( !isset( $this->db ) ) { |
| | 44 | + $this->db = wfGetDB( DB_SLAVE ); |
| | 45 | + } |
| | 46 | + return $this->db; |
| | 47 | + } |
| | 48 | + |
| | 49 | + /** |
| | 50 | + * Get the backlinks for a given table. Cached in process memory only. |
| | 51 | + * @param string $table |
| | 52 | + * @return TitleArray |
| | 53 | + */ |
| | 54 | + public function getLinks( $table, $startId = false, $endId = false ) { |
| | 55 | + wfProfileIn( __METHOD__ ); |
| | 56 | + |
| | 57 | + if ( $startId || $endId ) { |
| | 58 | + // Partial range, not cached |
| | 59 | + wfDebug( __METHOD__.": from DB (uncacheable range)\n" ); |
| | 60 | + $conds = $this->getConditions( $table ); |
| | 61 | + // Use the from field in the condition rather than the joined page_id, |
| | 62 | + // because databases are stupid and don't necessarily propagate indexes. |
| | 63 | + $fromField = $this->getPrefix( $table ) . '_from'; |
| | 64 | + if ( $startId ) { |
| | 65 | + $conds[] = "$fromField >= " . intval( $startId ); |
| | 66 | + } |
| | 67 | + if ( $endId ) { |
| | 68 | + $conds[] = "$fromField <= " . intval( $endId ); |
| | 69 | + } |
| | 70 | + $res = $this->getDB()->select( |
| | 71 | + array( 'page', $table ), |
| | 72 | + array( 'page_namespace', 'page_title', 'page_id' ), |
| | 73 | + $conds, |
| | 74 | + __METHOD__ ); |
| | 75 | + $ta = TitleArray::newFromResult( $res ); |
| | 76 | + wfProfileOut( __METHOD__ ); |
| | 77 | + return $ta; |
| | 78 | + } |
| | 79 | + |
| | 80 | + if ( !isset( $this->fullResultCache[$table] ) ) { |
| | 81 | + wfDebug( __METHOD__.": from DB\n" ); |
| | 82 | + $res = $this->getDB()->select( |
| | 83 | + array( 'page', $table ), |
| | 84 | + array( 'page_namespace', 'page_title', 'page_id' ), |
| | 85 | + $this->getConditions( $table ), |
| | 86 | + __METHOD__ ); |
| | 87 | + $this->fullResultCache[$table] = $res; |
| | 88 | + } |
| | 89 | + $ta = TitleArray::newFromResult( $this->fullResultCache[$table] ); |
| | 90 | + wfProfileOut( __METHOD__ ); |
| | 91 | + return $ta; |
| | 92 | + } |
| | 93 | + |
| | 94 | + /** |
| | 95 | + * Get the field name prefix for a given table |
| | 96 | + */ |
| | 97 | + protected function getPrefix( $table ) { |
| | 98 | + static $prefixes = array( |
| | 99 | + 'pagelinks' => 'pl', |
| | 100 | + 'imagelinks' => 'il', |
| | 101 | + 'categorylinks' => 'cl', |
| | 102 | + 'templatelinks' => 'tl', |
| | 103 | + 'redirect' => 'rd', |
| | 104 | + ); |
| | 105 | + if ( isset( $prefixes[$table] ) ) { |
| | 106 | + return $prefixes[$table]; |
| | 107 | + } else { |
| | 108 | + throw new MWException( "Invalid table \"$table\" in " . __CLASS__ ); |
| | 109 | + } |
| | 110 | + } |
| | 111 | + |
| | 112 | + /** |
| | 113 | + * Get the SQL condition array for selecting backlinks, with a join on the page table |
| | 114 | + */ |
| | 115 | + protected function getConditions( $table ) { |
| | 116 | + $prefix = $this->getPrefix( $table ); |
| | 117 | + switch ( $table ) { |
| | 118 | + case 'pagelinks': |
| | 119 | + case 'templatelinks': |
| | 120 | + case 'redirect': |
| | 121 | + $conds = array( |
| | 122 | + "{$prefix}_namespace" => $this->title->getNamespace(), |
| | 123 | + "{$prefix}_title" => $this->title->getDBkey(), |
| | 124 | + "page_id={$prefix}_from" |
| | 125 | + ); |
| | 126 | + break; |
| | 127 | + case 'imagelinks': |
| | 128 | + $conds = array( |
| | 129 | + 'il_to' => $this->title->getDBkey(), |
| | 130 | + 'page_id=il_from' |
| | 131 | + ); |
| | 132 | + break; |
| | 133 | + case 'categorylinks': |
| | 134 | + $conds = array( |
| | 135 | + 'cl_to' => $this->title->getDBkey(), |
| | 136 | + 'page_id=cl_from', |
| | 137 | + ); |
| | 138 | + break; |
| | 139 | + default: |
| | 140 | + throw new MWException( "Invalid table \"$table\" in " . __CLASS__ ); |
| | 141 | + } |
| | 142 | + return $conds; |
| | 143 | + } |
| | 144 | + |
| | 145 | + /** |
| | 146 | + * Get the approximate number of backlinks |
| | 147 | + */ |
| | 148 | + public function getNumLinks( $table ) { |
| | 149 | + if ( isset( $this->fullResultCache[$table] ) ) { |
| | 150 | + return $this->fullResultCache[$table]->numRows(); |
| | 151 | + } |
| | 152 | + if ( isset( $this->partitionCache[$table] ) ) { |
| | 153 | + $entry = reset( $this->partitionCache[$table] ); |
| | 154 | + return $entry['numRows']; |
| | 155 | + } |
| | 156 | + $titleArray = $this->getLinks( $table ); |
| | 157 | + return $titleArray->count(); |
| | 158 | + } |
| | 159 | + |
| | 160 | + /** |
| | 161 | + * Partition the backlinks into batches. |
| | 162 | + * Returns an array giving the start and end of each range. The first batch has |
| | 163 | + * a start of false, and the last batch has an end of false. |
| | 164 | + * |
| | 165 | + * @param string $table The links table name |
| | 166 | + * @param integer $batchSize |
| | 167 | + * @return array |
| | 168 | + */ |
| | 169 | + public function partition( $table, $batchSize ) { |
| | 170 | + // Try cache |
| | 171 | + if ( isset( $this->partitionCache[$table][$batchSize] ) ) { |
| | 172 | + wfDebug( __METHOD__.": got from partition cache\n" ); |
| | 173 | + return $this->partitionCache[$table][$batchSize]['batches']; |
| | 174 | + } |
| | 175 | + $this->partitionCache[$table][$batchSize] = false; |
| | 176 | + $cacheEntry =& $this->partitionCache[$table][$batchSize]; |
| | 177 | + |
| | 178 | + // Try full result cache |
| | 179 | + if ( isset( $this->fullResultCache[$table] ) ) { |
| | 180 | + $cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize ); |
| | 181 | + wfDebug( __METHOD__.": got from full result cache\n" ); |
| | 182 | + return $cacheEntry['batches']; |
| | 183 | + } |
| | 184 | + // Try memcached |
| | 185 | + global $wgMemc; |
| | 186 | + $memcKey = wfMemcKey( 'backlinks', md5( $this->title->getPrefixedDBkey() ), |
| | 187 | + $table, $batchSize ); |
| | 188 | + $memcValue = $wgMemc->get( $memcKey ); |
| | 189 | + if ( is_array( $memcValue ) ) { |
| | 190 | + $cacheEntry = $memcValue; |
| | 191 | + wfDebug( __METHOD__.": got from memcached $memcKey\n" ); |
| | 192 | + return $cacheEntry['batches']; |
| | 193 | + } |
| | 194 | + // Fetch from database |
| | 195 | + $this->getLinks( $table ); |
| | 196 | + $cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize ); |
| | 197 | + // Save to memcached |
| | 198 | + $wgMemc->set( $memcKey, $cacheEntry, self::CACHE_EXPIRY ); |
| | 199 | + wfDebug( __METHOD__.": got from database\n" ); |
| | 200 | + return $cacheEntry['batches']; |
| | 201 | + } |
| | 202 | + |
| | 203 | + /** |
| | 204 | + * Partition a DB result with backlinks in it into batches |
| | 205 | + */ |
| | 206 | + protected function partitionResult( $res, $batchSize ) { |
| | 207 | + $batches = array(); |
| | 208 | + $numRows = $res->numRows(); |
| | 209 | + $numBatches = ceil( $numRows / $batchSize ); |
| | 210 | + if ( !$numRows ) { |
| | 211 | + $batches = array( array( false, false ) ); |
| | 212 | + } else { |
| | 213 | + for ( $i = 0; $i < $numBatches; $i++ ) { |
| | 214 | + if ( $i == 0 ) { |
| | 215 | + $start = false; |
| | 216 | + } else { |
| | 217 | + $rowNum = intval( $numRows * $i / $numBatches ); |
| | 218 | + $res->seek( $rowNum ); |
| | 219 | + $row = $res->fetchObject(); |
| | 220 | + $start = $row->page_id; |
| | 221 | + } |
| | 222 | + if ( $i == $numBatches - 1 ) { |
| | 223 | + $end = false; |
| | 224 | + } else { |
| | 225 | + $rowNum = intval( $numRows * ( $i + 1 ) / $numBatches ); |
| | 226 | + $res->seek( $rowNum ); |
| | 227 | + $row = $res->fetchObject(); |
| | 228 | + $end = $row->page_id - 1; |
| | 229 | + } |
| | 230 | + $batches[] = array( $start, $end ); |
| | 231 | + } |
| | 232 | + } |
| | 233 | + return array( 'numRows' => $numRows, 'batches' => $batches ); |
| | 234 | + } |
| | 235 | +} |
| Property changes on: trunk/phase3/includes/BacklinkCache.php |
| ___________________________________________________________________ |
| Name: svn:eol-style |
| 1 | 236 | + native |
| Index: trunk/phase3/includes/Title.php |
| — | — | @@ -69,6 +69,7 @@ |
| 70 | 70 | var $mLength = -1; ///< The page length, 0 for special pages |
| 71 | 71 | var $mRedirect = null; ///< Is the article at this title a redirect? |
| 72 | 72 | var $mNotificationTimestamp = array(); ///< Associative array of user ID -> timestamp/false |
| | 73 | + var $mBacklinkCache = null; ///< Cache of links to this title |
| 73 | 74 | //@} |
| 74 | 75 | |
| 75 | 76 | |
| — | — | @@ -3664,4 +3665,14 @@ |
| 3665 | 3666 | |
| 3666 | 3667 | return true; |
| 3667 | 3668 | } |
| | 3669 | + |
| | 3670 | + /** |
| | 3671 | + * Get a backlink cache object |
| | 3672 | + */ |
| | 3673 | + function getBacklinkCache() { |
| | 3674 | + if ( is_null( $this->mBacklinkCache ) ) { |
| | 3675 | + $this->mBacklinkCache = new BacklinkCache( $this ); |
| | 3676 | + } |
| | 3677 | + return $this->mBacklinkCache; |
| | 3678 | + } |
| 3668 | 3679 | } |
| Index: trunk/phase3/includes/HTMLCacheUpdate.php |
| — | — | @@ -35,146 +35,76 @@ |
| 36 | 36 | $this->mTable = $table; |
| 37 | 37 | $this->mRowsPerJob = $wgUpdateRowsPerJob; |
| 38 | 38 | $this->mRowsPerQuery = $wgUpdateRowsPerQuery; |
| | 39 | + $this->mCache = $this->mTitle->getBacklinkCache(); |
| 39 | 40 | } |
| 40 | 41 | |
| 41 | 42 | public function doUpdate() { |
| 42 | 43 | # Fetch the IDs |
| 43 | | - $cond = $this->getToCondition(); |
| 44 | | - $dbr = wfGetDB( DB_SLAVE ); |
| 45 | | - $res = $dbr->select( $this->mTable, $this->getFromField(), $cond, __METHOD__ ); |
| | 44 | + $numRows = $this->mCache->getNumLinks( $this->mTable ); |
| 46 | 45 | |
| 47 | | - if ( $dbr->numRows( $res ) != 0 ) { |
| 48 | | - if ( $dbr->numRows( $res ) > $this->mRowsPerJob ) { |
| 49 | | - $this->insertJobs( $res ); |
| | 46 | + if ( $numRows != 0 ) { |
| | 47 | + if ( $numRows > $this->mRowsPerJob ) { |
| | 48 | + $this->insertJobs(); |
| 50 | 49 | } else { |
| 51 | | - $this->invalidateIDs( $res ); |
| | 50 | + $this->invalidate(); |
| 52 | 51 | } |
| 53 | 52 | } |
| 54 | 53 | wfRunHooks( 'HTMLCacheUpdate::doUpdate', array($this->mTitle) ); |
| 55 | 54 | } |
| 56 | 55 | |
| 57 | | - protected function insertJobs( ResultWrapper $res ) { |
| 58 | | - $numRows = $res->numRows(); |
| 59 | | - $numBatches = ceil( $numRows / $this->mRowsPerJob ); |
| 60 | | - $realBatchSize = $numRows / $numBatches; |
| 61 | | - $start = false; |
| 62 | | - $jobs = array(); |
| 63 | | - do { |
| 64 | | - for ( $i = 0; $i <= $realBatchSize - 1; $i++ ) { |
| 65 | | - $row = $res->fetchRow(); |
| 66 | | - if ( $row ) { |
| 67 | | - $id = $row[0]; |
| 68 | | - } else { |
| 69 | | - $id = false; |
| 70 | | - break; |
| 71 | | - } |
| 72 | | - } |
| 73 | | - |
| | 56 | + protected function insertJobs() { |
| | 57 | + $batches = $this->mCache->partition( $this->mTable, $this->mRowsPerJob ); |
| | 58 | + if ( !$batches ) { |
| | 59 | + return; |
| | 60 | + } |
| | 61 | + foreach ( $batches as $batch ) { |
| 74 | 62 | $params = array( |
| 75 | 63 | 'table' => $this->mTable, |
| 76 | | - 'start' => $start, |
| 77 | | - 'end' => ( $id !== false ? $id - 1 : false ), |
| | 64 | + 'start' => $batch[0], |
| | 65 | + 'end' => $batch[1], |
| 78 | 66 | ); |
| 79 | 67 | $jobs[] = new HTMLCacheUpdateJob( $this->mTitle, $params ); |
| 80 | | - |
| 81 | | - $start = $id; |
| 82 | | - } while ( $start ); |
| 83 | | - |
| | 68 | + } |
| 84 | 69 | Job::batchInsert( $jobs ); |
| 85 | 70 | } |
| 86 | 71 | |
| 87 | | - protected function getPrefix() { |
| 88 | | - static $prefixes = array( |
| 89 | | - 'pagelinks' => 'pl', |
| 90 | | - 'imagelinks' => 'il', |
| 91 | | - 'categorylinks' => 'cl', |
| 92 | | - 'templatelinks' => 'tl', |
| 93 | | - 'redirect' => 'rd', |
| 94 | | - ); |
| 95 | 72 | |
| 96 | | - if ( is_null( $this->mPrefix ) ) { |
| 97 | | - $this->mPrefix = $prefixes[$this->mTable]; |
| 98 | | - if ( is_null( $this->mPrefix ) ) { |
| 99 | | - throw new MWException( "Invalid table type \"{$this->mTable}\" in " . __CLASS__ ); |
| 100 | | - } |
| 101 | | - } |
| 102 | | - return $this->mPrefix; |
| 103 | | - } |
| 104 | | - |
| 105 | | - public function getFromField() { |
| 106 | | - return $this->getPrefix() . '_from'; |
| 107 | | - } |
| 108 | | - |
| 109 | | - public function getToCondition() { |
| 110 | | - $prefix = $this->getPrefix(); |
| 111 | | - switch ( $this->mTable ) { |
| 112 | | - case 'pagelinks': |
| 113 | | - case 'templatelinks': |
| 114 | | - case 'redirect': |
| 115 | | - return array( |
| 116 | | - "{$prefix}_namespace" => $this->mTitle->getNamespace(), |
| 117 | | - "{$prefix}_title" => $this->mTitle->getDBkey() |
| 118 | | - ); |
| 119 | | - case 'imagelinks': |
| 120 | | - return array( 'il_to' => $this->mTitle->getDBkey() ); |
| 121 | | - case 'categorylinks': |
| 122 | | - return array( 'cl_to' => $this->mTitle->getDBkey() ); |
| 123 | | - } |
| 124 | | - throw new MWException( 'Invalid table type in ' . __CLASS__ ); |
| 125 | | - } |
| 126 | | - |
| 127 | 73 | /** |
| 128 | | - * Invalidate a set of IDs, right now |
| | 74 | + * Invalidate a set of pages, right now |
| 129 | 75 | */ |
| 130 | | - public function invalidateIDs( ResultWrapper $res ) { |
| | 76 | + public function invalidate( $startId = false, $endId = false ) { |
| 131 | 77 | global $wgUseFileCache, $wgUseSquid; |
| 132 | 78 | |
| 133 | | - if ( $res->numRows() == 0 ) { |
| | 79 | + $titleArray = $this->mCache->getLinks( $this->mTable, $startId, $endId ); |
| | 80 | + if ( $titleArray->count() == 0 ) { |
| 134 | 81 | return; |
| 135 | 82 | } |
| 136 | 83 | |
| 137 | 84 | $dbw = wfGetDB( DB_MASTER ); |
| 138 | 85 | $timestamp = $dbw->timestamp(); |
| 139 | | - $done = false; |
| 140 | 86 | |
| 141 | | - while ( !$done ) { |
| 142 | | - # Get all IDs in this query into an array |
| 143 | | - $ids = array(); |
| 144 | | - for ( $i = 0; $i < $this->mRowsPerQuery; $i++ ) { |
| 145 | | - $row = $res->fetchRow(); |
| 146 | | - if ( $row ) { |
| 147 | | - $ids[] = $row[0]; |
| 148 | | - } else { |
| 149 | | - $done = true; |
| 150 | | - break; |
| 151 | | - } |
| 152 | | - } |
| | 87 | + # Get all IDs in this query into an array |
| | 88 | + $ids = array(); |
| | 89 | + foreach ( $titleArray as $title ) { |
| | 90 | + $ids[] = $title->getArticleID(); |
| | 91 | + } |
| | 92 | + # Update page_touched |
| | 93 | + $dbw->update( 'page', |
| | 94 | + array( 'page_touched' => $timestamp ), |
| | 95 | + array( 'page_id IN (' . $dbw->makeList( $ids ) . ')' ), |
| | 96 | + __METHOD__ |
| | 97 | + ); |
| 153 | 98 | |
| 154 | | - if ( !count( $ids ) ) { |
| 155 | | - break; |
| 156 | | - } |
| | 99 | + # Update squid |
| | 100 | + if ( $wgUseSquid ) { |
| | 101 | + $u = SquidUpdate::newFromTitles( $titleArray ); |
| | 102 | + $u->doUpdate(); |
| | 103 | + } |
| 157 | 104 | |
| 158 | | - # Update page_touched |
| 159 | | - $dbw->update( 'page', |
| 160 | | - array( 'page_touched' => $timestamp ), |
| 161 | | - array( 'page_id IN (' . $dbw->makeList( $ids ) . ')' ), |
| 162 | | - __METHOD__ |
| 163 | | - ); |
| 164 | | - |
| 165 | | - # Update squid |
| 166 | | - if ( $wgUseSquid || $wgUseFileCache ) { |
| 167 | | - $titles = Title::newFromIDs( $ids ); |
| 168 | | - if ( $wgUseSquid ) { |
| 169 | | - $u = SquidUpdate::newFromTitles( $titles ); |
| 170 | | - $u->doUpdate(); |
| 171 | | - } |
| 172 | | - |
| 173 | | - # Update file cache |
| 174 | | - if ( $wgUseFileCache ) { |
| 175 | | - foreach ( $titles as $title ) { |
| 176 | | - HTMLFileCache::clearFileCache( $title ); |
| 177 | | - } |
| 178 | | - } |
| | 105 | + # Update file cache |
| | 106 | + if ( $wgUseFileCache ) { |
| | 107 | + foreach ( $titleArray as $title ) { |
| | 108 | + HTMLFileCache::clearFileCache( $title ); |
| 179 | 109 | } |
| 180 | 110 | } |
| 181 | 111 | } |
| — | — | @@ -204,20 +134,7 @@ |
| 205 | 135 | |
| 206 | 136 | public function run() { |
| 207 | 137 | $update = new HTMLCacheUpdate( $this->title, $this->table ); |
| 208 | | - |
| 209 | | - $fromField = $update->getFromField(); |
| 210 | | - $conds = $update->getToCondition(); |
| 211 | | - if ( $this->start ) { |
| 212 | | - $conds[] = "$fromField >= {$this->start}"; |
| 213 | | - } |
| 214 | | - if ( $this->end ) { |
| 215 | | - $conds[] = "$fromField <= {$this->end}"; |
| 216 | | - } |
| 217 | | - |
| 218 | | - $dbr = wfGetDB( DB_SLAVE ); |
| 219 | | - $res = $dbr->select( $this->table, $fromField, $conds, __METHOD__ ); |
| 220 | | - $update->invalidateIDs( $res ); |
| 221 | | - |
| | 138 | + $update->invalidate( $this->start, $this->end ); |
| 222 | 139 | return true; |
| 223 | 140 | } |
| 224 | 141 | } |
| Index: trunk/phase3/includes/SquidUpdate.php |
| — | — | @@ -52,13 +52,17 @@ |
| 53 | 53 | return new SquidUpdate( $blurlArr ); |
| 54 | 54 | } |
| 55 | 55 | |
| 56 | | - static function newFromTitles( &$titles, $urlArr = array() ) { |
| | 56 | + /** |
| | 57 | + * Create a SquidUpdate from an array of Title objects, or a TitleArray object |
| | 58 | + */ |
| | 59 | + static function newFromTitles( $titles, $urlArr = array() ) { |
| 57 | 60 | global $wgMaxSquidPurgeTitles; |
| 58 | | - if ( count( $titles ) > $wgMaxSquidPurgeTitles ) { |
| 59 | | - $titles = array_slice( $titles, 0, $wgMaxSquidPurgeTitles ); |
| 60 | | - } |
| | 61 | + $i = 0; |
| 61 | 62 | foreach ( $titles as $title ) { |
| 62 | 63 | $urlArr[] = $title->getInternalURL(); |
| | 64 | + if ( $i++ > $wgMaxSquidPurgeTitles ) { |
| | 65 | + break; |
| | 66 | + } |
| 63 | 67 | } |
| 64 | 68 | return new SquidUpdate( $urlArr ); |
| 65 | 69 | } |
| Index: trunk/phase3/includes/Import.php |
| — | — | @@ -223,7 +223,7 @@ |
| 224 | 224 | |
| 225 | 225 | } elseif( $changed ) { |
| 226 | 226 | wfDebug( __METHOD__ . ": running onArticleEdit\n" ); |
| 227 | | - Article::onArticleEdit( $this->title, 'skiptransclusions' ); // leave templatelinks for editUpdates() |
| | 227 | + Article::onArticleEdit( $this->title ); |
| 228 | 228 | |
| 229 | 229 | wfDebug( __METHOD__ . ": running edit updates\n" ); |
| 230 | 230 | $article->editUpdates( |
| Index: trunk/phase3/RELEASE-NOTES |
| — | — | @@ -181,6 +181,7 @@ |
| 182 | 182 | use correct message 'allpagesprefix' for input form label, replace _ with ' ' |
| 183 | 183 | in next page link |
| 184 | 184 | * (bug 17506) Exceptions within exceptions now respect $wgShowExceptionDetails |
| | 185 | +* Fixed excessive job queue utilisation |
| 185 | 186 | |
| 186 | 187 | == API changes in 1.15 == |
| 187 | 188 | * (bug 16858) Revamped list=deletedrevs to make listing deleted contributions |