r86447 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r86446‎ | r86447 | r86448 >
Date:00:12, 20 April 2011
Author:platonides
Status:resolved (Comments)
Tags:
Comment:
Merge r86398 into the old wfWaitForSlaves. Update core calls.
maintenance/waitForSlave.php still calls the old prototype.
Most instances were replaced with
sed -i 's/wfWaitForSlaves( [0-9]* );/wfWaitForSlaves();/' ./maintenance/initEditCount.php ./maintenance/updateRestrictions.php ./maintenance/updateSpecialPages.php ./maintenance/importDump.php ./maintenance/moveBatch.php ./maintenance/storage/resolveStubs.php ./maintenance/storage/trackBlobs.php ./maintenance/storage/fixBug20757.php ./maintenance/storage/moveToExternal.php ./maintenance/storage/compressOld.inc ./maintenance/populateSha1.php ./maintenance/deleteDefaultMessages.php ./maintenance/migrateUserGroup.php ./maintenance/importImages.php ./maintenance/runJobs.php ./maintenance/archives/upgradeLogging.php ./maintenance/deleteBatch.php ./maintenance/populateLogSearch.php ./maintenance/populateLogUsertext.php ./maintenance/gearman/gearmanWorker.php ./maintenance/populateRevisionLength.php ./maintenance/refreshLinks.php ./maintenance/deleteSelfExternals.php ./maintenance/upgrade1_5.php ./maintenance/rebuildFileCache.php ./includes/job/RefreshLinksJob.php ./includes/installer/MysqlUpdater.php ./maintenance/convertUserOptions.php ./maintenance/populateParentId.php ./maintenance/runBatchedQuery.php ./maintenance/upgrade1_5.php ./maintenance/waitForSlave.php ./maintenance/populateCategory.php ./maintenance/importImages.php
Modified paths:
  • /trunk/phase3/includes/GlobalFunctions.php (modified) (history)
  • /trunk/phase3/includes/installer/MysqlUpdater.php (modified) (history)
  • /trunk/phase3/includes/job/RefreshLinksJob.php (modified) (history)
  • /trunk/phase3/maintenance/archives/upgradeLogging.php (modified) (history)
  • /trunk/phase3/maintenance/convertUserOptions.php (modified) (history)
  • /trunk/phase3/maintenance/deleteBatch.php (modified) (history)
  • /trunk/phase3/maintenance/deleteDefaultMessages.php (modified) (history)
  • /trunk/phase3/maintenance/deleteSelfExternals.php (modified) (history)
  • /trunk/phase3/maintenance/gearman/gearmanWorker.php (modified) (history)
  • /trunk/phase3/maintenance/importDump.php (modified) (history)
  • /trunk/phase3/maintenance/importImages.php (modified) (history)
  • /trunk/phase3/maintenance/initEditCount.php (modified) (history)
  • /trunk/phase3/maintenance/migrateUserGroup.php (modified) (history)
  • /trunk/phase3/maintenance/moveBatch.php (modified) (history)
  • /trunk/phase3/maintenance/populateCategory.php (modified) (history)
  • /trunk/phase3/maintenance/populateLogSearch.php (modified) (history)
  • /trunk/phase3/maintenance/populateLogUsertext.php (modified) (history)
  • /trunk/phase3/maintenance/populateParentId.php (modified) (history)
  • /trunk/phase3/maintenance/populateRevisionLength.php (modified) (history)
  • /trunk/phase3/maintenance/populateSha1.php (modified) (history)
  • /trunk/phase3/maintenance/rebuildFileCache.php (modified) (history)
  • /trunk/phase3/maintenance/refreshLinks.php (modified) (history)
  • /trunk/phase3/maintenance/runBatchedQuery.php (modified) (history)
  • /trunk/phase3/maintenance/runJobs.php (modified) (history)
  • /trunk/phase3/maintenance/storage/compressOld.inc (modified) (history)
  • /trunk/phase3/maintenance/storage/fixBug20757.php (modified) (history)
  • /trunk/phase3/maintenance/storage/moveToExternal.php (modified) (history)
  • /trunk/phase3/maintenance/storage/resolveStubs.php (modified) (history)
  • /trunk/phase3/maintenance/storage/trackBlobs.php (modified) (history)
  • /trunk/phase3/maintenance/updateRestrictions.php (modified) (history)
  • /trunk/phase3/maintenance/updateSpecialPages.php (modified) (history)
  • /trunk/phase3/maintenance/upgrade1_5.php (modified) (history)

Diff [purge]

Index: trunk/phase3/maintenance/runBatchedQuery.php
——@@ -45,7 +45,7 @@
4646 $dbw->query( $query, __METHOD__ );
4747 $affected = $dbw->affectedRows();
4848 $this->output( "$affected rows\n" );
49 - wfWaitForSlaves_masterPos();
 49+ wfWaitForSlaves();
5050 } while ( $affected > 0 );
5151 }
5252
Index: trunk/phase3/maintenance/archives/upgradeLogging.php
——@@ -164,7 +164,7 @@
165165 $this->dbw->insert( $dstTable, $batch, __METHOD__ );
166166 $numRowsCopied += count( $batch );
167167
168 - wfWaitForSlaves( 5 );
 168+ wfWaitForSlaves();
169169 }
170170 echo "Copied $numRowsCopied rows\n";
171171 }
Index: trunk/phase3/maintenance/storage/fixBug20757.php
——@@ -260,7 +260,7 @@
261261 static $iteration = 0;
262262 ++$iteration;
263263 if ( ++$iteration > 50 == 0 ) {
264 - wfWaitForSlaves( 5 );
 264+ wfWaitForSlaves();
265265 $iteration = 0;
266266 }
267267 }
Index: trunk/phase3/maintenance/storage/compressOld.inc
——@@ -155,7 +155,7 @@
156156 #$conds[] = 'page_id=rev_page AND rev_id != page_latest';
157157
158158 for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) {
159 - wfWaitForSlaves( 5 );
 159+ wfWaitForSlaves();
160160
161161 # Wake up
162162 $dbr->ping();
——@@ -309,7 +309,7 @@
310310 print "/";
311311 $dbw->commit();
312312 $i += $thisChunkSize;
313 - wfWaitForSlaves( 5 );
 313+ wfWaitForSlaves();
314314 }
315315 print "\n";
316316 }
Index: trunk/phase3/maintenance/storage/moveToExternal.php
——@@ -66,7 +66,7 @@
6767
6868 if ( !( $block % REPORTING_INTERVAL ) ) {
6969 print "oldid=$blockStart, moved=$numMoved\n";
70 - wfWaitForSlaves( 2 );
 70+ wfWaitForSlaves();
7171 }
7272
7373 $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ),
Index: trunk/phase3/maintenance/storage/resolveStubs.php
——@@ -45,7 +45,7 @@
4646 $numBlocks = intval( $maxID / $blockSize ) + 1;
4747
4848 for ( $b = 0; $b < $numBlocks; $b++ ) {
49 - wfWaitForSlaves( 2 );
 49+ wfWaitForSlaves();
5050
5151 printf( "%5.2f%%\n", $b / $numBlocks * 100 );
5252 $start = intval( $maxID / $numBlocks ) * $b + 1;
Index: trunk/phase3/maintenance/storage/trackBlobs.php
——@@ -206,7 +206,7 @@
207207 if ( $batchesDone >= $this->reportingInterval ) {
208208 $batchesDone = 0;
209209 echo "$startId / $endId\n";
210 - wfWaitForSlaves( 5 );
 210+ wfWaitForSlaves();
211211 }
212212 }
213213 echo "Found $rowsInserted revisions\n";
——@@ -290,7 +290,7 @@
291291 if ( $batchesDone >= $this->reportingInterval ) {
292292 $batchesDone = 0;
293293 echo "$startId / $endId\n";
294 - wfWaitForSlaves( 5 );
 294+ wfWaitForSlaves();
295295 }
296296 }
297297 echo "Found $rowsInserted orphan text rows\n";
Index: trunk/phase3/maintenance/populateSha1.php
——@@ -68,7 +68,7 @@
6969 foreach ( $res as $row ) {
7070 if ( $i % 100 == 0 ) {
7171 $this->output( sprintf( "Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100 ) );
72 - wfWaitForSlaves( 5 );
 72+ wfWaitForSlaves();
7373 }
7474 $file = wfLocalFile( $row->img_name );
7575 if ( !$file ) {
Index: trunk/phase3/maintenance/rebuildFileCache.php
——@@ -109,7 +109,7 @@
110110 }
111111 $blockStart += $this->mBatchSize;
112112 $blockEnd += $this->mBatchSize;
113 - wfWaitForSlaves( 5 );
 113+ wfWaitForSlaves();
114114 }
115115 $this->output( "Done!\n" );
116116
Index: trunk/phase3/maintenance/refreshLinks.php
——@@ -107,7 +107,7 @@
108108 foreach ( $res as $row ) {
109109 if ( !( ++$i % $reportingInterval ) ) {
110110 $this->output( "$i\n" );
111 - wfWaitForSlaves( $maxLag );
 111+ wfWaitForSlaves();
112112 }
113113 $this->fixRedirect( $row->page_id );
114114 }
——@@ -127,7 +127,7 @@
128128 foreach ( $res as $row ) {
129129 if ( !( ++$i % $reportingInterval ) ) {
130130 $this->output( "$i\n" );
131 - wfWaitForSlaves( $maxLag );
 131+ wfWaitForSlaves();
132132 }
133133 if ( $redirectsOnly ) {
134134 $this->fixRedirect( $row->page_id );
——@@ -148,7 +148,7 @@
149149
150150 if ( !( $id % $reportingInterval ) ) {
151151 $this->output( "$id\n" );
152 - wfWaitForSlaves( $maxLag );
 152+ wfWaitForSlaves();
153153 }
154154 $this->fixRedirect( $id );
155155 }
——@@ -161,7 +161,7 @@
162162
163163 if ( !( $id % $reportingInterval ) ) {
164164 $this->output( "$id\n" );
165 - wfWaitForSlaves( $maxLag );
 165+ wfWaitForSlaves();
166166 }
167167 self::fixLinksFromArticle( $id );
168168 }
——@@ -237,7 +237,7 @@
238238 * @author Merlijn van Deen <valhallasw@arctus.nl>
239239 */
240240 private function deleteLinksFromNonexistent( $maxLag = 0, $batchSize = 100 ) {
241 - wfWaitForSlaves( $maxLag );
 241+ wfWaitForSlaves();
242242
243243 $dbw = wfGetDB( DB_MASTER );
244244
——@@ -273,7 +273,7 @@
274274 $counter++;
275275 $list[] = $row->$field;
276276 if ( ( $counter % $batchSize ) == 0 ) {
277 - wfWaitForSlaves( 5 );
 277+ wfWaitForSlaves();
278278 $dbw->delete( $table, array( $field => $list ), __METHOD__ );
279279
280280 $this->output( $counter . ".." );
Index: trunk/phase3/maintenance/deleteSelfExternals.php
——@@ -39,7 +39,7 @@
4040 $this->output( "Deleting self externals from $wgServer\n" );
4141 $db = wfGetDB( DB_MASTER );
4242 while ( 1 ) {
43 - wfWaitForSlaves( 2 );
 43+ wfWaitForSlaves();
4444 $db->commit();
4545 $q = $db->limitResult( "DELETE /* deleteSelfExternals */ FROM externallinks WHERE el_to"
4646 . $db->buildLike( $wgServer . '/', $db->anyString() ), $this->mBatchSize );
Index: trunk/phase3/maintenance/moveBatch.php
——@@ -101,7 +101,7 @@
102102 if ( $interval ) {
103103 sleep( $interval );
104104 }
105 - wfWaitForSlaves( 5 );
 105+ wfWaitForSlaves();
106106 }
107107 }
108108 }
Index: trunk/phase3/maintenance/populateCategory.php
——@@ -119,7 +119,7 @@
120120 ++$i;
121121 if ( !( $i % self::REPORTING_INTERVAL ) ) {
122122 $this->output( "$name\n" );
123 - wfWaitForSlaves( $maxlag );
 123+ wfWaitForSlaves();
124124 }
125125 usleep( $throttle * 1000 );
126126 }
Index: trunk/phase3/maintenance/importImages.php
——@@ -249,8 +249,8 @@
250250 $article = new Article( $title );
251251 echo "\nWaiting for slaves...\n";
252252 // Wait for slaves.
253 - sleep( 2.0 );
254 - wfWaitForSlaves( 1.0 );
 253+ sleep( 2.0 ); # Why this sleep?
 254+ wfWaitForSlaves();
255255
256256 echo( "\nSetting image restrictions ... " );
257257 if ( $article->updateRestrictions( $restrictions ) )
Index: trunk/phase3/maintenance/importDump.php
——@@ -154,7 +154,7 @@
155155 $this->progress( "$this->revCount ($revrate revs/sec)" );
156156 }
157157 }
158 - wfWaitForSlaves( 5 );
 158+ wfWaitForSlaves();
159159 // XXX: Don't let deferred jobs array get absurdly large (bug 24375)
160160 wfDoUpdates( 'commit' );
161161 }
Index: trunk/phase3/maintenance/updateSpecialPages.php
——@@ -58,7 +58,7 @@
5959 }
6060 $this->output( sprintf( "completed in %.2fs\n", $seconds ) );
6161 # Wait for the slave to catch up
62 - wfWaitForSlaves( 5 );
 62+ wfWaitForSlaves();
6363 }
6464
6565 // This is needed to initialise $wgQueryPages
——@@ -130,7 +130,7 @@
131131 $dbw->commit();
132132 }
133133 # Wait for the slave to catch up
134 - wfWaitForSlaves( 5 );
 134+ wfWaitForSlaves();
135135 } else {
136136 $this->output( "cheap, skipped\n" );
137137 }
Index: trunk/phase3/maintenance/deleteBatch.php
——@@ -104,7 +104,7 @@
105105 if ( $interval ) {
106106 sleep( $interval );
107107 }
108 - wfWaitForSlaves( 5 );
 108+ wfWaitForSlaves();
109109 }
110110 }
111111 }
Index: trunk/phase3/maintenance/runJobs.php
——@@ -72,7 +72,7 @@
7373 if ( !$job )
7474 break;
7575
76 - wfWaitForSlaves( 5 );
 76+ wfWaitForSlaves();
7777 $t = microtime( true );
7878 $offset = $job->id;
7979 $status = $job->run();
Index: trunk/phase3/maintenance/convertUserOptions.php
——@@ -46,7 +46,7 @@
4747 $id = $this->convertOptionBatch( $res, $dbw );
4848 $dbw->commit();
4949
50 - wfWaitForSlaves( 1 );
 50+ wfWaitForSlaves();
5151
5252 if ( $id )
5353 $this->output( "--Converted to ID $id\n" );
Index: trunk/phase3/maintenance/deleteDefaultMessages.php
——@@ -64,7 +64,7 @@
6565
6666 foreach ( $res as $row ) {
6767 if ( function_exists( 'wfWaitForSlaves' ) ) {
68 - wfWaitForSlaves( 5 );
 68+ wfWaitForSlaves();
6969 }
7070 $dbw->ping();
7171 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
Index: trunk/phase3/maintenance/migrateUserGroup.php
——@@ -60,7 +60,7 @@
6161 $dbw->commit();
6262 $blockStart += $this->mBatchSize;
6363 $blockEnd += $this->mBatchSize;
64 - wfWaitForSlaves( 5 );
 64+ wfWaitForSlaves();
6565 }
6666 $this->output( "Done! $count user(s) in group '$oldGroup' are now in '$newGroup' instead.\n" );
6767 }
Index: trunk/phase3/maintenance/populateRevisionLength.php
——@@ -78,7 +78,7 @@
7979 }
8080 $blockStart += $this->mBatchSize;
8181 $blockEnd += $this->mBatchSize;
82 - wfWaitForSlaves( 5 );
 82+ wfWaitForSlaves();
8383 }
8484 $logged = $db->insert( 'updatelog',
8585 array( 'ul_key' => 'populate rev_len' ),
Index: trunk/phase3/maintenance/populateParentId.php
——@@ -98,7 +98,7 @@
9999 }
100100 $blockStart += $this->mBatchSize;
101101 $blockEnd += $this->mBatchSize;
102 - wfWaitForSlaves( 5 );
 102+ wfWaitForSlaves();
103103 }
104104 $logged = $db->insert( 'updatelog',
105105 array( 'ul_key' => 'populate rev_parent_id' ),
Index: trunk/phase3/maintenance/updateRestrictions.php
——@@ -96,7 +96,7 @@
9797 }
9898 $blockStart += $this->mBatchSize - 1;
9999 $blockEnd += $this->mBatchSize - 1;
100 - wfWaitForSlaves( 5 );
 100+ wfWaitForSlaves();
101101 }
102102 $this->output( "...removing dead rows from page_restrictions\n" );
103103 // Kill any broken rows from previous imports
Index: trunk/phase3/maintenance/upgrade1_5.php
——@@ -101,7 +101,6 @@
102102
103103 $this->cleanupSwaps = array();
104104 $this->emailAuth = false; # don't preauthenticate emails
105 - $this->maxLag = 10; # if slaves are lagged more than 10 secs, wait
106105 $this->step = $this->getOption( 'step', null );
107106 }
108107
——@@ -316,7 +315,7 @@
317316 */
318317 function insertChunk( &$chunk ) {
319318 // Give slaves a chance to catch up
320 - wfWaitForSlaves( $this->maxLag );
 319+ wfWaitForSlaves();
321320 $this->dbw->insert( $this->chunkTable, $chunk, $this->chunkFunction, $this->chunkOptions );
322321 }
323322
Index: trunk/phase3/maintenance/populateLogUsertext.php
——@@ -59,7 +59,7 @@
6060 $db->commit();
6161 $blockStart += $this->mBatchSize;
6262 $blockEnd += $this->mBatchSize;
63 - wfWaitForSlaves( 5 );
 63+ wfWaitForSlaves();
6464 }
6565 if ( $db->insert(
6666 'updatelog',
Index: trunk/phase3/maintenance/populateLogSearch.php
——@@ -130,7 +130,7 @@
131131 }
132132 $blockStart += self::LOG_SEARCH_BATCH_SIZE;
133133 $blockEnd += self::LOG_SEARCH_BATCH_SIZE;
134 - wfWaitForSlaves( 5 );
 134+ wfWaitForSlaves();
135135 }
136136 if ( $db->insert(
137137 'updatelog',
Index: trunk/phase3/maintenance/initEditCount.php
——@@ -93,7 +93,7 @@
9494 $delta,
9595 $rate ) );
9696
97 - wfWaitForSlaves( 10 );
 97+ wfWaitForSlaves();
9898 }
9999 } else {
100100 // Subselect should work on modern MySQLs etc
Index: trunk/phase3/maintenance/gearman/gearmanWorker.php
——@@ -36,7 +36,7 @@
3737 $interval = 5;
3838 $now = time();
3939 if ( $now - $lastSleep >= $interval ) {
40 - wfWaitForSlaves( $interval );
 40+ wfWaitForSlaves();
4141 $lastSleep = $now;
4242 }
4343 return false;
Index: trunk/phase3/includes/GlobalFunctions.php
——@@ -3146,43 +3146,17 @@
31473147 }
31483148
31493149 /**
3150 - * Sleep until the worst slave's replication lag is less than or equal to
3151 - * $maxLag, in seconds. Use this when updating very large numbers of rows, as
 3150+ * Modern version of wfWaitForSlaves(). Instead of looking at replication lag
 3151+ * and waiting for it to go down, this waits for the slaves to catch up to the
 3152+ * master position. Use this when updating very large numbers of rows, as
31523153 * in maintenance scripts, to avoid causing too much lag. Of course, this is
31533154 * a no-op if there are no slaves.
3154 - *
3155 - * Every time the function has to wait for a slave, it will print a message to
3156 - * that effect (and then sleep for a little while), so it's probably not best
3157 - * to use this outside maintenance scripts in its present form.
3158 - *
3159 - * @param $maxLag Integer
 3155+ *
 3156+ * @param $maxLag Integer (deprecated)
31603157 * @param $wiki mixed Wiki identifier accepted by wfGetLB
31613158 * @return null
31623159 */
3163 -function wfWaitForSlaves( $maxLag, $wiki = false ) {
3164 - if( $maxLag ) {
3165 - $lb = wfGetLB( $wiki );
3166 - list( $host, $lag ) = $lb->getMaxLag( $wiki );
3167 - while( $lag > $maxLag ) {
3168 - wfSuppressWarnings();
3169 - $name = gethostbyaddr( $host );
3170 - wfRestoreWarnings();
3171 - if( $name !== false ) {
3172 - $host = $name;
3173 - }
3174 - print "Waiting for $host (lagged $lag seconds)...\n";
3175 - sleep( $maxLag );
3176 - list( $host, $lag ) = $lb->getMaxLag();
3177 - }
3178 - }
3179 -}
3180 -
3181 -/**
3182 - * Modern version of wfWaitForSlaves(). Instead of looking at replication lag
3183 - * and waiting for it to go down, this waits for the slaves to catch up to the
3184 - * master position. This is much better for lag control than wfWaitForSlaves()
3185 - */
3186 -function wfWaitForSlaves_masterPos() {
 3160+function wfWaitForSlaves( $maxLag = false, $wiki = false ) {
31873161 $lb = wfGetLB();
31883162 // bug 27975 - Don't try to wait for slaves if there are none
31893163 // Prevents permission error when getting master position
Index: trunk/phase3/includes/installer/MysqlUpdater.php
——@@ -652,7 +652,7 @@
653653 foreach ( $res as $row ) {
654654 $count = ( $count + 1 ) % 100;
655655 if ( $count == 0 ) {
656 - wfWaitForSlaves( 10 );
 656+ wfWaitForSlaves();
657657 }
658658 $this->db->insert( 'templatelinks',
659659 array(
Index: trunk/phase3/includes/job/RefreshLinksJob.php
——@@ -119,7 +119,7 @@
120120 $update = new LinksUpdate( $title, $parserOutput, false );
121121 $update->doUpdate();
122122 wfProfileOut( __METHOD__.'-update' );
123 - wfWaitForSlaves( 5 );
 123+ wfWaitForSlaves();
124124 }
125125 wfProfileOut( __METHOD__ );
126126

Follow-up revisions

RevisionCommit summaryAuthorDate
r86484Follow up r86447. Support the $wiki parameterplatonides15:32, 20 April 2011

Past revisions this follows-up on

RevisionCommit summaryAuthorDate
r86398Put the code for waiting for slave lag the new-and-improved way (using master...catrope14:52, 19 April 2011

Comments

#Comment by Catrope (talk | contribs)   09:45, 20 April 2011 
-function wfWaitForSlaves_masterPos() {
+function wfWaitForSlaves( $maxLag = false, $wiki = false ) {
 	$lb = wfGetLB();

The wiki param is accepted but ignored. I don't know if anything was actually using it, but it would be trivial to support it. Looks good to me otherwise.

#Comment by Platonides (talk | contribs)   15:50, 20 April 2011

It was my intention to support it (and leave $maxLag as dummy). Fixed in r86484.

Status & tagging log