| Index: trunk/phase3/maintenance/refreshLinks.inc |
| — | — | @@ -136,41 +136,90 @@ |
| 137 | 137 | $dbw->immediateCommit(); |
| 138 | 138 | } |
| 139 | 139 | |
| 140 | | -function deleteLinksFromNonexistent( $maxLag = 0 ) { |
| | 140 | +/* |
| | 141 | + * Removes non-existing links from pages from pagelinks, imagelinks, |
| | 142 | + * categorylinks, templatelinks and externallinks tables. |
| | 143 | + * |
| | 144 | + * @param $maxLag |
| | 145 | + * @param $batchSize The size of deletion batches |
| | 146 | + * |
| | 147 | + * @author Merlijn van Deen <valhallasw@arctus.nl> |
| | 148 | + */ |
| | 149 | +function deleteLinksFromNonexistent( $maxLag = 0, $batchSize = 100 ) { |
| 141 | 150 | $fname = 'deleteLinksFromNonexistent'; |
| 142 | | - |
| 143 | 151 | wfWaitForSlaves( $maxLag ); |
| 144 | | - |
| | 152 | + |
| 145 | 153 | $dbw = wfGetDB( DB_MASTER ); |
| 146 | | - |
| 147 | | - $linksTables = array( |
| | 154 | + $dbr = wfGetDB( DB_SLAVE ); |
| | 155 | + $dbr->bufferResults(false); |
| | 156 | + |
| | 157 | + $linksTables = array( // table name => page_id field |
| 148 | 158 | 'pagelinks' => 'pl_from', |
| 149 | 159 | 'imagelinks' => 'il_from', |
| 150 | 160 | 'categorylinks' => 'cl_from', |
| 151 | 161 | 'templatelinks' => 'tl_from', |
| 152 | 162 | 'externallinks' => 'el_from', |
| 153 | 163 | ); |
| 154 | | - |
| 155 | | - $page = $dbw->tableName( 'page' ); |
| 156 | | - |
| 157 | | - |
| | 164 | + |
| | 165 | + $readPage = $dbr->tableName( 'page' ); |
| | 166 | + |
| 158 | 167 | foreach ( $linksTables as $table => $field ) { |
| 159 | | - if ( !$dbw->ping() ) { |
| 160 | | - print "DB disconnected, reconnecting..."; |
| 161 | | - while ( !$dbw->ping() ) { |
| 162 | | - print "."; |
| 163 | | - sleep(10); |
| | 168 | + $readLinks = $dbr->tableName( $table ); |
| | 169 | + |
| | 170 | + print "Retrieving illegal entries from $table... "; |
| | 171 | + |
| | 172 | + $sql = "SELECT DISTINCT( $field ) FROM $readLinks LEFT JOIN $readPage ON $field=page_id WHERE page_id IS NULL;"; |
| | 173 | + $results = $dbr->query( $sql, $fname . ':' . $readLinks ); |
| | 174 | + |
| | 175 | + print $results->numRows() . " illegal " . $field. "s. "; |
| | 176 | + |
| | 177 | + if ( $results->numRows() > 0 ) { |
| | 178 | + $counter = 0; |
| | 179 | + $list = array(); |
| | 180 | + print "Removing illegal links: 1.."; |
| | 181 | + |
| | 182 | + foreach( $results as $row ) { |
| | 183 | + $counter++; |
| | 184 | + $list[] = $row->$field; |
| | 185 | + if ( ( $counter % $batchSize ) == 0 ) { |
| | 186 | + print $counter . ".."; |
| | 187 | + deleteBatch($dbw, $table, $field, $list); |
| | 188 | + $list = array(); |
| | 189 | + } |
| 164 | 190 | } |
| 165 | | - print "\n"; |
| | 191 | + print $counter; |
| | 192 | + deleteBatch($dbw, $table, $field, $list); |
| 166 | 193 | } |
| | 194 | + |
| | 195 | + print "\n"; |
| | 196 | + } |
| | 197 | +} |
| 167 | 198 | |
| 168 | | - $pTable = $dbw->tableName( $table ); |
| 169 | | - $sql = "DELETE $pTable FROM $pTable LEFT JOIN $page ON page_id=$field WHERE page_id IS NULL"; |
| | 199 | +/* Deletes a batch of items from a table. |
| | 200 | + * Runs the query: DELETE FROM <$table> WHERE <$field> IN (<$list>) |
| | 201 | + * |
| | 202 | + * @param $dbw Database Database object to run the DELETE query on |
| | 203 | + * @param $table table to work on; will be converted via $dbw->tableName. |
| | 204 | + * @param $field column to search in |
| | 205 | + * @param $list values to remove. Array with SQL-safe (!) values. |
| | 206 | + * |
| | 207 | + * @author Merlijn van Deen <valhallasw@arctus.nl> |
| | 208 | + */ |
| | 209 | +function deleteBatch($dbw, $table, $field, $list) { |
| | 210 | + if (count($list) == 0) return; |
| | 211 | + |
| | 212 | + $masterLinks = $dbw->tableName( $table ); |
| | 213 | + $fname = "deleteBatch:masterLinks"; |
| | 214 | + |
| | 215 | + if ( !$dbw->ping() ) { |
| | 216 | + print "\nDB disconnected, reconnecting..."; |
| | 217 | + while ( !$dbw->ping() ) { |
| | 218 | + print "."; |
| | 219 | + sleep(10); |
| | 220 | + } |
| | 221 | + print "\n"; |
| | 222 | + } |
| 170 | 223 | |
| 171 | | - print "Deleting $table from non-existent articles..."; |
| 172 | | - $dbw->query( $sql, $fname ); |
| 173 | | - print " fixed " .$dbw->affectedRows() . " row(s)\n"; |
| 174 | | - } |
| | 224 | + $sql = "DELETE FROM $masterLinks WHERE $field IN (" . join("," , $list) . ");"; |
| | 225 | + $dbw->query($sql, $fname); |
| 175 | 226 | } |
| 176 | | - |
| 177 | | -?> |
| Index: trunk/phase3/maintenance/refreshLinks.php |
| — | — | @@ -18,14 +18,16 @@ |
| 19 | 19 | [--new-only] [--redirects-only] |
| 20 | 20 | php refreshLinks.php [<start>] [-e <end>] [-m <maxlag>] --old-redirects-only |
| 21 | 21 | |
| 22 | | - --help : This help message |
| 23 | | - --dfn-only : Delete links from nonexistent articles only |
| 24 | | - --new-only : Only affect articles with just a single edit |
| 25 | | - --redirects-only : Only fix redirects, not all links |
| 26 | | - --old-redirects-only : Only fix redirects with no redirect table entry |
| 27 | | - -m <number> : Maximum replication lag |
| 28 | | - <start> : First page id to refresh |
| 29 | | - -e <number> : Last page id to refresh |
| | 22 | + --help : This help message |
| | 23 | + --dfn-only : Delete links from nonexistent articles only |
| | 24 | + --batch-size <number> : The delete batch size when removing links from |
| | 25 | + nonexistent articles (default 100) |
| | 26 | + --new-only : Only affect articles with just a single edit |
| | 27 | + --redirects-only : Only fix redirects, not all links |
| | 28 | + --old-redirects-only : Only fix redirects with no redirect table entry |
| | 29 | + -m <number> : Maximum replication lag |
| | 30 | + <start> : First page id to refresh |
| | 31 | + -e <number> : Last page id to refresh |
| 30 | 32 | |
| 31 | 33 | TEXT; |
| 32 | 34 | exit(0); |
| — | — | @@ -42,12 +44,9 @@ |
| 43 | 45 | |
| 44 | 46 | refreshLinks( $start, $options['new-only'], $options['m'], $options['e'], $options['redirects-only'], $options['old-redirects-only'] ); |
| 45 | 47 | } |
| 46 | | -// this bit's bad for replication: disabling temporarily |
| 47 | | -// --brion 2005-07-16 |
| 48 | | -//deleteLinksFromNonexistent(); |
| 49 | 48 | |
| | 49 | +deleteLinksFromNonexistent($options['m'], $options['batch-size']); |
| | 50 | + |
| 50 | 51 | if ( $options['globals'] ) { |
| 51 | 52 | print_r( $GLOBALS ); |
| 52 | 53 | } |
| 53 | | - |
| 54 | | - |