| Index: trunk/extensions/MWSearch/luceneUpdate.php |
| — | — | @@ -2,12 +2,13 @@ |
| 3 | 3 | |
| 4 | 4 | // symlink me into maintenance/ |
| 5 | 5 | |
| | 6 | +$options = array( 'skip' ); |
| 6 | 7 | require_once( 'commandLine.inc' ); |
| 7 | 8 | |
| 8 | 9 | |
| 9 | 10 | if( !isset( $args[0] ) ) { |
| 10 | 11 | print "Call MWUpdateDaemon remotely for status or updates.\n"; |
| 11 | | - print "Usage: php luceneUpdate.php [database] {status|stop|start|restart|rebuild}\n"; |
| | 12 | + print "Usage: php luceneUpdate.php [database] [--skip=n] {status|flush|stop|start|restart|rebuild}\n"; |
| 12 | 13 | exit( -1 ); |
| 13 | 14 | } |
| 14 | 15 | |
| — | — | @@ -35,6 +36,9 @@ |
| 36 | 37 | break; |
| 37 | 38 | case 'rebuild': |
| 38 | 39 | $builder = new LuceneBuilder(); |
| | 40 | + if( isset( $options['skip'] ) ) { |
| | 41 | + $builder->skip( intval( $options['skip'] ) ); |
| | 42 | + } |
| 39 | 43 | $ret = $builder->rebuildAll(); |
| 40 | 44 | break; |
| 41 | 45 | default: |
| — | — | @@ -57,6 +61,7 @@ |
| 58 | 62 | function LuceneBuilder() { |
| 59 | 63 | $this->db =& wfGetDB( DB_SLAVE ); |
| 60 | 64 | $this->dbstream =& $this->streamingSlave( $this->db ); |
| | 65 | + $this->offset = 0; |
| 61 | 66 | } |
| 62 | 67 | |
| 63 | 68 | function &streamingSlave( $db ) { |
| — | — | @@ -70,6 +75,10 @@ |
| 71 | 76 | return $stream; |
| 72 | 77 | } |
| 73 | 78 | |
| | 79 | + function skip( $offset ) { |
| | 80 | + $this->offset = intval( $offset ); |
| | 81 | + } |
| | 82 | + |
| 74 | 83 | function init( $max ) { |
| 75 | 84 | $this->max = $max; |
| 76 | 85 | $this->count = 0; |
| — | — | @@ -94,9 +103,45 @@ |
| 95 | 104 | $this->count, |
| 96 | 105 | $this->max, |
| 97 | 106 | $rate ); |
| | 107 | + |
| | 108 | + $this->wait(); |
| 98 | 109 | } |
| 99 | 110 | } |
| 100 | 111 | |
| | 112 | + /** |
| | 113 | + * See if the daemon's getting overloaded and pause if so |
| | 114 | + */ |
| | 115 | + function wait() { |
| | 116 | + $cutoff = 500; |
| | 117 | + $waittime = 10; |
| | 118 | + |
| | 119 | + while( true ) { |
| | 120 | + $status = MWSearchUpdater::getStatus(); |
| | 121 | + if( WikiError::isError( $status ) ) { |
| | 122 | + echo $status->getMessage() . "\n"; |
| | 123 | + sleep( $waittime ); |
| | 124 | + continue; |
| | 125 | + } |
| | 126 | + |
| | 127 | + // Updater IS running; 90418 items queued. |
| | 128 | + if( !preg_match( '/([0-9]+) items queued/', $status, $matches ) ) { |
| | 129 | + // ?! confused |
| | 130 | + break; |
| | 131 | + } |
| | 132 | + |
| | 133 | + $count = intval( $matches[1] ); |
| | 134 | + if( $count < $cutoff ) { |
| | 135 | + break; |
| | 136 | + } |
| | 137 | + |
| | 138 | + printf( "%s: %s\n", |
| | 139 | + wfTimestamp( TS_DB ), |
| | 140 | + $status ); |
| | 141 | + |
| | 142 | + sleep( $waittime ); |
| | 143 | + } |
| | 144 | + } |
| | 145 | + |
| 101 | 146 | function final() { |
| 102 | 147 | global $wgDBname; |
| 103 | 148 | $now = wfTime(); |
| — | — | @@ -120,16 +165,23 @@ |
| 121 | 166 | $lastError = true; |
| 122 | 167 | |
| 123 | 168 | $maxId = $this->db->selectField( 'page', 'MAX(page_id)', '', $fname ); |
| | 169 | + $maxId -= $this->offset; // hack for percentages |
| 124 | 170 | $this->init( $maxId ); |
| 125 | 171 | if( $maxId < 1 ) { |
| 126 | 172 | echo "Nothing to do.\n"; |
| 127 | 173 | return; |
| 128 | 174 | } |
| 129 | 175 | |
| | 176 | + $limit = array(); |
| | 177 | + if( $this->offset ) { |
| | 178 | + $limit = array( 'LIMIT $offset, 10000000' ); |
| | 179 | + } |
| | 180 | + |
| 130 | 181 | $result = $this->dbstream->select( array( 'page' ), |
| 131 | 182 | array( 'page_namespace', 'page_title', 'page_latest' ), |
| 132 | 183 | '', |
| 133 | | - $fname ); |
| | 184 | + $fname, |
| | 185 | + $limit ); |
| 134 | 186 | |
| 135 | 187 | $errorCount = 0; |
| 136 | 188 | while( $row = $this->dbstream->fetchObject( $result ) ) { |