r97344 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r97343‎ | r97344 | r97345 >
Date:22:54, 16 September 2011
Author:reedy
Status:ok
Tags:
Comment:
Copy a few scripts out of 1.17wmf1 for preservation
Modified paths:
  • /trunk/extensions/WikimediaMaintenance/jeluf.php (added) (history)
  • /trunk/extensions/WikimediaMaintenance/jobs-loop.sh (added) (history)
  • /trunk/extensions/WikimediaMaintenance/storage/make-all-blobs (added) (history)
  • /trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php (added) (history)

Diff [purge]

Index: trunk/extensions/WikimediaMaintenance/storage/make-all-blobs
——@@ -0,0 +1,18 @@
 2+#!/bin/bash
 3+
 4+if [ -z $1 ];then
 5+ echo "Usage: make-all-blobs <server> [<table name>]"
 6+ exit 1
 7+fi
 8+server=$1
 9+if [ -z $2 ]; then
 10+ table=blobs
 11+else
 12+ table=$2
 13+fi
 14+
 15+for db in `</home/wikipedia/common/all.dblist`;do
 16+ echo "CREATE DATABASE IF NOT EXISTS $db" | mysql -u wikiadmin -p`wikiadmin_pass` -h $server && \
 17+ sed "s/blobs\>/$table/" blobs.sql | mysql -u wikiadmin -p`wikiadmin_pass` -h $server $db
 18+done
 19+
Property changes on: trunk/extensions/WikimediaMaintenance/storage/make-all-blobs
___________________________________________________________________
Added: svn:mergeinfo
120 Merged /branches/wmf-deployment/maintenance/storage/make-all-blobs:r60970
221 Merged /branches/REL1_15/phase3/maintenance/storage/make-all-blobs:r51646
322 Merged /branches/wmf/1.16wmf4/maintenance/storage/make-all-blobs:r67177,69199,76243,77266
423 Merged /trunk/phase3/maintenance/storage/make-all-blobs:r52290,52402,52404,52718,52737,52759,52776,52791,52800,52808,52812-52813,52815-52819,52822,52846,52850,52852-52853,52855-52857,52859,52924,52986,53128-53129,53190,53197,53199,53203-53204,53210-53211,53247,53249,53252,53267,53270,53293,53305,53344,53369,53427,53502-53504,53506,53777,54384,54494,54592,54599-54602,54604,54613,54764,54793,54806,55178,55626,56325,56862,56867,57154-57447,57541,57916,58151,58219,58633,58816,77555,77558-77560,77563-77565,77573
Added: svn:executable
524 + *
Index: trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php
——@@ -0,0 +1,22 @@
 2+<?php
 3+require_once( dirname(__FILE__).'/../commandLine.inc' );
 4+
 5+$bad = 0;
 6+$good = 0;
 7+foreach ( $wgLocalDatabases as $wiki ) {
 8+ $lb = wfGetLB( $wiki );
 9+ $db = $lb->getConnection( DB_SLAVE, array(), $wiki );
 10+ if ( $db->tableExists( 'blob_tracking' ) ) {
 11+ $notDone = $db->selectField( 'blob_tracking', '1',
 12+ array( 'bt_moved' => 0 ) );
 13+ if ( $notDone ) {
 14+ $bad++;
 15+ echo "$wiki\n";
 16+ } else {
 17+ $good++;
 18+ }
 19+ }
 20+ $lb->reuseConnection( $db );
 21+}
 22+echo "$bad wiki(s) incomplete\n";
 23+echo "$good wiki(s) complete\n";
Property changes on: trunk/extensions/WikimediaMaintenance/storage/testRctComplete.php
___________________________________________________________________
Added: svn:mergeinfo
124 Merged /trunk/phase3/maintenance/storage/testRctComplete.php:r52290,52402,52404,52718,52737,52759,52776,52791,52800,52808,52812-52813,52815-52819,52822,52846,52850,52852-52853,52855-52857,52859,52924,52986,53128-53129,53190,53197,53199,53203-53204,53210-53211,53247,53249,53252,53267,53270,53293,53305,53344,53369,53427,53502-53504,53506,53777,54384,54494,54592,54599-54602,54604,54613,54764,54793,54806,55178,55626,56325,56862,56867,57154-57447,57541,57916,58151,58219,58633,58816,77555,77558-77560,77563-77565,77573
225 Merged /branches/wmf-deployment/maintenance/storage/testRctComplete.php:r60970
326 Merged /branches/REL1_15/phase3/maintenance/storage/testRctComplete.php:r51646
427 Merged /branches/wmf/1.16wmf4/maintenance/storage/testRctComplete.php:r67177,69199,76243,77266
Added: svn:eol-style
528 + native
Index: trunk/extensions/WikimediaMaintenance/jeluf.php
——@@ -0,0 +1,103 @@
 2+<?php
 3+/**
 4+ * This script starts pending jobs.
 5+ *
 6+ * Usage:
 7+ * --maxjobs <num> (default 10000)
 8+ * --type <job_cmd>
 9+ *
 10+ * This program is free software; you can redistribute it and/or modify
 11+ * it under the terms of the GNU General Public License as published by
 12+ * the Free Software Foundation; either version 2 of the License, or
 13+ * (at your option) any later version.
 14+ *
 15+ * This program is distributed in the hope that it will be useful,
 16+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
 17+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 18+ * GNU General Public License for more details.
 19+ *
 20+ * You should have received a copy of the GNU General Public License along
 21+ * with this program; if not, write to the Free Software Foundation, Inc.,
 22+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 23+ * http://www.gnu.org/copyleft/gpl.html
 24+ *
 25+ * @ingroup Maintenance
 26+ */
 27+
 28+require_once( dirname(__FILE__) . '/Maintenance.php' );
 29+
 30+class RunJobs extends Maintenance {
 31+ public function __construct() {
 32+ parent::__construct();
 33+ $this->mDescription = "Run pending jobs";
 34+ $this->addOption( 'maxjobs', 'Maximum number of jobs to run', false, true );
 35+ $this->addOption( 'type', 'Type of job to run', false, true );
 36+ $this->addOption( 'procs', 'Number of processes to use', false, true );
 37+ }
 38+
 39+ public function memoryLimit() {
 40+ // Don't eat all memory on the machine if we get a bad job.
 41+ return "150M";
 42+ }
 43+
 44+ public function execute() {
 45+ global $wgTitle;
 46+ if ( $this->hasOption( 'procs' ) ) {
 47+ $procs = intval( $this->getOption('procs') );
 48+ if ( $procs < 1 || $procs > 1000 ) {
 49+ $this->error( "Invalid argument to --procs", true );
 50+ }
 51+ $fc = new ForkController( $procs );
 52+ if ( $fc->start( $procs ) != 'child' ) {
 53+ exit( 0 );
 54+ }
 55+ }
 56+ $maxJobs = $this->getOption( 'maxjobs', 10000 );
 57+ $type = $this->getOption( 'type', false );
 58+ $wgTitle = Title::newFromText( 'RunJobs.php' );
 59+ $dbw = wfGetDB( DB_MASTER );
 60+ $n = 0;
 61+ $conds = '';
 62+ if ($type !== false)
 63+ $conds = "job_cmd = " . $dbw->addQuotes($type);
 64+
 65+ while ( $dbw->selectField( 'job', 'job_id', $conds, 'runJobs.php' ) ) {
 66+ $offset=0;
 67+ for (;;) {
 68+ $job = ($type == false) ?
 69+ Job::pop($offset)
 70+ : Job::pop_type($type);
 71+
 72+ if ($job == false)
 73+ break;
 74+
 75+ wfWaitForSlaves( 5 );
 76+ $t = microtime( true );
 77+ $offset=$job->id;
 78+ $status = $job->run();
 79+ $t = microtime( true ) - $t;
 80+ $timeMs = intval( $t * 1000 );
 81+ if ( !$status ) {
 82+ $this->runJobsLog( $job->toString() . " t=$timeMs error={$job->error}" );
 83+ } else {
 84+ $this->runJobsLog( $job->toString() . " t=$timeMs good" );
 85+ }
 86+ if ( $maxJobs && ++$n > $maxJobs ) {
 87+ break 2;
 88+ }
 89+ }
 90+ }
 91+ }
 92+
 93+ /**
 94+ * Log the job message
 95+ * @param $msg String The message to log
 96+ */
 97+ private function runJobsLog( $msg ) {
 98+ $this->output( wfTimestamp( TS_DB ) . " $msg\n" );
 99+ wfDebugLog( 'runJobs', $msg );
 100+ }
 101+}
 102+
 103+$maintClass = "RunJobs";
 104+require_once( DO_MAINTENANCE );
Index: trunk/extensions/WikimediaMaintenance/jobs-loop.sh
——@@ -0,0 +1,41 @@
 2+#!/bin/bash
 3+
 4+trap 'kill %-; exit' SIGTERM
 5+[ ! -z "$1" ] && {
 6+ echo "starting type-specific job runner: $1"
 7+ type=$1
 8+}
 9+
 10+#types="htmlCacheUpdate sendMail enotifNotify uploadFromUrl fixDoubleRedirect renameUser"
 11+types="sendMail enotifNotify uploadFromUrl fixDoubleRedirect"
 12+
 13+cd `readlink -f /usr/local/apache/common/multiversion`
 14+while [ 1 ];do
 15+ # Do the prioritised types
 16+ moreprio=y
 17+ while [ -n "$moreprio" ] ; do
 18+ moreprio=
 19+ for type in $types; do
 20+ db=`php -n MWScript.php nextJobDB.php --wiki=aawiki --type="$type"`
 21+ if [ -n "$db" ]; then
 22+ echo "$db $type"
 23+ nice -n 20 php MWScript.php runJobs.php --wiki="$db" --procs=5 --type="$type" --maxtime=300 &
 24+ wait
 25+ moreprio=y
 26+ fi
 27+ done
 28+ done
 29+
 30+ # Do the remaining types
 31+ db=`php -n MWScript.php nextJobDB.php --wiki=aawiki`
 32+
 33+ if [ -z "$db" ];then
 34+ # No jobs to do, wait for a while
 35+ echo "No jobs..."
 36+ sleep 5
 37+ else
 38+ echo "$db"
 39+ nice -n 20 php MWScript.php runJobs.php --wiki="$db" --procs=5 --maxtime=300 &
 40+ wait
 41+ fi
 42+done
Property changes on: trunk/extensions/WikimediaMaintenance/jobs-loop.sh
___________________________________________________________________
Added: svn:mergeinfo
143 Merged /branches/wmf/1.16wmf4/maintenance/jobs-loop.sh:r67177,69199,76243,77266
244 Merged /trunk/phase3/maintenance/jobs-loop.sh:r52290,52402,52404,52718,52737,52759,52776,52791,52800,52808,52812-52813,52815-52819,52822,52846,52850,52852-52853,52855-52857,52859,52924,52986,53128-53129,53190,53197,53199,53203-53204,53210-53211,53247,53249,53252,53267,53270,53293,53305,53344,53369,53427,53502-53504,53506,53777,54384,54494,54592,54599-54602,54604,54613,54764,54793,54806,55178,55626,56325,56862,56867,57154-57447,57541,57916,58151,58219,58633,58816,77555,77558-77560,77563-77565,77573
345 Merged /branches/wmf-deployment/maintenance/jobs-loop.sh:r60970
446 Merged /branches/REL1_15/phase3/maintenance/jobs-loop.sh:r51646
Added: svn:eol-style
547 + native
Added: svn:executable
648 + *

Status & tagging log