r47799 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r47798‎ | r47799 | r47800 >
Date:13:04, 25 February 2009
Author:tstarling
Status:deferred
Tags:
Comment:
Proof of concept parallel processing with Danga Gearman and PEAR Net_Gearman.
Modified paths:
  • /trunk/phase3/maintenance/gearman (added) (history)
  • /trunk/phase3/maintenance/gearman/gearman.inc (added) (history)
  • /trunk/phase3/maintenance/gearman/gearmanRefreshLinks.php (added) (history)
  • /trunk/phase3/maintenance/gearman/gearmanWorker.php (added) (history)

Diff [purge]

Index: trunk/phase3/maintenance/gearman/gearmanRefreshLinks.php
@@ -0,0 +1,26 @@
 2+<?php
 3+
 4+$optionsWithArgs = array( 'fake-job' );
 5+
 6+require( dirname(__FILE__).'/../commandLine.inc' );
 7+require( dirname(__FILE__).'/gearman.inc' );
 8+
 9+if ( !$args ) {
 10+ $args = array( 'localhost' );
 11+}
 12+$client = new Net_Gearman_Client( $args );
 13+
 14+$dbr = wfGetDB( DB_SLAVE );
 15+$res = $dbr->select( 'page', array( 'page_namespace', 'page_title' ), false,
 16+ __METHOD__, array( 'LIMIT' => 2 ) );
 17+foreach ( $res as $row ) {
 18+ $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 19+ $params = array(
 20+ 'wiki' => wfWikiID(),
 21+ 'title' => $title->getPrefixedDBkey(),
 22+ 'command' => 'refreshLinks',
 23+ 'params' => false,
 24+ );
 25+ $client->mw_job( $params );
 26+}
 27+
Property changes on: trunk/phase3/maintenance/gearman/gearmanRefreshLinks.php
___________________________________________________________________
Name: svn:eol-style
128 + native
Index: trunk/phase3/maintenance/gearman/gearman.inc
@@ -0,0 +1,91 @@
 2+<?php
 3+
 4+require( 'Net/Gearman/Client.php' );
 5+require( 'Net/Gearman/Worker.php' );
 6+
 7+class MWGearmanJob extends Net_Gearman_Job_Common {
 8+ function switchWiki( $wiki, $jobParams ) {
 9+ echo "Switching to $wiki\n";
 10+ $php = readlink( '/proc/' . posix_getpid() . '/exe' );
 11+ $args = array( $_SERVER['PHP_SELF'],
 12+ '--wiki', $wiki,
 13+ '--fake-job', serialize( $jobParams ) );
 14+ $args = array_merge( $args, $GLOBALS['args'] );
 15+ pcntl_exec( $php, $args, $_ENV );
 16+ echo "Error running exec\n";
 17+ }
 18+
 19+ function run( $params ) {
 20+ if ( wfWikiID() !== $params['wiki'] ) {
 21+ $this->switchWiki( $params['wiki'], $params );
 22+ }
 23+ self::runNoSwitch( $params );
 24+ }
 25+
 26+ static function runNoSwitch( $params ) {
 27+ echo implode( ' ', $params ) . "\n";
 28+ $title = Title::newFromText( $params['title'] );
 29+ $mwJob = Job::factory( $params['command'], $title, $params['params'] );
 30+ return $mwJob->run();
 31+ }
 32+}
 33+
 34+class NonScaryGearmanWorker extends Net_Gearman_Worker {
 35+
 36+ /**
 37+ * Copied from Net_Gearman_Worker but with the scary "run any PHP file in
 38+ * the filesystem" feature removed.
 39+ */
 40+ protected function doWork($socket) {
 41+ Net_Gearman_Connection::send($socket, 'grab_job');
 42+
 43+ $resp = array('function' => 'noop');
 44+ while (count($resp) && $resp['function'] == 'noop') {
 45+ $resp = Net_Gearman_Connection::blockingRead($socket);
 46+ }
 47+
 48+ if (in_array($resp['function'], array('noop', 'no_job'))) {
 49+ return false;
 50+ }
 51+
 52+ if ($resp['function'] != 'job_assign') {
 53+ throw new Net_Gearman_Exception('Holy Cow! What are you doing?!');
 54+ }
 55+
 56+ $name = $resp['data']['func'];
 57+ $handle = $resp['data']['handle'];
 58+ $arg = array();
 59+
 60+ if (isset($resp['data']['arg']) &&
 61+ Net_Gearman_Connection::stringLength($resp['data']['arg'])) {
 62+ $arg = json_decode($resp['data']['arg'], true);
 63+ }
 64+
 65+ ### START MW DIFFERENT BIT
 66+ if ( $name != 'mw_job' ) {
 67+ throw new Net_Gearman_Job_Exception('Invalid function');
 68+ }
 69+ $job = new MWGearmanJob($socket, $handle);
 70+ ### END MW DIFFERENT BIT
 71+
 72+ try {
 73+ $this->start($handle, $name, $arg);
 74+ $res = $job->run($arg);
 75+ if (!is_array($res)) {
 76+ $res = array('result' => $res);
 77+ }
 78+
 79+ $job->complete($res);
 80+ $this->complete($handle, $name, $res);
 81+ } catch (Net_Gearman_Job_Exception $e) {
 82+ $job->fail();
 83+ $this->fail($handle, $name, $e);
 84+ }
 85+
 86+ // Force the job's destructor to run
 87+ $job = null;
 88+
 89+ return true;
 90+ }
 91+}
 92+
Property changes on: trunk/phase3/maintenance/gearman/gearman.inc
___________________________________________________________________
Name: svn:eol-style
193 + native
Index: trunk/phase3/maintenance/gearman/gearmanWorker.php
@@ -0,0 +1,19 @@
 2+<?php
 3+
 4+$optionsWithArgs = array( 'fake-job' );
 5+require( dirname(__FILE__).'/../commandLine.inc' );
 6+require( dirname(__FILE__).'/gearman.inc' );
 7+
 8+if ( !$args ) {
 9+ $args = array( 'localhost' );
 10+}
 11+
 12+if ( isset( $options['fake-job'] ) ) {
 13+ $params = unserialize( $options['fake-job'] );
 14+ MWGearmanJob::runNoSwitch( $params );
 15+}
 16+
 17+$worker = new NonScaryGearmanWorker( $args );
 18+$worker->addAbility( 'mw_job' );
 19+$worker->beginWork();
 20+
Property changes on: trunk/phase3/maintenance/gearman/gearmanWorker.php
___________________________________________________________________
Name: svn:eol-style
121 + native

Status & tagging log