r101636 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r101635‎ | r101636 | r101637 >
Date:17:13, 2 November 2011
Author:hashar
Status:deferred (Comments)
Tags:ci 
Comment:
MW fetcher and installer for TestSwarm

Designed by Timo and Antoine and for use with MediaWiki software
with TestSwarm as a frontend.

Really experimental for now. This is the first working draft.
Modified paths:
  • /trunk/tools/testswarm/scripts/testswarm-mw-fetcher-run.php (added) (history)
  • /trunk/tools/testswarm/scripts/testswarm-mw-fetcher.php (added) (history)

Diff [purge]

Index: trunk/tools/testswarm/scripts/testswarm-mw-fetcher.php
——@@ -0,0 +1,462 @@
 2+<?php
 3+/**
 4+ * Script to prepare a MediaWiki-install from svn for TestSwarm testing.
 5+ *
 6+ * As of November 2nd 2011, this is still a work in progress.
 7+ *
 8+ * Latest version can be found in the Mediawiki repository under
 9+ * /trunk/tools/testswarm/
 10+ *
 11+ * Based on http://svn.wikimedia.org/viewvc/mediawiki/trunk/tools/testswarm/scripts/testswarm-mediawiki-svn.php?revision=94359&view=markup
 12+ * (which only did a static dump of /resources and /tests/qunit).
 13+ *
 14+ * @author Timo Tijhof, 2011
 15+ * @author Antoine "hashar" Musso, 2011
 16+ */
 17+
 18+/**
 19+ * One class doing everything! :D
 20+ *
 21+ * Subversion calls are made using the svn binary so we do not need
 22+ * to install any PECL extension.
 23+ *
 24+ * @todo We might want to abstract svn commands to later use git
 25+ * @todo FIXME: Get the classes/function implied from MediaWiki somehow.
 26+ *
 27+ * @example:
 28+ * @code
 29+ * $options = array(
 30+ * 'root' => '',
 31+ * 'url' => 'http://svn.wikimedia.org/svnroot/mediawiki/trunk/phase3',
 32+ * );
 33+ * $fetcher = new TestSwarmMWFetcher( $options );
 34+ * $fetcher->tryInstallNextRev();
 35+ * @endcode
 36+ */
 37+class TestSwarmMWFetcher {
 38+
 39+ /** Base path to run into */
 40+ protected $root;
 41+ /** URL to a subversion repository as supported by the Subversion cli */
 42+ protected $url;
 43+ /** subversion command line utility */
 44+ protected $svnCmd = '/usr/bin/svn';
 45+ /** whether to enable debugging */
 46+ protected $debugEnabled = false;
 47+ /** hold previous path when chdir() to a checkout directory */
 48+ protected $savedPath = null;
 49+ /** Minimum revision to start with. At least 1 */
 50+ protected $minRevision = 1;
 51+
 52+ /**
 53+ * Init the testswarm fetcher.
 54+ *
 55+ * @param @options Array: Required options are:
 56+ * 'root' => root path where all stuff happens
 57+ * 'url' => URL for the repository
 58+ * Other options:
 59+ * 'svnCmd' => path/to/svn (default: /usr/bin/svn)
 60+ * 'debug' => true/false
 61+ * 'minrevision' => int (revision to start at)
 62+ */
 63+ function __construct( $options = array() ) {
 64+
 65+ // Verify we have been given required options
 66+ if( !isset( $options['root'] )
 67+ && !isset( $options['url'] )
 68+ ) {
 69+ throw new Exception( __METHOD__ . ": " . __CLASS__ . " constructor must be passed 'root' and 'url' options\n" );
 70+ }
 71+ $this->root = $options['root'];
 72+ $this->url = $options['url'];
 73+
 74+ // and now the optional options
 75+ if( isset($options['svnCmd'] ) ) {
 76+ $this->svnCmd = $options['svnCmd'];
 77+ }
 78+ if( isset($options['debug'] ) ) {
 79+ $this->debugEnabled = true;
 80+ }
 81+ if( isset($options['minrevision'] ) ) {
 82+ if( $options['minrevision'] < 1 ) {
 83+ # minrevision = 0 will just screw any assumption made in this script.
 84+ # so we really do not want it.
 85+ throw new Exception( __METHOD__ . ": " . __CLASS__ . " option 'minrevision' must be >= 1 \n" );
 86+ }
 87+ $this->minRevision = $options['minrevision'];
 88+ }
 89+ }
 90+
 91+ /**
 92+ * Try to install the next revision after our latest install.
 93+ * This is the main entry point after construction.
 94+ */
 95+ function tryInstallNextRev() {
 96+ // Setup checkouts dir if it does not exist yet (happens on initial run).
 97+ $checkouts = "{$this->root}/checkouts";
 98+ if( !file_exists( $checkouts ) ) {
 99+ $this->mkdir( $checkouts );
 100+ }
 101+
 102+ // Now find out the next revision in the remote repository
 103+ $next = $this->getNextRevisionId();
 104+ if ( !$next ) {
 105+ $this->debug( __METHOD__ . " no next revision." );
 106+ return false;
 107+ } else {
 108+ // And install it
 109+ return $this->doInstallById( $next );
 110+ }
 111+ }
 112+
 113+ /**
 114+ * This is the main function doing checkout and installation for
 115+ * a given rev.
 116+ *
 117+ * @param $id integer: Revision id to install
 118+ * @return
 119+ */
 120+ function doInstallById( $id ) {
 121+ if( !is_int( $id ) ) {
 122+ throw new Exception( __METHOD__ . " passed a non integer revision number\n" );
 123+ }
 124+
 125+ $this->doCheckout( $id );
 126+ $this->doInstall( $id );
 127+ $this->doAppendSettings( $id );
 128+
 129+ # TODO:
 130+ // get list of tests (see the current file in svn/trunk/tools)
 131+ // request to testswarm to add jobs to run these tests
 132+ // --> 'api' POST request to TestSwarm/addjob.php (with login/auth token)
 133+ }
 134+
 135+ /**
 136+ * Checkout a given revision in our specific tree.
 137+ * Throw an exception if anything got wrong.
 138+ * @todo Output is not logged.
 139+ *
 140+ * @param $id integer: Revision id to checkout.
 141+ */
 142+ function doCheckout( $id ){
 143+ $this->msg( "Checking out r{$id}" );
 144+
 145+ // create checkout directory
 146+ $revPath = self::getPath( 'mw', $id );
 147+ $this->mkdir( $revPath );
 148+
 149+ # TODO FIXME : we might want to log the output of svn commands
 150+ $cmd = "{$this->svnCmd} checkout {$this->url}@r{$id} {$revPath}";
 151+ $this->exec( $cmd, $retval );
 152+ if( $retval !== 0 ) {
 153+ throw new Exception(__METHOD__." error running subversion checkout.\n" );
 154+ }
 155+
 156+ // TODO handle errors for above commands.
 157+ // $this->getPath( 'log' );
 158+ }
 159+
 160+ /**
 161+ * Install a given revision.
 162+ *
 163+ * @param $id integer: Revision id to run the installer for.
 164+ */
 165+ function doInstall( $id ) {
 166+ $this->msg( "Installing r{$id}" );
 167+
 168+ # Create database directory (needed on initial run)
 169+ $this->mkdir(
 170+ $this->getPath( 'db', $id )
 171+ );
 172+
 173+ # Erase MW_INSTALL_PATH which would interact with the install script
 174+ putenv( "MW_INSTALL_PATH" );
 175+
 176+ // Now simply run the CLI installer:
 177+ $cmd = "php {$this->getPath( 'mw', $id )}/maintenance/install.php \
 178+ --dbname=testwarm_mw_r{$id} \
 179+ --dbtype=sqlite \
 180+ --dbpath={$this->getPath( 'db', $id )} \
 181+ --pass=testpass \
 182+ --showexceptions=true \
 183+ --confpath={$this->getPath( 'mw', $id )} \
 184+ WIKINAME \
 185+ ADMINNAME
 186+ ";
 187+ $output = $this->exec( $cmd, $retval );
 188+
 189+ print "Installer output for r{$id}:\n";
 190+ print $output;
 191+
 192+ if( $retval !== 0 ) {
 193+ throw new Exception(__METHOD__." error running MediaWiki installer.\n" );
 194+ }
 195+ }
 196+
 197+ /**
 198+ * TODO: implement :-)
 199+ * @param $id integer: Revision id to append settings to.
 200+ */
 201+ function doAppendSettings( $id ) {
 202+ $this->msg( "Appending settings for r{$id} installation (not implemented)" );
 203+ return true;
 204+
 205+ # Notes for later implementation:
 206+
 207+ // append to mwPath/LocalSettings.php
 208+ // -- contents of LocalSettings.tpl.php
 209+ // -- require_once( '{$this->getPath('globalsettings')}' );"
 210+
 211+ /**
 212+ * Possible additional common settings to append to LocalSettings after install:
 213+ * See gerrit integration/jenkins.git :
 214+ * https://gerrit.wikimedia.org/r/gitweb?p=integration/jenkins.git;a=tree;f=jobs/MediaWiki-phpunit;hb=HEAD
 215+ *
 216+ * $wgShowExceptionDetails = true;
 217+ * $wgShowSQLErrors = true;
 218+ * #$wgDebugLogFile = dirname( __FILE__ ) . '/build/debug.log';
 219+ * $wgDebugDumpSql = true;
 220+ */
 221+ }
 222+
 223+ /**
 224+ * Utility function to log a message for a given id
 225+ *
 226+ * @param $msg String message to log. Will be prefixed with date("c")
 227+ * @param $id Integer Revision id.
 228+ */
 229+ function log( $msg, $id ) {
 230+ $file = $this->getLogFile( $id );
 231+ // append stuff to logfile
 232+ fopen( $file, "w+" );
 233+ fwrite( date("c").$msg, $file );
 234+ fclose( $file );
 235+ }
 236+
 237+ /**
 238+ * Utility function to output a dated message
 239+ *
 240+ * @param $msg String message to show. Will be prefixed with date("c")
 241+ */
 242+ function msg( $msg ) {
 243+ print date("c") . " $msg\n";
 244+ }
 245+
 246+ /**
 247+ * Print a message to STDOUT when debug mode is enabled
 248+ * Messages are prefixed with "DEBUG> "
 249+ * Multi lines messages will be prefixed as well.
 250+ *
 251+ * @param $msg string Message to print
 252+ */
 253+ function debug( $msg ) {
 254+ if( !$this->debugEnabled ) {
 255+ return;
 256+ }
 257+ foreach( explode( "\n", $msg ) as $line ) {
 258+ print "DEBUG> $line\n";
 259+ }
 260+ }
 261+
 262+ /** unused / unneeded? @author Antoine Musso */
 263+ function changePath( $path ) {
 264+ if( $this->savedPath !== null ) {
 265+ throw new Exception( __METHOD__ . " called but a saved path exist '" . $this->savedPath ."' Did you forget to call restorePath()?\n");
 266+ }
 267+ $this->debug( "chdir( $path )" );
 268+
 269+ $oldPath = getcwd();
 270+ if( chdir( $path ) ) {
 271+ $this->savedPath = $oldPath;
 272+ } else {
 273+ throw new Exception( __METHOD__ . " failed to chdir() to $path\n" );
 274+ }
 275+ }
 276+
 277+ /** unused / unneeded? @author Antoine Musso */
 278+ function restorePath() {
 279+ if( $this->savedPath === null ) {
 280+ return false;
 281+ }
 282+ chdir( $this->savedPath );
 283+ }
 284+
 285+ /**
 286+ * Execute a command!
 287+ * Ripped partially from wfExec()
 288+ * throw an exception if anything goes wrong.
 289+ *
 290+ * @param $cmd string Command which will be passed as is (no escaping FIXME)
 291+ * @param &$retval reference Will be given the command exit level
 292+ * @return Command output.
 293+ */
 294+ function exec( $cmd, &$retval = 0 ) {
 295+ $this->debug( __METHOD__ . " $cmd" );
 296+
 297+ // Pass command to shell and use ob to fetch the output
 298+ ob_start();
 299+ passthru( $cmd, $retval );
 300+ $output = ob_get_contents();
 301+ ob_end_clean();
 302+
 303+ if( $retval == 127 ) {
 304+ throw new Exception( __METHOD__ . "probably missing executable. Check env.\n" );
 305+ }
 306+
 307+ return $output;
 308+ }
 309+
 310+ /**
 311+ * Create a directory including parents
 312+ *
 313+ * @param $path String Path to create ex: /tmp/my/foo/bar
 314+ */
 315+ function mkdir( $path ) {
 316+ $this->debug( __METHOD__ . " mkdir( $path )" );
 317+ if(!file_exists( $path ) ) {
 318+ if( mkdir( $path, 0777, true ) ) {
 319+ $this->debug( __METHOD__ . ": Created directory $path" );
 320+ } else {
 321+ throw new Exception( __METHOD__ . " Error creating directory $path\n" );
 322+ }
 323+ } else {
 324+ $this->debug( __METHOD__ . " mkdir( $path ). Path already exist." );
 325+ }
 326+ }
 327+
 328+ /** unused / unneeded? @author Antoine Musso */
 329+ function getLogFile( $id ) {
 330+ return $logfile = $this->getPath( 'log', $id ). "/debug.log";
 331+ }
 332+
 333+ /** UTILITY FUNCTIONS **/
 334+
 335+ /**
 336+ * Get next changed revision for a given checkout
 337+ * @return String|Boolean: false if nothing changed, else the upstream revision just after.
 338+ */
 339+ function getNextRevisionId() {
 340+ $cur = $this->getCurrentRevisionId();
 341+ if( $cur === null ) {
 342+ $this->debug( __METHOD__ . " checkouts dir empty? Looking up remote repo." );
 343+ $next = $this->getFirstRevision();
 344+ } else {
 345+ $next = $this->getRevFollowing( $cur );
 346+ }
 347+
 348+ $this->debug( __METHOD__ . " returns $next" );
 349+ return $next;
 350+ }
 351+
 352+ function getFirstRevision() {
 353+ $start = $this->minRevision - 1;
 354+ $firstRevision = $this->getRevFollowing( $start );
 355+ $this->debug( __METHOD__ . " using first revision '$firstRevision'" );
 356+ return $firstRevision;
 357+ }
 358+
 359+ function getRevFollowing( $id ) {
 360+ $nextRev = $id + 1;
 361+ // FIXME looking up for 1:HEAD takes a loooooooongg time
 362+ $cmd = "{$this->svnCmd} log -q -r{$nextRev}:HEAD --limit 1 {$this->url}";
 363+ $output = $this->exec( $cmd, $retval );
 364+
 365+ if( $retval !== 0 ) {
 366+ throw new Exception(__METHOD__." error running subversion log.\n" );
 367+ }
 368+
 369+ preg_match( "/r(\d+)/m", $output, $m );
 370+ $followingRev = (int) $m[1];
 371+ if( $followingRev === 0 ) {
 372+ throw new Exception( __METHOD__ . " remote gave us non integer revision: '{$m[1]}'\n" );
 373+ }
 374+ return $followingRev;
 375+ }
 376+ /**
 377+ * Get latest revision fetched in the working copy.
 378+ * @return integer
 379+ */
 380+ function getCurrentRevisionId() {
 381+ $checkoutsDir = dirname( $this->getPath( 'mw', 0 ) );
 382+ // scandir sort in descending order if passing a nonzero value
 383+ // PHP 5.4 accepts constant SCANDIR_SORT_DESCENDING
 384+ $dirs = scandir( $checkoutsDir, 1 );
 385+ $this->debug( "From '$checkoutsDir' Got directories:\n".implode("\n", $dirs ) );
 386+ // On first run, we will have to take care of that.
 387+ if ( $dirs[0][0] === 'r' ) {
 388+ return substr( $dirs[0], 1 );
 389+ } else {
 390+ return null;
 391+ }
 392+ }
 393+
 394+ /**
 395+ * This function is where most of the directory layout is kept
 396+ * All other methods should use getPath whenever they are looking for a path
 397+ *
 398+ * @param $type string: Resource to fetch:
 399+ * 'db': path to DB dir to put testwarm_mw_r123.sqlite file in
 400+ * 'mw': path to MW dir
 401+ * 'globalsettings': path to global settings file
 402+ * 'localsettingstpl': path to LocalSettings.php template file
 403+ * 'log': path to log file
 404+ * @param $id integer: Revision number.
 405+ * @return Full path to ressource or 'false'
 406+ */
 407+ function getPath( $type, $id ) {
 408+ if ( !in_array( $type, array( 'globalsettings', 'localsettingstpl' ) )
 409+ && !is_int( $id ) ) {
 410+ throw new Exception( __METHOD__ . "given non numerical revision" );
 411+ }
 412+
 413+ switch ( $type ) {
 414+
 415+ case 'db':
 416+ return "{$this->root}/dbs/";
 417+
 418+ case 'mw':
 419+ return "{$this->root}/checkouts/r{$id}";
 420+
 421+ case 'globalsettings':
 422+ return "{$this->root}/conf/GlobalSettings.php";
 423+
 424+ case 'localsettingstpl':
 425+ return "{$this->root}/conf/LocalSettings.tpl.php";
 426+
 427+ case 'log':
 428+ return "{$this->root}/log/r{$id}";
 429+
 430+ default:
 431+ return false;
 432+ }
 433+ }
 434+}
 435+
 436+# Remaning notes from design session. Leave them for now unless you are Timo.
 437+/** GENERAL:
 438+format: php
 439+Directory structure:
 440+dbs/
 441+testswarm_mw_r123.sqlite
 442+conf/
 443+GlobalSettings.php
 444+// global conf, empty in most cases. Could be used to globally do something important
 445+LocalSettingsTemplate.php
 446+// copied/appended to LocalSettings.php that install.php makes
 447+checkouts/
 448+// publicly available through Apache; symlinked to /var/www/testswarm-mw
 449+r123/
 450+ */
 451+
 452+/** INIT:
 453+get latest svn revision number for trunk/phase3
 454+you want the next changed revision. Not the latest.
 455+svn log -r BASE:HEAD -l 2 -q
 456+I didn't even know that was an option, even better (no risk of missing a rev if > 1 commit between runs). Awesome!
 457+If BASE is at HEAD, you will only get one line though. So need to verify!
 458+Getting the latest checked out directory is all about ls -1 | tail -1
 459+check: already checked out ? Abort otherwise
 460+(file_exists(checkouts/r...)
 461+svn checkout (or export) into the checkouts/r..
 462+ */
 463+
Property changes on: trunk/tools/testswarm/scripts/testswarm-mw-fetcher.php
___________________________________________________________________
Added: svn:eol-style
1464 + native
Index: trunk/tools/testswarm/scripts/testswarm-mw-fetcher-run.php
——@@ -0,0 +1,42 @@
 2+<?php
 3+/**
 4+ * Testswarm fetcher example.
 5+ *
 6+ * Licensed under GPL version 2
 7+ *
 8+ * @author Antoine "hashar" Musso © 2011
 9+ */
 10+require_once( "testswarm-mw-fetcher.php" );
 11+
 12+// Choose a mode below and the switch structure will forge options for you!
 13+#$mode = 'dev';
 14+$mode = 'preprod';
 15+#$mode = 'prod';
 16+
 17+
 18+# Magic stuff for lazy people
 19+switch( $mode ) {
 20+ # Options for local debuggings
 21+ case 'dev': $options = array(
 22+ 'debug' => true,
 23+ 'root' => '/tmp/tsmw',
 24+ 'url' => 'http://svn.wikimedia.org/svnroot/mediawiki/trunk/tools/testswarm/scripts',
 25+ 'minrevision' => 88439, # will not fetch anything before that rev
 26+ ); break;
 27+
 28+ # Options fetching from phase3. Debug on.
 29+ case 'preprod':
 30+ $options = array(
 31+ 'debug' => true,
 32+ 'root' => '/tmp/tsmw-trunk',
 33+ 'url' => 'http://svn.wikimedia.org/svnroot/mediawiki/trunk/phase3',
 34+ 'minrevision' => 101591,
 35+ ); break;
 36+
 37+ default:
 38+ print "Mode $mode unimplemented. Please edit ".__FILE__."\n";
 39+ exit( 1 );
 40+}
 41+
 42+$fetcher = new TestSwarmMWFetcher( $options );
 43+$fetcher->tryInstallNextRev();
Property changes on: trunk/tools/testswarm/scripts/testswarm-mw-fetcher-run.php
___________________________________________________________________
Added: svn:eol-style
144 + native

Follow-up revisions

RevisionCommit summaryAuthorDate
r101744[JSTesting] partial rewrite of TestSwarmMWFetcher...krinkle23:17, 2 November 2011
r101745[JSTesting] fix require_once path for testswarm-mw-fetcher.php...krinkle23:21, 2 November 2011

Comments

#Comment by Krinkle (talk | contribs)   20:50, 2 November 2011

Nice work. Working on a follow-up revision currently splitting this up into two classes (one main class for the management, and one class for the installing)

#Comment by Hashar (talk | contribs)   20:58, 2 November 2011

That make sense. I guess I spend the afternoon just pilling up methods one after the other and did not take care to properly split them.

All the subversion magic could probably get splir too so we can create a new one later on for git.

I hope it works more or less on your setup.

Status & tagging log