r91890 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r91889‎ | r91890 | r91891 >
Date:17:42, 11 July 2011
Author:krinkle
Status:deferred
Tags:
Comment:
Sync testswarm cli scripts:
- Note these are not being used by TestSwarm but for TestSwarm. This is the script in cronjob that checks svn and checks out revisions to the disk and submits jobs and runs to the Swarm after authenticating.

Rough summary of changes:
* Using cURL from PHP instead of the command-line. Cleaner and more flexible this way.
* Executed the script from another file in order to catch/log errors if the main file has fatals or syntax errors (instead of giving a blank)
* Add basic lock-file check to avoid checking out while another update is still running. Should never happen (a cronjob loop is long enough), but just in case (or when running manually from CLI)
Modified paths:
  • /trunk/tools/testswarm/scripts/do-svncheckaddjob.php (added) (history)
  • /trunk/tools/testswarm/scripts/testswarm-mediawiki-svn.php (modified) (history)

Diff [purge]

Index: trunk/tools/testswarm/scripts/testswarm-mediawiki-svn.php
——@@ -24,10 +24,6 @@
2525 }
2626 }
2727
28 -function stopper( $msg ) {
29 - die( "$msg\n" );
30 -}
31 -
3228 function getTestUrl( $rev, $suite ) {
3329 $url = $GLOBALS['testUrlPattern'];
3430 $url = str_replace( '$1', rawurlencode( $rev ), $url );
——@@ -110,6 +106,12 @@
111107 # 'jquery.baz.js',
112108 #);
113109
 110+$curlOpts = array(
 111+ CURLOPT_RETURNTRANSFER => 1,
 112+ CURLOPT_USERAGENT => 'TestSwarm/20110511 (Wikimedia Toolserver; toolserver.org/~krinkle) Contact/krinkle@toolserver.org',
 113+ CURLOPT_POST => true,
 114+);
 115+
114116 ########### NO NEED TO CONFIGURE BELOW HERE ############
115117
116118 /**
——@@ -117,130 +119,155 @@
118120 * ----------
119121 */
120122
121 -# Get latest revision number of HEAD for QUnit tests dir and Resources dir
122 -$svnHeadRevs = array(
123 - 'tests' => null,
124 - 'resources' => null,
125 -);
126 -$svnHeadRevTop = null;
127 -
128 -foreach ( array(
129 - 'tests' => $svnCoRepoInfo['rootBase'] . '/' . $svnCoRepoInfo['qunitDir'],
130 - 'resources' => $svnCoRepoInfo['rootBase'] . '/' . $svnCoRepoInfo['resourcesDir'],
131 -) as $dirKey => $dirUrl ) {
132 -
133 - $tmpCmd = array();
134 -
135 - exec( "svn info $dirUrl", $tmpCmd['output'], $tmpCmd['return'] );
 123+function doingStuff(){
 124+ extract($GLOBALS);
136125
137 - if ( is_array( $tmpCmd['output'] ) && count( $tmpCmd['output'] ) ) {
138 -
139 - foreach( $tmpCmd['output'] as $cmdLine ) {
140 -
141 - $lineParts = explode( ':', $cmdLine, 2 );
142 - if ( trim( $lineParts[0] ) == 'Last Changed Rev' ) {
143 -
144 - $svnHeadRevs[$dirKey] = trim( $lineParts[1] );
145 - break;
 126+ # Get latest revision number of HEAD for QUnit tests dir and Resources dir
 127+ $svnHeadRevs = array(
 128+ 'tests' => null,
 129+ 'resources' => null,
 130+ );
 131+ $svnHeadRevTop = null;
 132+
 133+ foreach ( array(
 134+ 'tests' => $svnCoRepoInfo['rootBase'] . '/' . $svnCoRepoInfo['qunitDir'],
 135+ 'resources' => $svnCoRepoInfo['rootBase'] . '/' . $svnCoRepoInfo['resourcesDir'],
 136+ ) as $dirKey => $dirUrl ) {
 137+
 138+ $tmpCmd = array();
 139+
 140+ exec( "svn info $dirUrl", $tmpCmd['output'], $tmpCmd['return'] );
 141+
 142+ if ( is_array( $tmpCmd['output'] ) && count( $tmpCmd['output'] ) ) {
 143+
 144+ foreach( $tmpCmd['output'] as $cmdLine ) {
 145+
 146+ $lineParts = explode( ':', $cmdLine, 2 );
 147+ if ( trim( $lineParts[0] ) == 'Last Changed Rev' ) {
 148+
 149+ $svnHeadRevs[$dirKey] = trim( $lineParts[1] );
 150+ break;
 151+ }
 152+
 153+ unset( $cmdLine, $lineParts );
146154 }
147 -
148 - unset( $cmdLine, $lineParts );
149155 }
 156+ unset( $tmpCmd, $key, $dirUrl );
150157 }
151 - unset( $tmpCmd, $key, $dirUrl );
152 -}
153 -
154 -if ( empty( $svnHeadRevs['tests'] ) || empty( $svnHeadRevs['resources'] ) ) {
155 - die("Problem getting svn info.");
156 -}
157 -
158 -# Determine the highest of each
159 -$svnHeadRevTop = max( intval( $svnHeadRevs['tests'] ), intval( $svnHeadRevs['resources'] ) );
160 -
161 -# Check out a specific revision
162 -# We're doing a sparse checkout of phase3
163 -# and get /resources and /tests/qunit
164 -
165 -$revTargetTmpDir = "$svnCoTargetDir/r$svnHeadRevTop";
166 -
167 -if ( is_dir( $svnCoTargetDir ) ) {
168 - // Already checked out ?
169 - if ( is_dir( "$revTargetTmpDir" ) ) {
170 - stopper("Last revision (r$svnHeadRevTop) has been done already. Skipping this loop.");
 158+
 159+ if ( empty( $svnHeadRevs['tests'] ) || empty( $svnHeadRevs['resources'] ) ) {
 160+ return "Problem getting svn info.";
171161 }
172 - $cmdExc = array();
173 - $cmd = null;
174 -
175 - // Checkout empty root of mediawiki
176 - $cmd = "svn checkout -r $svnHeadRevTop {$svnCoRepoInfo['rootBase']} $revTargetTmpDir --depth empty";
177 - logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
178 - logger( print_r( $cmdExc['output'], true ) );
179 -
180 - // Checkout full depth of the resources directory
181 - $cmd = "svn update -r $svnHeadRevTop --set-depth infinity $revTargetTmpDir/{$svnCoRepoInfo['resourcesDir']}";
182 - logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
183 - logger( print_r( $cmdExc['output'], true ) );
184 -
185 - // Checkout empty qunit's parent directory
186 - $cmd = "svn update -r $svnHeadRevTop --set-depth empty $revTargetTmpDir/{$svnCoRepoInfo['qunitBase']}";
187 - logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
188 - logger( print_r( $cmdExc['output'], true ) );
189 -
190 - // Checkout full depth of qunit directory
191 - $cmd = "svn update -r $svnHeadRevTop --set-depth infinity $revTargetTmpDir/{$svnCoRepoInfo['qunitDir']}";
192 - logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
193 - logger( print_r( $cmdExc['output'], true ) );
194 -
195 - unset( $cmd, $cmdExc );
196 -
197 -} else {
198 - stopper("Problem locating temporary checkout directory.");
199 -}
200 -
201 -# Get array of modules
202 -
203 -$unitDir = glob( "$revTargetTmpDir/{$svnCoRepoInfo['qunitDir']}/suites/resources/*/*.js" );
204 -$suites = array_map( 'basename', $unitDir );
205 -
206 -# Add jobs
207 -
208 -if ( true ) {
209 -
210 - $params = array(
211 - "state" => "addjob",
212 - "output" => "dump",
213 - "user" => $userName,
214 - "max" => $testMaxRuns,
215 - "job_name" => str_replace( '$1', $svnHeadRevTop, $jobNamePattern ),
216 - "browsers" => $browsers,
217 - "auth" => $userAuthToken
218 - );
219162
220 - $query = http_build_query( $params );
221 -
222 - foreach ( $suites as $suite ) {
223 - $query .= "&suites[]=" . rawurlencode( $suite ) .
224 - "&urls[]=" . getTestUrl( $svnHeadRevTop, $suite );
 163+ # Determine the highest of each
 164+ $svnHeadRevTop = max( intval( $svnHeadRevs['tests'] ), intval( $svnHeadRevs['resources'] ) );
 165+
 166+ # Check out a specific revision
 167+ # We're doing a sparse checkout of phase3
 168+ # and get /resources and /tests/qunit
 169+
 170+ $revTargetTmpDir = "$svnCoTargetDir/r$svnHeadRevTop";
 171+
 172+ if ( is_dir( $svnCoTargetDir ) ) {
 173+ // Already checked out ?
 174+ if ( is_dir( "$revTargetTmpDir" ) ) {
 175+ return "Last revision (r$svnHeadRevTop) has been done already. Skipping this loop.";
 176+ }
 177+ $cmdExc = array();
 178+ $cmd = null;
 179+
 180+ // Checkout empty root of mediawiki
 181+ $cmd = "svn checkout -r $svnHeadRevTop {$svnCoRepoInfo['rootBase']} $revTargetTmpDir --depth empty";
 182+ logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
 183+ logger( print_r( $cmdExc['output'], true ) );
 184+
 185+ // Checkout full depth of the resources directory
 186+ $cmd = "svn update -r $svnHeadRevTop --set-depth infinity $revTargetTmpDir/{$svnCoRepoInfo['resourcesDir']}";
 187+ logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
 188+ logger( print_r( $cmdExc['output'], true ) );
 189+
 190+ // Checkout empty qunit's parent directory
 191+ $cmd = "svn update -r $svnHeadRevTop --set-depth empty $revTargetTmpDir/{$svnCoRepoInfo['qunitBase']}";
 192+ logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
 193+ logger( print_r( $cmdExc['output'], true ) );
 194+
 195+ // Checkout full depth of qunit directory
 196+ $cmd = "svn update -r $svnHeadRevTop --set-depth infinity $revTargetTmpDir/{$svnCoRepoInfo['qunitDir']}";
 197+ logger( $cmd ); exec( $cmd, $cmdExc['output'], $cmdExc['return'] );
 198+ logger( print_r( $cmdExc['output'], true ) );
 199+
 200+ unset( $cmd, $cmdExc );
 201+
 202+ } else {
 203+ return "Problem locating temporary checkout directory.";
225204 }
226 -
227 - logger( "curl -d \"$query\" $swarmUrl" );
228 -
229 - $curlPostCMD = array();
230 - exec( "curl -d \"$query\" $swarmUrl", $curlPostCMD['output'], $curlPostCMD['return'] );
231 -
232 - logger( "Results: {$curlPostCMD['output']}" );
233 -
234 - if ( $curlPostCMD['output'] ) {
235 - file_put_contents(
236 - __DIR__ . '/testswarm-mediawiki-svn.log',
237 - '[' . date('r') . '] ' . implode( ' \\', $curlPostCMD['output'] ) . "\n",
238 - FILE_APPEND
 205+
 206+ # Get array of modules
 207+
 208+ $unitDir = glob( "$revTargetTmpDir/{$svnCoRepoInfo['qunitDir']}/suites/resources/*/*.js" );
 209+ $suites = array_map( 'basename', $unitDir );
 210+
 211+ # Add jobs
 212+
 213+ if ( true ) {
 214+
 215+ $params = array(
 216+ "state" => "addjob",
 217+ "output" => "dump",
 218+ "user" => $userName,
 219+ "max" => $testMaxRuns,
 220+ "job_name" => str_replace( '$1', $svnHeadRevTop, $jobNamePattern ),
 221+ "browsers" => $browsers,
 222+ "auth" => $userAuthToken
239223 );
240 -
 224+
 225+ $query = http_build_query( $params );
 226+
 227+ foreach ( $suites as $suite ) {
 228+ $suiteName = substr( $suite, -8 ) == '.test.js' ? substr( $suite, 0, -8 ) : $suite;
 229+ $query .= "&suites[]=" . rawurlencode( $suite ) .
 230+ "&urls[]=" . getTestUrl( $svnHeadRevTop, $suite );
 231+ }
 232+
 233+ logger( "cURL url: $swarmUrl" );
 234+ logger( "cURL postfields: $query" );
 235+
 236+ $curlPostCMD = array();
 237+
 238+ // cURL request
 239+ $ch = curl_init();
 240+ curl_setopt_array( $ch, $curlOpts );
 241+ curl_setopt( $ch, CURLOPT_URL, $swarmUrl );
 242+ /*
 243+ curl_setopt( $ch, CURLOPT_HTTPHEADER, array(
 244+ 'Cache-Control: no-cache',
 245+ 'Content-length: ' . strlen( $query )
 246+ )
 247+ );
 248+ */
 249+ curl_setopt( $ch, CURLOPT_HTTPHEADER, array( 'Expect:' ) );
 250+ curl_setopt( $ch, CURLOPT_POSTFIELDS, $query );
 251+ $curlPostCMD['return'] = curl_exec( $ch );
 252+ $curlPostCMD['error'] = curl_errno( $ch );
 253+ curl_close( $ch );
 254+
 255+ logger( 'Results: ' . print_r( $curlPostCMD['return'], true ) );
 256+ logger( 'Error: ' . print_r( $curlPostCMD['error'], true ) );
 257+
 258+
 259+ if ( $curlPostCMD['return'] ) {
 260+ file_put_contents(
 261+ __DIR__ . '/testswarm-mediawiki-svn.log',
 262+ '[' . date('r') . '] ' . $curlPostCMD['return'] . "\n",
 263+ FILE_APPEND
 264+ );
 265+
 266+ } else {
 267+ return "Job not submitted properly.";
 268+ }
 269+
241270 } else {
242 - stopper( "Job not submitted properly." );
 271+ return "No new revision.";
243272 }
244273
245 -} else {
246 - stopper( "No new revision." );
247 -}
 274+}
\ No newline at end of file
Index: trunk/tools/testswarm/scripts/do-svncheckaddjob.php
——@@ -0,0 +1,24 @@
 2+<?php
 3+# Locking system, do not start another check untill this one is finished
 4+$lockFile = __DIR__ . '/lock.addjob';
 5+if ( file_exists( $lockFile ) ) {
 6+ die;
 7+}
 8+file_put_contents( $lockFile, date('r') );
 9+
 10+# Start output
 11+echo "\n--\n\n[" . date( 'r' ) . "] start of do-svncheckaddjob.php:\n";
 12+
 13+error_reporting(E_ALL);ini_set('display_errors', 1);
 14+require_once( __DIR__ . '/testswarm-mediawiki-svn.php');
 15+
 16+# Echo actual script return
 17+echo doingStuff();
 18+
 19+# End output
 20+echo "\n[end of do-svncheckaddjob.php]\n\n";
 21+
 22+# Locking system, clean up after we're done
 23+# Comment out the following line to disable the updater
 24+# (as next time it'll create a lock without unlocking)
 25+unlink( $lockFile );
Property changes on: trunk/tools/testswarm/scripts/do-svncheckaddjob.php
___________________________________________________________________
Added: svn:eol-style
126 + native

Status & tagging log