r9916 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r9915‎ | r9916 | r9917 >
Date:03:16, 5 July 2005
Author:vibber
Status:old
Tags:
Comment:
Some code cleanup on Special:Import, add initial version of command-line
dump importer script (importDump.php). Can read from stdin or from file
on command-line (file can be auto-decompressed if gzip and zlib support).
Still needs some work, but basically functions.
Modified paths:
  • /trunk/phase3/includes/SpecialExport.php (modified) (history)
  • /trunk/phase3/includes/SpecialImport.php (modified) (history)
  • /trunk/phase3/maintenance/dumpBackup.php (modified) (history)
  • /trunk/phase3/maintenance/importDump.php (added) (history)

Diff [purge]

Index: trunk/phase3/maintenance/dumpBackup.php
——@@ -50,7 +50,7 @@
5151 $db =& $this->backupDb();
5252 $exporter = new WikiExporter( $db, $history, MW_EXPORT_STREAM );
5353 $exporter->setPageCallback( array( &$this, 'reportPage' ) );
54 - $exporter->setRevCallback( array( &$this, 'revCount' ) );
 54+ $exporter->setRevisionCallback( array( &$this, 'revCount' ) );
5555
5656 $exporter->openStream();
5757 $exporter->allPages();
Index: trunk/phase3/maintenance/importDump.php
——@@ -0,0 +1,127 @@
 2+<?php
 3+/**
 4+ * Copyright (C) 2005 Brion Vibber <brion@pobox.com>
 5+ * http://www.mediawiki.org/
 6+ *
 7+ * This program is free software; you can redistribute it and/or modify
 8+ * it under the terms of the GNU General Public License as published by
 9+ * the Free Software Foundation; either version 2 of the License, or
 10+ * (at your option) any later version.
 11+ *
 12+ * This program is distributed in the hope that it will be useful,
 13+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
 14+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 15+ * GNU General Public License for more details.
 16+ *
 17+ * You should have received a copy of the GNU General Public License along
 18+ * with this program; if not, write to the Free Software Foundation, Inc.,
 19+ * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 20+ * http://www.gnu.org/copyleft/gpl.html
 21+ *
 22+ * @package MediaWiki
 23+ * @subpackage Maintenance
 24+ */
 25+
 26+$options = array( 'full', 'verbose', 'dry-run', 'preserve' );
 27+
 28+require_once( 'commandLine.inc' );
 29+require_once( 'SpecialImport.php' );
 30+
 31+class BackupReader {
 32+ var $reportingInterval = 100;
 33+ var $reporting = true;
 34+ var $pageCount = 0;
 35+ var $revCount = 0;
 36+ var $dryRun = false;
 37+
 38+ function BackupReader() {
 39+ $this->stderr = fopen( "php://stderr", "wt" );
 40+ }
 41+
 42+ function reportPage( $page ) {
 43+ $this->pageCount++;
 44+ }
 45+
 46+ function handleRevision( $rev ) {
 47+ $title = $rev->getTitle();
 48+ $display = $title->getPrefixedText();
 49+ $timestamp = $rev->getTimestamp();
 50+ #echo "$display $timestamp\n";
 51+
 52+ $this->revCount++;
 53+ $this->report();
 54+
 55+ if( !$this->dryRun ) {
 56+ call_user_func( $this->importCallback, $rev );
 57+ }
 58+ }
 59+
 60+ function report( $final = false ) {
 61+ if( $final xor ( $this->pageCount % $this->reportingInterval == 0 ) ) {
 62+ $this->showReport();
 63+ }
 64+ }
 65+
 66+ function showReport() {
 67+ if( $this->reporting ) {
 68+ $delta = wfTime() - $this->startTime;
 69+ if( $delta ) {
 70+ $rate = $this->pageCount / $delta;
 71+ $revrate = $this->revCount / $delta;
 72+ } else {
 73+ $rate = '-';
 74+ $revrate = '-';
 75+ }
 76+ $this->progress( "$this->pageCount ($rate pages/sec $revrate revs/sec)" );
 77+ }
 78+ }
 79+
 80+ function progress( $string ) {
 81+ fwrite( $this->stderr, $string . "\n" );
 82+ }
 83+
 84+ function importFromFile( $filename ) {
 85+ if( preg_match( '/\.gz$/', $filename ) ) {
 86+ $filename = 'compress.zlib://' . $filename;
 87+ }
 88+ $file = fopen( $filename, 'rt' );
 89+ $this->importFromHandle( $file );
 90+ }
 91+
 92+ function importFromStdin() {
 93+ $file = fopen( 'php://stdin', 'rt' );
 94+ $this->importFromHandle( $file );
 95+ }
 96+
 97+ function importFromHandle( $handle ) {
 98+ $this->startTime = wfTime();
 99+
 100+ $source = new ImportStreamSource( $handle );
 101+ $importer = new WikiImporter( $source );
 102+
 103+ $importer->setPageCallback( array( &$this, 'reportPage' ) );
 104+ $this->importCallback = $importer->setRevisionCallback(
 105+ array( &$this, 'handleRevision' ) );
 106+
 107+ $importer->doImport();
 108+ }
 109+}
 110+
 111+$reader = new BackupReader();
 112+if( isset( $options['quiet'] ) ) {
 113+ $reader->reporting = false;
 114+}
 115+if( isset( $options['report'] ) ) {
 116+ $reader->reportingInterval = IntVal( $options['report'] );
 117+}
 118+if( isset( $options['dry-run'] ) ) {
 119+ $reader->dryRun = true;
 120+}
 121+
 122+if( isset( $args[0] ) ) {
 123+ $reader->importFromFile( $args[0] );
 124+} else {
 125+ $reader->importFromStdin();
 126+}
 127+
 128+?>
\ No newline at end of file
Property changes on: trunk/phase3/maintenance/importDump.php
___________________________________________________________________
Added: svn:eol-style
1129 + native
Added: svn:keywords
2130 + Author Date Id Revision
Index: trunk/phase3/includes/SpecialImport.php
——@@ -38,36 +38,34 @@
3939 ###
4040
4141 if( $wgRequest->wasPosted() && $wgRequest->getVal( 'action' ) == 'submit') {
42 - $importer = new WikiImporter();
43 -
4442 switch( $wgRequest->getVal( "source" ) ) {
4543 case "upload":
4644 if( $wgUser->isAllowed( 'importupload' ) ) {
47 - $result = $importer->setupFromUpload( "xmlimport" );
 45+ $source = ImportStreamSource::newFromUpload( "xmlimport" );
4846 } else {
4947 return $wgOut->permissionRequired( 'importupload' );
5048 }
5149 break;
5250 case "interwiki":
53 - $result = $importer->setupFromInterwiki(
 51+ $source = ImportStreamSource::newFromInterwiki(
5452 $wgRequest->getVal( "interwiki" ),
5553 $wgRequest->getText( "frompage" ) );
5654 break;
5755 default:
58 - $result = new WikiError( "Unknown import source type" );
 56+ $source = new WikiError( "Unknown import source type" );
5957 }
6058
61 - if( WikiError::isError( $result ) ) {
62 - $wgOut->addWikiText( wfEscapeWikiText( $result->getMessage() ) );
 59+ if( WikiError::isError( $source ) ) {
 60+ $wgOut->addWikiText( wfEscapeWikiText( $source->getMessage() ) );
6361 } else {
64 - $importer->setRevisionHandler( "wfImportOldRevision" );
 62+ $importer = new WikiImporter( $source );
6563 $result = $importer->doImport();
6664 if( WikiError::isError( $result ) ) {
67 - $wgOut->addHTML( "<p>" . wfMsg( "importfailed",
68 - htmlspecialchars( $result->getMessage() ) ) . "</p>" );
 65+ $wgOut->addWikiText( wfMsg( "importfailed",
 66+ wfEscapeWikiText( $result->getMessage() ) ) );
6967 } else {
7068 # Success!
71 - $wgOut->addHTML( "<p>" . wfMsg( "importsuccess" ) . "</p>" );
 69+ $wgOut->addWikiText( wfMsg( "importsuccess" ) );
7270 }
7371 }
7472 }
——@@ -78,13 +76,13 @@
7977 $wgOut->addWikiText( wfMsg( "importtext" ) );
8078 $wgOut->addHTML( "
8179 <fieldset>
82 - <legend>" . wfMsg('upload') . "</legend>
 80+ <legend>" . wfMsgHtml('upload') . "</legend>
8381 <form enctype='multipart/form-data' method='post' action=\"$action\">
8482 <input type='hidden' name='action' value='submit' />
8583 <input type='hidden' name='source' value='upload' />
8684 <input type='hidden' name='MAX_FILE_SIZE' value='2000000' />
8785 <input type='file' name='xmlimport' value='' size='30' />
88 - <input type='submit' value='" . htmlspecialchars( wfMsg( "uploadbtn" ) ) . "'/>
 86+ <input type='submit' value='" . wfMsgHtml( "uploadbtn" ) . "'/>
8987 </form>
9088 </fieldset>
9189 " );
——@@ -97,7 +95,7 @@
9896 if( !empty( $wgImportSources ) ) {
9997 $wgOut->addHTML( "
10098 <fieldset>
101 - <legend>" . wfMsg('importinterwiki') . "</legend>
 99+ <legend>" . wfMsgHtml('importinterwiki') . "</legend>
102100 <form method='post' action=\"$action\">
103101 <input type='hidden' name='action' value='submit' />
104102 <input type='hidden' name='source' value='interwiki' />
——@@ -117,11 +115,6 @@
118116 }
119117 }
120118
121 -function wfImportOldRevision( &$revision ) {
122 - $dbw =& wfGetDB( DB_MASTER );
123 - $dbw->deadlockLoop( array( &$revision, 'importOldRevision' ) );
124 -}
125 -
126119 /**
127120 *
128121 * @package MediaWiki
——@@ -186,6 +179,13 @@
187180
188181 # Sneak a single revision into place
189182 $user = User::newFromName( $this->getUser() );
 183+ if( $user ) {
 184+ $userId = IntVal( $user->getId() );
 185+ $userText = $user->getName();
 186+ } else {
 187+ $userId = 0;
 188+ $userText = $this->getUser();
 189+ }
190190
191191 $article = new Article( $this->title );
192192 $pageId = $article->getId();
——@@ -207,8 +207,8 @@
208208 'page' => $pageId,
209209 'text' => $this->getText(),
210210 'comment' => $this->getComment(),
211 - 'user' => IntVal( $user->getId() ),
212 - 'user_text' => $user->getName(),
 211+ 'user' => $userId,
 212+ 'user_text' => $userText,
213213 'timestamp' => $this->timestamp,
214214 'minor_edit' => 0
215215 ) );
——@@ -226,65 +226,20 @@
227227 * @subpackage SpecialPage
228228 */
229229 class WikiImporter {
230 - var $mSource = NULL;
231 - var $mRevisionHandler = NULL;
 230+ var $mSource = null;
 231+ var $mPageCallback = null;
 232+ var $mRevisionCallback = null;
232233 var $lastfield;
233234
234 - function WikiImporter() {
235 - $this->setRevisionHandler( array( &$this, "defaultRevisionHandler" ) );
 235+ function WikiImporter( $source ) {
 236+ $this->setRevisionCallback( array( &$this, "importRevision" ) );
 237+ $this->mSource = $source;
236238 }
237239
238240 function throwXmlError( $err ) {
239241 $this->debug( "FAILURE: $err" );
240242 }
241243
242 - function setupFromFile( $filename ) {
243 - $this->mSource = @file_get_contents( $filename );
244 - if( $this->mSource === false ) {
245 - return new WikiError( "Couldn't open import file" );
246 - }
247 - return true;
248 - }
249 -
250 - function setupFromUpload( $fieldname = "xmlimport" ) {
251 - global $wgOut;
252 -
253 - $upload =& $_FILES[$fieldname];
254 -
255 - if( !isset( $upload ) ) {
256 - return new WikiErrorMsg( 'importnofile' );
257 - }
258 - if( !empty( $upload['error'] ) ) {
259 - return new WikiErrorMsg( 'importuploaderror', $upload['error'] );
260 - }
261 - $fname = $upload['tmp_name'];
262 - if( is_uploaded_file( $fname ) ) {
263 - return $this->setupFromFile( $fname );
264 - } else {
265 - return new WikiErrorMsg( 'importnofile' );
266 - }
267 - }
268 -
269 - function setupFromURL( $url ) {
270 - # fopen-wrappers are normally turned off for security.
271 - ini_set( "allow_url_fopen", true );
272 - $ret = $this->setupFromFile( $url );
273 - ini_set( "allow_url_fopen", false );
274 - return $ret;
275 - }
276 -
277 - function setupFromInterwiki( $interwiki, $page ) {
278 - $base = Title::getInterwikiLink( $interwiki );
279 - if( empty( $base ) ) {
280 - return new WikiError( 'Bad interwiki link' );
281 - } else {
282 - $import = wfUrlencode( "Special:Export/$page" );
283 - $url = str_replace( "$1", $import, $base );
284 - $this->notice( "Importing from $url" );
285 - return $this->setupFromURL( $url );
286 - }
287 - }
288 -
289244 # --------------
290245
291246 function doImport() {
——@@ -300,16 +255,19 @@
301256 xml_set_object( $parser, &$this );
302257 xml_set_element_handler( $parser, "in_start", "" );
303258
304 - if( !xml_parse( $parser, $this->mSource, true ) ) {
305 - return new WikiXmlError( $parser );
306 - }
 259+ do {
 260+ $chunk = $this->mSource->readChunk();
 261+ if( !xml_parse( $parser, $chunk, $this->mSource->atEnd() ) ) {
 262+ return new WikiXmlError( $parser );
 263+ }
 264+ } while( $chunk !== false && !$this->mSource->atEnd() );
307265 xml_parser_free( $parser );
308266
309267 return true;
310268 }
311269
312270 function debug( $data ) {
313 - #$this->notice( "DEBUG: $data\n" );
 271+ #wfDebug( "IMPORT: $data\n" );
314272 }
315273
316274 function notice( $data ) {
——@@ -322,11 +280,44 @@
323281 }
324282 }
325283
326 - function setRevisionHandler( $functionref ) {
327 - $this->mRevisionHandler = $functionref;
 284+ /**
 285+ * Sets the action to perform as each new page in the stream is reached.
 286+ * @param callable $callback
 287+ * @return callable
 288+ */
 289+ function setPageCallback( $callback ) {
 290+ $previous = $this->mPageCallback;
 291+ $this->mPageCallback = $callback;
 292+ return $previous;
328293 }
329294
330 - function defaultRevisionHandler( &$revision ) {
 295+ /**
 296+ * Sets the action to perform as each page revision is reached.
 297+ * @param callable $callback
 298+ * @return callable
 299+ */
 300+ function setRevisionCallback( $callback ) {
 301+ $previous = $this->mRevisionCallback;
 302+ $this->mRevisionCallback = $callback;
 303+ return $previous;
 304+ }
 305+
 306+ /**
 307+ * Default per-revision callback, performs the import.
 308+ * @param WikiRevision $revision
 309+ * @access private
 310+ */
 311+ function importRevision( &$revision ) {
 312+ $dbw =& wfGetDB( DB_MASTER );
 313+ $dbw->deadlockLoop( array( &$revision, 'importOldRevision' ) );
 314+ }
 315+
 316+ /**
 317+ * Alternate per-revision callback, for debugging.
 318+ * @param WikiRevision $revision
 319+ * @access private
 320+ */
 321+ function debugRevisionHandler( &$revision ) {
331322 $this->debug( "Got revision:" );
332323 if( is_object( $revision->title ) ) {
333324 $this->debug( "-- Title: " . $revision->title->getPrefixedText() );
——@@ -339,6 +330,16 @@
340331 $this->debug( "-- Text: " . $revision->text );
341332 }
342333
 334+ /**
 335+ * Notify the callback function when a new <page> is reached.
 336+ * @param Title $title
 337+ * @access private
 338+ */
 339+ function pageCallback( $title ) {
 340+ if( is_callable( $this->mPageCallback ) ) {
 341+ call_user_func( $this->mPageCallback, $title );
 342+ }
 343+ }
343344
344345
345346 # XML parser callbacks from here out -- beware!
——@@ -356,10 +357,13 @@
357358
358359 function in_mediawiki( $parser, $name, $attribs ) {
359360 $this->debug( "in_mediawiki $name" );
360 - if( $name != "page" ) {
 361+ if( $name == 'siteinfo' ) {
 362+ xml_set_element_handler( $parser, "in_siteinfo", "out_siteinfo" );
 363+ } elseif( $name == 'page' ) {
 364+ xml_set_element_handler( $parser, "in_page", "out_page" );
 365+ } else {
361366 return $this->throwXMLerror( "Expected <page>, got <$name>" );
362367 }
363 - xml_set_element_handler( $parser, "in_page", "out_page" );
364368 }
365369 function out_mediawiki( $parser, $name ) {
366370 $this->debug( "out_mediawiki $name" );
——@@ -368,6 +372,29 @@
369373 }
370374 xml_set_element_handler( $parser, "donothing", "donothing" );
371375 }
 376+
 377+
 378+ function in_siteinfo( $parser, $name, $attribs ) {
 379+ // no-ops for now
 380+ $this->debug( "in_siteinfo $name" );
 381+ switch( $name ) {
 382+ case "sitename":
 383+ case "generator":
 384+ case "case":
 385+ case "namespaces":
 386+ case "namespace":
 387+ break;
 388+ default:
 389+ return $this->throwXMLerror( "Element <$name> not allowed in <siteinfo>." );
 390+ }
 391+ }
 392+
 393+ function out_siteinfo( $parser, $name ) {
 394+ if( $name == "siteinfo" ) {
 395+ xml_set_element_handler( $parser, "in_mediawiki", "out_mediawiki" );
 396+ }
 397+ }
 398+
372399
373400 function in_page( $parser, $name, $attribs ) {
374401 $this->debug( "in_page $name" );
——@@ -417,9 +444,11 @@
418445 }
419446 xml_set_element_handler( $parser, "in_$this->parenttag", "out_$this->parenttag" );
420447 xml_set_character_data_handler( $parser, "donothing" );
 448+
421449 switch( $this->appendfield ) {
422450 case "title":
423451 $this->workTitle = $this->appenddata;
 452+ $this->pageCallback( $this->workTitle );
424453 break;
425454 case "text":
426455 $this->workRevision->setText( $this->appenddata );
——@@ -470,7 +499,9 @@
471500 }
472501 xml_set_element_handler( $parser, "in_page", "out_page" );
473502
474 - $out = call_user_func( $this->mRevisionHandler, &$this->workRevision, &$this );
 503+ $out = call_user_func( $this->mRevisionCallback,
 504+ &$this->workRevision,
 505+ &$this );
475506 if( !empty( $out ) ) {
476507 global $wgOut;
477508 $wgOut->addHTML( "<li>" . $out . "</li>\n" );
——@@ -502,5 +533,85 @@
503534
504535 }
505536
 537+class ImportStringSource {
 538+ function ImportStringSource( $string ) {
 539+ $this->mString = $string;
 540+ $this->mRead = false;
 541+ }
 542+
 543+ function atEnd() {
 544+ return $this->mRead;
 545+ }
 546+
 547+ function readChunk() {
 548+ if( $this->atEnd() ) {
 549+ return false;
 550+ } else {
 551+ $this->mRead = true;
 552+ return $this->mString;
 553+ }
 554+ }
 555+}
506556
 557+class ImportStreamSource {
 558+ function ImportStreamSource( $handle ) {
 559+ $this->mHandle = $handle;
 560+ }
 561+
 562+ function atEnd() {
 563+ return feof( $this->mHandle );
 564+ }
 565+
 566+ function readChunk() {
 567+ return fread( $this->mHandle, 32768 );
 568+ }
 569+
 570+ function newFromFile( $filename ) {
 571+ $file = @fopen( $filename, 'rt' );
 572+ if( !$file ) {
 573+ return new WikiError( "Couldn't open import file" );
 574+ }
 575+ return new ImportStreamSource( $file );
 576+ }
 577+
 578+ function newFromUpload( $fieldname = "xmlimport" ) {
 579+ global $wgOut;
 580+
 581+ $upload =& $_FILES[$fieldname];
 582+
 583+ if( !isset( $upload ) ) {
 584+ return new WikiErrorMsg( 'importnofile' );
 585+ }
 586+ if( !empty( $upload['error'] ) ) {
 587+ return new WikiErrorMsg( 'importuploaderror', $upload['error'] );
 588+ }
 589+ $fname = $upload['tmp_name'];
 590+ if( is_uploaded_file( $fname ) ) {
 591+ return ImportStreamSource::newFromFile( $fname );
 592+ } else {
 593+ return new WikiErrorMsg( 'importnofile' );
 594+ }
 595+ }
 596+
 597+ function newFromURL( $url ) {
 598+ # fopen-wrappers are normally turned off for security.
 599+ ini_set( "allow_url_fopen", true );
 600+ $ret = ImportStreamSource::newFromFile( $url );
 601+ ini_set( "allow_url_fopen", false );
 602+ return $ret;
 603+ }
 604+
 605+ function newFromInterwiki( $interwiki, $page ) {
 606+ $base = Title::getInterwikiLink( $interwiki );
 607+ if( empty( $base ) ) {
 608+ return new WikiError( 'Bad interwiki link' );
 609+ } else {
 610+ $import = wfUrlencode( "Special:Export/$page" );
 611+ $url = str_replace( "$1", $import, $base );
 612+ return ImportStreamSource::newFromURL( $url );
 613+ }
 614+ }
 615+}
 616+
 617+
507618 ?>
Index: trunk/phase3/includes/SpecialExport.php
——@@ -121,7 +121,7 @@
122122 *
123123 * @param mixed $callback
124124 */
125 - function setRevCallback( $callback ) {
 125+ function setRevisionCallback( $callback ) {
126126 $this->revCallback = $callback;
127127 }
128128

Status & tagging log