r93744 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r93743‎ | r93744 | r93745 >
Date:14:05, 2 August 2011
Author:catrope
Status:resolved (Comments)
Tags:
Comment:
Add --no-updates flag to importDump.php that allows the user to skip updating the links table. On my test dataset (an old (~2 years) MediaWiki namespace dump of enwiki) this speeds up the import from 9m35s to 10s.
Modified paths:
  • /trunk/phase3/includes/Import.php (modified) (history)
  • /trunk/phase3/maintenance/importDump.php (modified) (history)

Diff [purge]

Index: trunk/phase3/maintenance/importDump.php
@@ -63,6 +63,7 @@
6464 $this->addOption( 'dry-run', 'Parse dump without actually importing pages' );
6565 $this->addOption( 'debug', 'Output extra verbose debug information' );
6666 $this->addOption( 'uploads', 'Process file upload data if included (experimental)' );
 67+ $this->addOption( 'no-updates', 'Disable link table updates. Is faster but leaves the wiki in an inconsistent state' );
6768 $this->addOption( 'image-base-path', 'Import files from a specified path', false, true );
6869 $this->addArg( 'file', 'Dump file to import [else use stdin]', false );
6970 }
@@ -243,6 +244,9 @@
244245 if( $this->hasOption( 'debug' ) ) {
245246 $importer->setDebug( true );
246247 }
 248+ if ( $this->hasOption( 'no-updates' ) ) {
 249+ $importer->setNoUpdates( true );
 250+ }
247251 $importer->setPageCallback( array( &$this, 'reportPage' ) );
248252 $this->importCallback = $importer->setRevisionCallback(
249253 array( &$this, 'handleRevision' ) );
Index: trunk/phase3/includes/Import.php
@@ -36,6 +36,7 @@
3737 private $mSiteInfoCallback, $mTargetNamespace, $mPageOutCallback;
3838 private $mDebug;
3939 private $mImportUploads, $mImageBasePath;
 40+ private $mNoUpdates = false;
4041
4142 /**
4243 * Creates an ImportXMLReader drawing from the source provided
@@ -90,6 +91,13 @@
9192 function setDebug( $debug ) {
9293 $this->mDebug = $debug;
9394 }
 95+
 96+ /**
 97+ * Set 'no updates' mode. In this mode, the link tables will not be updated by the importer
 98+ */
 99+ function setNoUpdates( $noupdates ) {
 100+ $this->mNoUpdates = $noupdates;
 101+ }
94102
95103 /**
96104 * Sets the action to perform as each new page in the stream is reached.
@@ -453,6 +461,7 @@
454462 $revision->setTimestamp( $logInfo['timestamp'] );
455463 $revision->setParams( $logInfo['params'] );
456464 $revision->setTitle( Title::newFromText( $logInfo['logtitle'] ) );
 465+ $revision->setNoUpdates( $this->mNoUpdates );
457466
458467 if ( isset( $logInfo['comment'] ) ) {
459468 $revision->setComment( $logInfo['comment'] );
@@ -587,6 +596,7 @@
588597 if ( isset( $revisionInfo['contributor']['username'] ) ) {
589598 $revision->setUserName( $revisionInfo['contributor']['username'] );
590599 }
 600+ $revision->setNoUpdates( $this->mNoUpdates );
591601
592602 return $this->revisionCallback( $revision );
593603 }
@@ -677,6 +687,7 @@
678688 if ( isset( $uploadInfo['contributor']['username'] ) ) {
679689 $revision->setUserName( $uploadInfo['contributor']['username'] );
680690 }
 691+ $revision->setNoUpdates( $this->mNoUpdates );
681692
682693 return call_user_func( $this->mUploadCallback, $revision );
683694 }
@@ -853,6 +864,7 @@
854865 var $sha1base36 = false;
855866 var $isTemp = false;
856867 var $archiveName = '';
 868+ private $mNoUpdates = false;
857869
858870 function setTitle( $title ) {
859871 if( is_object( $title ) ) {
@@ -926,6 +938,10 @@
927939 function setParams( $params ) {
928940 $this->params = $params;
929941 }
 942+
 943+ public function setNoUpdates( $noupdates ) {
 944+ $this->mNoUpdates = $noupdates;
 945+ }
930946
931947 /**
932948 * @return Title
@@ -1056,8 +1072,9 @@
10571073 $revision->insertOn( $dbw );
10581074 $changed = $article->updateIfNewerOn( $dbw, $revision );
10591075
1060 - if ( $changed !== false ) {
 1076+ if ( $changed !== false && !$this->mNoUpdates ) {
10611077 wfDebug( __METHOD__ . ": running updates\n" );
 1078+ throw new MWException("BROKEN: calling doEditUpdates()");
10621079 $article->doEditUpdates( $revision, $userObj, array( 'created' => $created, 'oldcountable' => $oldcountable ) );
10631080 }
10641081

Follow-up revisions

RevisionCommit summaryAuthorDate
r93879Fix embarassing debugging code left in in r93744catrope12:13, 4 August 2011
r94426Add RELEASE-NOTES for r93744catrope19:02, 13 August 2011

Comments

#Comment by Aaron Schulz (talk | contribs)   17:01, 2 August 2011

I meant to do this a while ago. Very useful.

#Comment by Aaron Schulz (talk | contribs)   17:02, 2 August 2011

throw new MWException("BROKEN: calling doEditUpdates()");

What this left by mistake? :)

Status & tagging log