r73645 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r73644‎ | r73645 | r73646 >
Date:21:23, 23 September 2010
Author:tparscal
Status:ok (Comments)
Tags:
Comment:
Improved the performance of ResourceLoader by pre-loading module information in a batch query. This was mostly code written by catrope and patched in / gotten working by me.
Modified paths:
  • /trunk/phase3/includes/ResourceLoader.php (modified) (history)
  • /trunk/phase3/includes/ResourceLoaderModule.php (modified) (history)

Diff [purge]

Index: trunk/phase3/includes/ResourceLoader.php
@@ -33,7 +33,7 @@
3434
3535 /* Protected Static Methods */
3636
37 - /*
 37+ /**
3838 * Registers core modules and runs registration hooks
3939 */
4040 protected static function initialize() {
@@ -50,6 +50,55 @@
5151 wfProfileOut( __METHOD__ );
5252 }
5353 }
 54+
 55+ protected static function preloadModuleInfo( $modules, ResourceLoaderContext $context ) {
 56+ $dbr = wfGetDb( DB_SLAVE );
 57+ $skin = $context->getSkin();
 58+ $lang = $context->getLanguage();
 59+
 60+ // Get file dependency information
 61+ $res = $dbr->select( 'module_deps', array( 'md_module', 'md_deps' ), array(
 62+ 'md_module' => $modules,
 63+ 'md_skin' => $context->getSkin()
 64+ ), __METHOD__
 65+ );
 66+
 67+ $modulesWithDeps = array();
 68+ foreach ( $res as $row ) {
 69+ self::$modules[$row->md_module]->setFileDependencies( $skin,
 70+ FormatJson::decode( $row->md_deps, true )
 71+ );
 72+ $modulesWithDeps[] = $row->md_module;
 73+ }
 74+ // Register the absence of a dependencies row too
 75+ foreach ( array_diff( $modules, $modulesWithDeps ) as $name ) {
 76+ self::$modules[$name]->setFileDependencies( $skin, array() );
 77+ }
 78+
 79+ // Get message blob mtimes. Only do this for modules with messages
 80+ $modulesWithMessages = array();
 81+ $modulesWithoutMessages = array();
 82+ foreach ( $modules as $name ) {
 83+ if ( count( self::$modules[$name]->getMessages() ) ) {
 84+ $modulesWithMessages[] = $name;
 85+ } else {
 86+ $modulesWithoutMessages[] = $name;
 87+ }
 88+ }
 89+ if ( count( $modulesWithMessages ) ) {
 90+ $res = $dbr->select( 'msg_resource', array( 'mr_resource', 'mr_timestamp' ), array(
 91+ 'mr_resource' => $modulesWithMessages,
 92+ 'mr_lang' => $lang
 93+ ), __METHOD__
 94+ );
 95+ foreach ( $res as $row ) {
 96+ self::$modules[$row->mr_resource]->setMsgBlobMtime( $lang, $row->mr_timestamp );
 97+ }
 98+ }
 99+ foreach ( $modulesWithoutMessages as $name ) {
 100+ self::$modules[$name]->setMsgBlobMtime( $lang, 0 );
 101+ }
 102+ }
54103
55104 /**
56105 * Runs text through a filter, caching the filtered result for future calls
@@ -279,6 +328,9 @@
280329 $smaxage = $wgResourceLoaderMaxage['versioned']['server'];
281330 }
282331
 332+ // Preload information needed to the mtime calculation below
 333+ self::preloadModuleInfo( $modules, $context );
 334+
283335 // To send Last-Modified and support If-Modified-Since, we need to detect
284336 // the last modified time
285337 wfProfileIn( __METHOD__.'-getModifiedTime' );
@@ -307,7 +359,7 @@
308360
309361 // Pre-fetch blobs
310362 $blobs = $context->shouldIncludeMessages() ?
311 - MessageBlobStore::get( $modules, $context->getLanguage() ) : array();
 363+ MessageBlobStore::get( $modules, $context->getLanguage() ) : array();
312364
313365 // Generate output
314366 foreach ( $modules as $name ) {
Index: trunk/phase3/includes/ResourceLoaderModule.php
@@ -27,6 +27,11 @@
2828 /* Protected Members */
2929
3030 protected $name = null;
 31+
 32+ // In-object cache for file dependencies
 33+ protected $fileDeps = array();
 34+ // In-object cache for message blob mtime
 35+ protected $msgBlobMtime = array();
3136
3237 /* Methods */
3338
@@ -131,7 +136,73 @@
132137 // Stub, override expected
133138 return array();
134139 }
 140+
 141+ /**
 142+ * Get the files this module depends on indirectly for a given skin.
 143+ * Currently these are only image files referenced by the module's CSS.
 144+ *
 145+ * @param $skin String: skin name
 146+ * @return array of files
 147+ */
 148+ public function getFileDependencies( $skin ) {
 149+ // Try in-object cache first
 150+ if ( isset( $this->fileDeps[$skin] ) ) {
 151+ return $this->fileDeps[$skin];
 152+ }
135153
 154+ $dbr = wfGetDB( DB_SLAVE );
 155+ $deps = $dbr->selectField( 'module_deps', 'md_deps', array(
 156+ 'md_module' => $this->getName(),
 157+ 'md_skin' => $skin,
 158+ ), __METHOD__
 159+ );
 160+ if ( !is_null( $deps ) ) {
 161+ $this->fileDeps[$skin] = (array) FormatJson::decode( $deps, true );
 162+ }
 163+ return $this->fileDeps[$skin];
 164+ }
 165+
 166+ /**
 167+ * Set preloaded file dependency information. Used so we can load this
 168+ * information for all modules at once.
 169+ * @param $skin string Skin name
 170+ * @param $deps array Array of file names
 171+ */
 172+ public function setFileDependencies( $skin, $deps ) {
 173+ $this->fileDeps[$skin] = $deps;
 174+ }
 175+
 176+ /**
 177+ * Get the last modification timestamp of the message blob for this
 178+ * module in a given language.
 179+ * @param $lang string Language code
 180+ * @return int UNIX timestamp, or 0 if no blob found
 181+ */
 182+ public function getMsgBlobMtime( $lang ) {
 183+ if ( !count( $this->getMessages() ) )
 184+ return 0;
 185+
 186+ $dbr = wfGetDB( DB_SLAVE );
 187+ $msgBlobMtime = $dbr->selectField( 'msg_resource', 'mr_timestamp', array(
 188+ 'mr_resource' => $this->getName(),
 189+ 'mr_lang' => $lang
 190+ ), __METHOD__
 191+ );
 192+ $this->msgBlobMtime[$lang] = $msgBlobMtime ? wfTimestamp( TS_UNIX, $msgBlobMtime ) : 0;
 193+ return $this->msgBlobMtime[$lang];
 194+ }
 195+
 196+ /**
 197+ * Set a preloaded message blob last modification timestamp. Used so we
 198+ * can load this information for all modules at once.
 199+ * @param $lang string Language code
 200+ * @param $mtime int UNIX timestamp or 0 if there is no such blob
 201+ */
 202+ public function setMsgBlobMtime( $lang, $mtime ) {
 203+ $this->msgBlobMtime[$lang] = $mtime;
 204+ }
 205+
 206+
136207 /* Abstract Methods */
137208
138209 /**
@@ -483,24 +554,11 @@
484555 $this->loaders,
485556 $this->getFileDependencies( $context->getSkin() )
486557 );
 558+
487559 wfProfileIn( __METHOD__.'-filemtime' );
488560 $filesMtime = max( array_map( 'filemtime', array_map( array( __CLASS__, 'remapFilename' ), $files ) ) );
489561 wfProfileOut( __METHOD__.'-filemtime' );
490 - // Only get the message timestamp if there are messages in the module
491 - $msgBlobMtime = 0;
492 - if ( count( $this->messages ) ) {
493 - // Get the mtime of the message blob
494 - // TODO: This timestamp is queried a lot and queried separately for each module.
495 - // Maybe it should be put in memcached?
496 - $dbr = wfGetDB( DB_SLAVE );
497 - $msgBlobMtime = $dbr->selectField( 'msg_resource', 'mr_timestamp', array(
498 - 'mr_resource' => $this->getName(),
499 - 'mr_lang' => $context->getLanguage()
500 - ), __METHOD__
501 - );
502 - $msgBlobMtime = $msgBlobMtime ? wfTimestamp( TS_UNIX, $msgBlobMtime ) : 0;
503 - }
504 - $this->modifiedTime[$context->getHash()] = max( $filesMtime, $msgBlobMtime );
 562+ $this->modifiedTime[$context->getHash()] = max( $filesMtime, $this->getMsgBlobMtime( $context->getLanguage() ) );
505563 wfProfileOut( __METHOD__ );
506564 return $this->modifiedTime[$context->getHash()];
507565 }
@@ -590,43 +648,6 @@
591649 }
592650
593651 /**
594 - * Get the files this module depends on indirectly for a given skin.
595 - * Currently these are only image files referenced by the module's CSS.
596 - *
597 - * @param $skin String: skin name
598 - * @return array of files
599 - */
600 - protected function getFileDependencies( $skin ) {
601 - // Try in-object cache first
602 - if ( isset( $this->fileDeps[$skin] ) ) {
603 - return $this->fileDeps[$skin];
604 - }
605 -
606 - // Now try memcached
607 - global $wgMemc;
608 -
609 - $key = wfMemcKey( 'resourceloader', 'module_deps', $this->getName(), $skin );
610 - $deps = $wgMemc->get( $key );
611 -
612 - if ( !$deps ) {
613 - $dbr = wfGetDB( DB_SLAVE );
614 - $deps = $dbr->selectField( 'module_deps', 'md_deps', array(
615 - 'md_module' => $this->getName(),
616 - 'md_skin' => $skin,
617 - ), __METHOD__
618 - );
619 - if ( !$deps ) {
620 - $deps = '[]'; // Empty array so we can do negative caching
621 - }
622 - $wgMemc->set( $key, $deps );
623 - }
624 -
625 - $this->fileDeps = FormatJson::decode( $deps, true );
626 -
627 - return $this->fileDeps;
628 - }
629 -
630 - /**
631652 * Get the contents of a set of files and concatenate them, with
632653 * newlines in between. Each file is used only once.
633654 *

Follow-up revisions

RevisionCommit summaryAuthorDate
r73666Fixed issue in r73645 where an unset value was being returned in some cases.tparscal17:19, 24 September 2010
r73671Fix r73645: also do negative cachingcatrope18:16, 24 September 2010
r74455Remove $wgMemc->set() call left over from r73645catrope19:46, 7 October 2010
r81689Fix bug from r73645: setMsgBlobMtime() requires its timestamp parameter to be...tstarling05:33, 8 February 2011

Comments

#Comment by Nikerabbit (talk | contribs)   21:37, 23 September 2010
-               MessageBlobStore::get( $modules, $context->getLanguage() ) : array();
+                       MessageBlobStore::get( $modules, $context->getLanguage() ) : array();

Accidental extra tab here?

#Comment by Trevor Parscal (WMF) (talk | contribs)   21:39, 23 September 2010

No, it's part of a ternary operator.

#Comment by Catrope (talk | contribs)   15:32, 24 September 2010
+		if ( !is_null( $deps ) ) {
+			$this->fileDeps[$skin] = (array) FormatJson::decode( $deps, true );
+		}
+		return $this->fileDeps[$skin];

This looks like it'll issue a warning when $deps is null because $this->fileDeps[$skin] is never set in that case. A ternary along the lines of $this->fileDeps[$skin] = $deps ? FormatJson::decode( $deps, true ) : array(); was used in the past IIRC.

#Comment by Trevor Parscal (WMF) (talk | contribs)   17:21, 24 September 2010

Nah, just reuturned inside the conditional and returned an empty array outside of it. See r73666.

Status & tagging log