r84610 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r84609‎ | r84610 | r84611 >
Date:17:35, 23 March 2011
Author:aaron
Status:ok (Comments)
Tags:
Comment:
* Put parser output file version tracking to core
* Added some ParserOutput accessors
* A few cleanups to fetchFile()
Modified paths:
  • /trunk/phase3/includes/ImageGallery.php (modified) (history)
  • /trunk/phase3/includes/OutputPage.php (modified) (history)
  • /trunk/phase3/includes/parser/Parser.php (modified) (history)
  • /trunk/phase3/includes/parser/ParserOutput.php (modified) (history)

Diff [purge]

Index: trunk/phase3/includes/parser/Parser.php
@@ -53,7 +53,7 @@
5454 * changes in an incompatible way, so the parser cache
5555 * can automatically discard old data.
5656 */
57 - const VERSION = '1.6.4';
 57+ const VERSION = '1.6.5';
5858
5959 /**
6060 * Update this version number when the output of serialiseHalfParsedText()
@@ -1869,7 +1869,6 @@
18701870 }
18711871 wfProfileOut( __METHOD__."-image" );
18721872 continue;
1873 -
18741873 }
18751874
18761875 if ( $ns == NS_CATEGORY ) {
@@ -1914,14 +1913,11 @@
19151914 wfRunHooks( 'BeforeParserMakeImageLinkObj',
19161915 array( &$this, &$nt, &$skip, &$time, &$descQuery, &$sha1 ) );
19171916 if ( $skip ) {
1918 - $this->mOutput->addImage( $nt->getDBkey() ); // register
 1917+ $this->mOutput->addImage( $nt->getDBkey(), null, null ); // register
19191918 $link = $sk->link( $nt );
19201919 } else {
1921 - # Fetch and register the file
1922 - $file = $this->fetchFile( $nt, $time, $sha1 );
1923 - if ( $file ) {
1924 - $nt = $file->getTitle(); // file title may be different (via hooks)
1925 - }
 1920+ # Fetch and register the file (file title may be different via hooks)
 1921+ list( $file, $nt ) = $this->fetchFileAndTitle( $nt, $time, $sha1 );
19261922 $link = $sk->makeMediaLinkFile( $nt, $file, $text );
19271923 }
19281924 # Cloak with NOPARSE to avoid replacement in replaceExternalLinks
@@ -3314,6 +3310,8 @@
33153311
33163312 /**
33173313 * Fetch the unparsed text of a template and register a reference to it.
 3314+ * @param Title $title
 3315+ * @return Array ( string or false, Title )
33183316 */
33193317 function fetchTemplateAndTitle( $title ) {
33203318 $templateCb = $this->mOptions->getTemplateCallback(); # Defaults to Parser::statelessFetchTemplate()
@@ -3328,6 +3326,11 @@
33293327 return array( $text, $finalTitle );
33303328 }
33313329
 3330+ /**
 3331+ * Fetch the unparsed text of a template and register a reference to it.
 3332+ * @param Title $title
 3333+ * @return mixed string or false
 3334+ */
33323335 function fetchTemplate( $title ) {
33333336 $rv = $this->fetchTemplateAndTitle( $title );
33343337 return $rv[0];
@@ -3398,11 +3401,28 @@
33993402 }
34003403
34013404 /**
3402 - * Fetch a file and register a reference to it.
3403 - * @TODO: register and track file version info too
 3405+ * Fetch a file and its title and register a reference to it.
 3406+ * @param Title $title
 3407+ * @param string $time MW timestamp
 3408+ * @param string $sha1 base 36 SHA-1
 3409+ * @return mixed File or false
34043410 */
34053411 function fetchFile( $title, $time = false, $sha1 = false ) {
3406 - if ( $sha1 ) { // get by (sha1,timestamp)
 3412+ $res = $this->fetchFileAndTitle( $title, $time, $sha1 );
 3413+ return $res[0];
 3414+ }
 3415+
 3416+ /**
 3417+ * Fetch a file and its title and register a reference to it.
 3418+ * @param Title $title
 3419+ * @param string $time MW timestamp
 3420+ * @param string $sha1 base 36 SHA-1
 3421+ * @return Array ( File or false, Title )
 3422+ */
 3423+ function fetchFileAndTitle( $title, $time = false, $sha1 = false ) {
 3424+ if ( $time === '0' ) {
 3425+ $file = false; // broken thumbnail forced by hook
 3426+ } elseif ( $sha1 ) { // get by (sha1,timestamp)
34073427 $file = RepoGroup::singleton()->findFileFromKey( $sha1, array( 'time' => $time ) );
34083428 if ( $file ) {
34093429 $title = $file->getTitle(); // file title may not match $title
@@ -3410,8 +3430,12 @@
34113431 } else { // get by (name,timestamp)
34123432 $file = wfFindFile( $title, array( 'time' => $time ) );
34133433 }
3414 - $this->mOutput->addImage( $title->getDBkey() );
3415 - return $file;
 3434+ # Register the file as a dependancy
 3435+ $time = $file ? $file->getTimestamp() : null;
 3436+ $sha1 = $file ? $file->getSha1() : null;
 3437+ $this->mOutput->addImage( $title->getDBkey(), $time, $sha1 );
 3438+
 3439+ return array( $file, $title );
34163440 }
34173441
34183442 /**
@@ -4670,14 +4694,11 @@
46714695 wfRunHooks( 'BeforeParserMakeImageLinkObj',
46724696 array( &$this, &$title, &$skip, &$time, &$descQuery, &$sha1 ) );
46734697 if ( $skip ) {
4674 - $this->mOutput->addImage( $title->getDBkey() ); // register
 4698+ $this->mOutput->addImage( $title->getDBkey(), null, null ); // register
46754699 return $sk->link( $title );
46764700 }
4677 - # Fetch and register the file
4678 - $file = $this->fetchFile( $title, $time, $sha1 );
4679 - if ( $file ) {
4680 - $title = $file->getTitle(); // file title may be different (via hooks)
4681 - }
 4701+ # Fetch and register the file (file title may be different via hooks)
 4702+ list( $file, $title ) = $this->fetchFileAndTitle( $title, $time, $sha1 );
46824703 # Get parameter map
46834704 $handler = $file ? $file->getHandler() : false;
46844705
Index: trunk/phase3/includes/parser/ParserOutput.php
@@ -109,6 +109,7 @@
110110 $mTemplates = array(), # 2-D map of NS/DBK to ID for the template references. ID=zero for broken.
111111 $mTemplateIds = array(), # 2-D map of NS/DBK to rev ID for the template references. ID=zero for broken.
112112 $mImages = array(), # DB keys of the images used, in the array key only
 113+ $mImageTimeKeys = array(), # DB keys of the images used mapped to sha1 and MW timestamp
113114 $mExternalLinks = array(), # External link URLs, in the key only
114115 $mInterwikiLinks = array(), # 2-D map of prefix/DBK (in keys only) for the inline interwiki links in the document.
115116 $mNewSection = false, # Show a new section link?
@@ -174,7 +175,9 @@
175176 function getEditSectionTokens() { return $this->mEditSectionTokens; }
176177 function &getLinks() { return $this->mLinks; }
177178 function &getTemplates() { return $this->mTemplates; }
 179+ function &getTemplateIds() { return $this->mTemplateIds; }
178180 function &getImages() { return $this->mImages; }
 181+ function &getImageTimeKeys() { return $this->mImageTimeKeys; }
179182 function &getExternalLinks() { return $this->mExternalLinks; }
180183 function getNoGallery() { return $this->mNoGallery; }
181184 function getHeadItems() { return $this->mHeadItems; }
@@ -256,14 +259,21 @@
257260 $this->mLinks[$ns][$dbk] = $id;
258261 }
259262
260 - function addImage( $name ) {
 263+ /**
 264+ * @param $name string Title dbKey
 265+ * @param $timestamp string MW timestamp of file creation
 266+ * @param $sha string base 36 SHA-1 of file
 267+ * @return void
 268+ */
 269+ function addImage( $name, $timestamp, $sha1 ) {
261270 $this->mImages[$name] = 1;
 271+ $this->mImageTimeKeys[$name] = array( 'time' => $timestamp, 'sha1' => $sha1 );
262272 }
263273
264274 /**
265275 * @param $title Title
266 - * @param $page_id
267 - * @param $rev_id
 276+ * @param $page_id
 277+ * @param $rev_id
268278 * @return void
269279 */
270280 function addTemplate( $title, $page_id, $rev_id ) {
Index: trunk/phase3/includes/OutputPage.php
@@ -123,6 +123,7 @@
124124 var $mInlineMsg = array();
125125
126126 var $mTemplateIds = array();
 127+ var $mImageTimeKeys = array();
127128
128129 # What level of 'untrustworthiness' is allowed in CSS/JS modules loaded on this page?
129130 # @see ResourceLoaderModule::$origin
@@ -1308,14 +1309,19 @@
13091310 $this->mNoGallery = $parserOutput->getNoGallery();
13101311 $this->mHeadItems = array_merge( $this->mHeadItems, $parserOutput->getHeadItems() );
13111312 $this->addModules( $parserOutput->getModules() );
1312 - // Versioning...
1313 - foreach ( (array)$parserOutput->mTemplateIds as $ns => $dbks ) {
 1313+
 1314+ // Template versioning...
 1315+ foreach ( (array)$parserOutput->getTemplateIds() as $ns => $dbks ) {
13141316 if ( isset( $this->mTemplateIds[$ns] ) ) {
13151317 $this->mTemplateIds[$ns] = $dbks + $this->mTemplateIds[$ns];
13161318 } else {
13171319 $this->mTemplateIds[$ns] = $dbks;
13181320 }
13191321 }
 1322+ // File versioning...
 1323+ foreach ( (array)$parserOutput->getImageTimeKeys() as $dbk => $data ) {
 1324+ $this->mImageTimeKeys[$dbk] = $data;
 1325+ }
13201326
13211327 // Hooks registered in the object
13221328 global $wgParserOutputHooks;
Index: trunk/phase3/includes/ImageGallery.php
@@ -257,11 +257,8 @@
258258 $time = $sha1 = $descQuery = false;
259259 wfRunHooks( 'BeforeGalleryFindFile',
260260 array( &$this, &$nt, &$time, &$descQuery, &$sha1 ) );
261 - # Get the file and register it
262 - $img = $this->mParser->fetchFile( $nt, $time, $sha1 );
263 - if ( $img ) {
264 - $nt = $img->getTitle(); // file title may be different (via hooks)
265 - }
 261+ # Fetch and register the file (file title may be different via hooks)
 262+ list( $img, $nt ) = $this->mParser->fetchFileAndTitle( $nt, $time, $sha1 );
266263 } else {
267264 $img = false;
268265 }

Follow-up revisions

RevisionCommit summaryAuthorDate
r84618Straighten out dependency handling from r84610 (files) and before (templates)aaron18:23, 23 March 2011
r84619Follow-up r84610: removed fr_fileSHA1Keys and use core file version tracking ...aaron18:27, 23 March 2011
r84632* Follow-up r84610: don't assume a Parser object is attached...aaron21:21, 23 March 2011
r84636Follow-up r84610: made makeImage() b/c for callersaaron21:45, 23 March 2011
r85305Follow-up changes to r84610:...aaron01:22, 4 April 2011
r96163Reverted parser bump from r84601 per CRaaron23:14, 2 September 2011
r96164Follow-up r96163: changed r84610 et al FlaggedRevs code to deal with parser o...aaron23:19, 2 September 2011
r96357Cleanup to r84610 per CR: changed BeforeParserMakeImageLinkObj hook to use a ...aaron18:11, 6 September 2011
r97087FU r84610: Renamed getImageTimeKeys() to getFileSearchOptions()aaron19:05, 14 September 2011
r97173Merged revisions 97087,97091-97092,97094,97096-97098,97100-97101,97103,97136,...dantman16:19, 15 September 2011

Past revisions this follows-up on

RevisionCommit summaryAuthorDate
r84591* Made BeforeParserMakeImageLinkObj/BeforeGalleryFindFile let hooks set sha1 ...aaron03:13, 23 March 2011

Comments

#Comment by TheDJ (talk | contribs)   20:55, 23 March 2011

Fatal error: Call to a member function fetchFileAndTitle() on a non-object in /Users/hartman/Development/phase3/includes/ImageGallery.php on line 261

bug 28205

#Comment by 😂 (talk | contribs)   20:57, 23 March 2011

Making scaptrap, bumping the version causes mass cache invalidations, right?

#Comment by OverlordQ (talk | contribs)   21:29, 23 March 2011

Breaks ParserFunctions as well.

#Comment by Aaron Schulz (talk | contribs)   21:31, 23 March 2011

What error? The same?

#Comment by OverlordQ (talk | contribs)   21:35, 23 March 2011
Running test {{#ifexist}}... PHP Warning:  Missing argument 2 for ParserOutput::addImage(), called in /var/www/site/w/extensions/ParserFunctions/ParserFunctions_body.php on line 339 and defined in /var/www/site/w/includes/parser/ParserOutput.php on line 268

Warning: Missing argument 2 for ParserOutput::addImage(), called in /var/www/site/w/extensions/ParserFunctions/ParserFunctions_body.php on line 339 and defined in /var/www/site/w/includes/parser/ParserOutput.php on line 268
PHP Warning:  Missing argument 3 for ParserOutput::addImage(), called in /var/www/site/w/extensions/ParserFunctions/ParserFunctions_body.php on line 339 and defined in /var/www/site/w/includes/parser/ParserOutput.php on line 268

Warning: Missing argument 3 for ParserOutput::addImage(), called in /var/www/site/w/extensions/ParserFunctions/ParserFunctions_body.php on line 339 and defined in /var/www/site/w/includes/parser/ParserOutput.php on line 268
PHP Notice:  Undefined variable: timestamp in /var/www/site/w/includes/parser/ParserOutput.php on line 270

Notice: Undefined variable: timestamp in /var/www/site/w/includes/parser/ParserOutput.php on line 270
PHP Notice:  Undefined variable: sha1 in /var/www/site/w/includes/parser/ParserOutput.php on line 270

Notice: Undefined variable: sha1 in /var/www/site/w/includes/parser/ParserOutput.php on line 270

In the ifexist handler, PFuncs makes use of $parser->mOutput->addImage( $file->getName() );

#Comment by Tim Starling (talk | contribs)   07:19, 1 September 2011

What is the reason for changing fetchFile() to fetchFileAndTitle()? Is there some scenario where the title it returns is different to $file->getTitle()? Or is it just to save two lines of code in the caller?

#Comment by Aaron Schulz (talk | contribs)   23:30, 2 September 2011

Probably was just for consistency with fetchTemplateAndTitle() and convenience.

#Comment by Tim Starling (talk | contribs)   05:32, 2 September 2011

Comments on this project in general:

I think you should provide an associative array of options to Parser::fetchFileAndTitle(), suitable for passing through to RepoGroup::findFile or RepoGroup::findFileFromKey(), instead of passing through the $time and $sha1 parameters separately. Also, I think instead of adding a $sha1 parameter to the BeforeParserMakeImageLinkObj hook (which you did in r84591), you should give a reference to the options array, and the use of the $time parameter should be deprecated. Then instead of the rather strange and undocumented convention you introduced here, allowing the hook to set $time to the string "0" to indicate that the image should not be shown, you should add an optional element to the options array specifically for that purpose.

Similarly, the time and sha1 tracking introduced here to ParserOutput and OutputPage should be altered to track these options arrays instead. ParserOutput::getImageTimeKeys() could be replaced by ParserOutput::getImageSearchOptions().

The increment to Parser::VERSION is unnecessary and damaging and should be reverted. Instead, make FlaggedRevs gracefully handle the case where the relevant ParserOutput key is missing: have it use fr_fileSHA1Keys instead.

Note that Parser::VERSION is meant to be the MediaWiki version where it was last changed. Presumably if you updated it now you would change it to 1.18.0.

#Comment by Aaron Schulz (talk | contribs)   23:39, 6 September 2011

The changes should be more or less done now, except for using ParserOutput::getImageSearchOptions(), which I can't say I'm sold on.

#Comment by Tim Starling (talk | contribs)   04:32, 13 September 2011

Why are you not sold on it?

#Comment by Aaron Schulz (talk | contribs)   17:30, 13 September 2011

I can't quite see the rationale for the change. FlaggedRevs would have to use a hook and the search params to refetch all the files again after Parser (the RepoGroup cache doesn't cover fetch-by-hash). Part of this project was to avoid duplicated queries.

There are two ways files end up fetched on parse: (a) Only current versions are used for all files. In this case, FR wants to know the sha1/time of the current files. (b) FR tries to use stable versions (falling back to the current version if no version is specified) for all files. Here, FR wants to know the template versions as well. Certainly when a current version had to be used a fetch is needed. It's also needed if FR specified the version, since the file may not exist.

We already have template ID tracking. I don't see what's wrong with file tracking.

#Comment by Tim Starling (talk | contribs)   11:42, 14 September 2011

There's nothing wrong with file tracking. I'm just saying that the time should be stored in an associative array along with any other relevant options, so that the ParserOutput interface can be more stable. Something like

Index: dataclasses/FlaggedRevs.class.php
===================================================================
--- dataclasses/FlaggedRevs.class.php	(revision 96956)
+++ dataclasses/FlaggedRevs.class.php	(working copy)
@@ -595,7 +595,7 @@
 	 * @return bool
 	 */
 	public static function parserOutputIsVersioned( ParserOutput $pOut ) {
-		return ( $pOut->getTemplateIds() !== null && $pOut->getImageTimeKeys() !== null );
+		return ( $pOut->getTemplateIds() !== null && $pOut->getFileSearchOptions() !== null );
 	}
 
 	# ################ Tracking/cache update update functions #################
Index: presentation/FlaggedPageView.php
===================================================================
--- presentation/FlaggedPageView.php	(revision 96956)
+++ presentation/FlaggedPageView.php	(working copy)
@@ -1103,7 +1103,13 @@
 			# Note: showStableVersion() already makes sure that $wgOut has the stable inclusion versions.
 			if ( $this->out->getRevisionId() == $rev->getId() && empty( $this->out->fr_unversionedIncludes ) ) {
 				$tmpVers = $this->out->getTemplateIds();
-				$fileVers = $this->out->getImageTimeKeys();
+				$fileOptions = $this->out->getFileSearchOptions();
+				$fileVers = array();
+				foreach ( $fileOptions as $fileName => $opt ) {
+					if ( isset( $opt['time'] ) ) {
+						$fileVers[$fileName] = $opt['time'];
+					}
+				}
 			} elseif ( $this->oldRevIncludes ) { // e.g. diffonly=1, stable diff
 				# We may have already fetched the inclusion IDs to get the template/file changes.
 				list( $tmpVers, $fileVers ) = $this->oldRevIncludes; // reuse
#Comment by Aaron Schulz (talk | contribs)   19:09, 14 September 2011

Oh, I thought you meant "params parser used to search" not "params to search to find the same file parser used".

I don't quite get the foreach loop change above. getImageTimeKeys() already returns an assoc array for each file name with a key for time/sha1. I didn't add 'broken' since only Parser uses that.

#Comment by Aaron Schulz (talk | contribs)   23:26, 2 September 2011

The Parser version change has been reverted.

Status & tagging log