r111031 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r111030‎ \| r111031 \| r111032 >
Date:	13:44, 9 February 2012
Author:	gwicke
Status:	deferred
Tags:
Comment:	Move attribute expansion back to separate handler, as this makes it easier to only expand used branches selected by parser functions. Template (and -argument) expansion is simply registered before general expansion. Additionally, a few more simple time-based magic words are added in ParserFunctions.
Modified paths:	/trunk/extensions/VisualEditor/modules/parser/ext.core.AttributeExpander.js (added) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
—	—	@@ -486,28 +486,7 @@
487	487
488	488	for ( var i = 0; i < tokensLength; i++ ) {
489	489	token = tokens[i];
490		-
491		~~- if ( token.attribs ) {~~
492		~~- // TODO: Transform token attributes (that need it) before general token processing.~~
493		~~- // Sync return -> continue as-is.~~
494		~~- // Async return -> return with async~~
495		~~- // per-item callback: function ( res ) {~~
496		~~- var atm = new AttributeTransformManager (~~
497		~~- this,~~
498		~~- this._returnAttributes.bind( this, token, cb )~~
499		~~- );~~
500	490
501		~~- if ( atm.process( token.attribs ) ) {~~
502		~~- // async attribute expansion ongoing~~
503		~~- activeAccum = accum;~~
504		~~- accum = new TokenAccumulator( this, activeAccum.getParentCB( 'sibling' ) );~~
505		~~- cb = accum.getParentCB( 'child' );~~
506		~~- // Only transform token once attributes are fully processed~~
507		~~- // _returnAttributes will call transformTokens if sync flag is not set~~
508		~~- continue;~~
509		~~- }~~
510		~~- }~~
511		-
512	491	switch ( token.constructor ) {
513	492	case String:
514	493	res = this._transformToken( token, phaseEndRank, ts.text, cb );
—	—	@@ -566,8 +545,6 @@
567	546	accum = new TokenAccumulator( this, activeAccum.getParentCB( 'sibling' ) );
568	547	cb = accum.getParentCB( 'child' );
569	548	}
570		-
571		-
572	549	}
573	550
574	551	// Return finished tokens directly to caller, and indicate if further
—	—	@@ -613,36 +590,6 @@
614	591	}
615	592	};
616	593
617		~~-AsyncTokenTransformManager.prototype._returnAttributes~~
618		~~- = function ( token, cb, attributes, async )~~
619		-{
620		~~- this.env.dp( 'AsyncTokenTransformManager._returnAttributes: ' +~~
621		~~- JSON.stringify( token, null, 2 ) +~~
622		~~- JSON.stringify( attributes, null, 2 ) );~~
623		~~- token.attribs = attributes;~~
624		-
625		-
626		~~- if ( async ) {~~
627		~~- // Attributes were expanded asynchronously, so now we still need to~~
628		~~- // process the token.~~
629		~~- var res = this.transformTokens( [token], cb );~~
630		~~- if ( res.async ) {~~
631		~~- // Token transformation is again asynchronous. There is already a~~
632		~~- // TokenAccumulator associated with this token, so we only need to~~
633		~~- // be concerned with synchronously returned tokens.~~
634		~~- if ( res.tokens ) {~~
635		~~- // Pass on already-returned tokens to the cb~~
636		~~- cb( res.tokens, true );~~
637		~~- }~~
638		~~- } else if ( res.token ) {~~
639		~~- cb ( [res.token], false );~~
640		~~- } else {~~
641		~~- cb ( res.tokens, false );~~
642		~~- }~~
643		~~- }~~
644		~~-};~~
645		-
646		-
647	594	/**
648	595	* Callback for the end event emitted from the tokenizer.
649	596	* Either signals the end of input to the tail of an ongoing asynchronous
—	—	@@ -858,11 +805,11 @@
859	806	if ( this.outstanding === 0 ) {
860	807	this._returnAttributes();
861	808	// synchronous / done
862		~~- return false;~~
	809	+ return true;
863	810	} else {
864	811	// async, will call back
865	812	this.async = true;
866		~~- return true;~~
	813	+ return false;
867	814	}
868	815	};
869	816
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
—	—	@@ -34,8 +34,8 @@
35	35	return argDict['#default'];
36	36	} else {
37	37	var lastKV = argList[argList.length - 1];
38		~~- if ( lastKV && ! lastKV[0].length ) {~~
39		~~- return lastKV[1];~~
	38	+ if ( lastKV && ! lastKV.v.length ) {
	39	+ return lastKV.v;
40	40	} else {
41	41	return [];
42	42	}
—	—	@@ -47,10 +47,10 @@
48	48	if ( argList.length < 2 ) {
49	49	return [];
50	50	} else {
51		~~- if ( target.trim() === this.manager.env.tokensToString( argList[0][1] ).trim() ) {~~
52		~~- return ( argList[1] && argList[1][1]) \|\| [];~~
	51	+ if ( target.trim() === this.manager.env.tokensToString( argList[0].v ).trim() ) {
	52	+ return ( argList[1] && argList[1].v) \|\| [];
53	53	} else {
54		~~- return ( argList[2] && argList[2][1]) \|\| [];~~
	54	+ return ( argList[2] && argList[2].v) \|\| [];
55	55	}
56	56	}
57	57	};
—	—	@@ -78,6 +78,17 @@
79	79	return [];
80	80	}
81	81	};
	82	+ParserFunctions.prototype['pf_padleft'] = function ( target, argList, argDict ) {
	83	+ if ( '1' in argDict ) {
	84	+ var n = argDict[1];
	85	+ while ( target.length < n ) {
	86	+ target = '0' + target;
	87	+ }
	88	+ return [target];
	89	+ } else {
	90	+ return [];
	91	+ }
	92	+};
82	93
83	94	ParserFunctions.prototype['pf_#tag'] = function ( target, argList, argDict ) {
84	95	// XXX: handle things like {{#tag:nowiki\|{{{input1\|[[shouldnotbelink]]}}}}}
—	—	@@ -87,6 +98,25 @@
88	99	[ new EndTagTk( target ) ] );
89	100	};
90	101
	102	+ParserFunctions.prototype['pf_currentyear'] = function ( target, argList, argDict ) {
	103	+ return this['pf_#time']( 'Y', [], {} );
	104	+};
	105	+ParserFunctions.prototype['pf_currentmonth'] = function ( target, argList, argDict ) {
	106	+ return this['pf_#time']( 'm', [], {} );
	107	+};
	108	+ParserFunctions.prototype['pf_currentmonthname'] = function ( target, argList, argDict ) {
	109	+ return this['pf_#time']( 'J', [], {} );
	110	+};
	111	+ParserFunctions.prototype['pf_currentmonthabbrev'] = function ( target, argList, argDict ) {
	112	+ return this['pf_#time']( 'M', [], {} );
	113	+};
	114	+ParserFunctions.prototype['pf_currentday'] = function ( target, argList, argDict ) {
	115	+ return this['pf_#time']( 'j', [], {} );
	116	+};
	117	+ParserFunctions.prototype['pf_currentdayname'] = function ( target, argList, argDict ) {
	118	+ return this['pf_#time']( 'l', [], {} );
	119	+};
	120	+
91	121	// A first approximation, anyway..
92	122	// Based on http://jacwright.com/projects/javascript/date_format/ for now, MIT
93	123	// licensed.
—	—	@@ -94,7 +124,7 @@
95	125	var res,
96	126	tpl = target.trim();
97	127	//try {
98		~~- // var date = new Date( this.manager.env.tokensToString( argList[0][1] ) );~~
	128	+ // var date = new Date( this.manager.env.tokensToString( argList[0].v ) );
99	129	// res = [ date.format( target ) ];
100	130	//} catch ( e ) {
101	131	// this.manager.env.dp( 'ERROR: #time ' + e );
—	—	@@ -191,16 +221,16 @@
192	222	return [ 'class="error" in expression ' + target ];
193	223	}
194	224	if ( res ) {
195		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
	225	+ return ( argList[0] && argList[0].v ) \|\| [];
196	226	} else {
197		~~- return ( argList[1] && argList[1][1] ) \|\| [];~~
	227	+ return ( argList[1] && argList[1].v ) \|\| [];
198	228	}
199	229	};
200	230	ParserFunctions.prototype['pf_#iferror'] = function ( target, argList, argDict ) {
201	231	if ( target.indexOf( 'class="error"' ) >= 0 ) {
202		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
	232	+ return ( argList[0] && argList[0].v ) \|\| [];
203	233	} else {
204		~~- return ( argList[1] && argList[1][1] ) \|\| [];~~
	234	+ return ( argList[1] && argList[1].v ) \|\| [];
205	235	}
206	236	};
207	237	ParserFunctions.prototype['pf_#expr'] = function ( target, argList, argDict ) {
—	—	@@ -234,7 +264,7 @@
235	265
236	266	// FIXME
237	267	ParserFunctions.prototype['pf_#ifexist'] = function ( target, argList, argDict ) {
238		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
	268	+ return ( argList[0] && argList[0].v ) \|\| [];
239	269	};
240	270	ParserFunctions.prototype['pf_formatnum'] = function ( target, argList, argDict ) {
241	271	return [ target ];
—	—	@@ -251,6 +281,9 @@
252	282	ParserFunctions.prototype['pf_pagename'] = function ( target, argList, argDict ) {
253	283	return [ target ];
254	284	};
	285	+ParserFunctions.prototype['pf_pagenamee'] = function ( target, argList, argDict ) {
	286	+ return [ target ];
	287	+};
255	288	ParserFunctions.prototype['pf_fullpagename'] = function ( target, argList, argDict ) {
256	289	return [target];
257	290	};
—	—	@@ -283,6 +316,9 @@
284	317	ParserFunctions.prototype['pf_talkspace'] = function ( target, argList, argDict ) {
285	318	return ['Talk'];
286	319	};
	320	+ParserFunctions.prototype['pf_numberofarticles'] = function ( target, argList, argDict ) {
	321	+ return ["1"];
	322	+};
287	323
288	324	if (typeof module == "object") {
289	325	module.exports.ParserFunctions = ParserFunctions;
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.AttributeExpander.js
—	—	@@ -0,0 +1,91 @@
	2	+/**
	3	+ * Geniric attribute expansion handler.
	4	+ *
	5	+ * @author Gabriel Wicke <gwicke@wikimedia.org>
	6	+ * @author Brion Vibber <brion@wikimedia.org>
	7	+ */
	8	+var $ = require('jquery'),
	9	+ request = require('request'),
	10	+ events = require('events'),
	11	+ qs = require('querystring'),
	12	+ ParserFunctions = require('./ext.core.ParserFunctions.js').ParserFunctions,
	13	+ AttributeTransformManager = require('./mediawiki.TokenTransformManager.js')
	14	+ .AttributeTransformManager,
	15	+ defines = require('./mediawiki.parser.defines.js');
	16	+
	17	+
	18	+function AttributeExpander ( manager ) {
	19	+ this.reset();
	20	+ this.register( manager );
	21	+}
	22	+
	23	+AttributeExpander.prototype.reset = function ( token ) {
	24	+ return {token: token};
	25	+};
	26	+
	27	+// constants
	28	+AttributeExpander.prototype.rank = 1.11;
	29	+
	30	+AttributeExpander.prototype.register = function ( manager ) {
	31	+ this.manager = manager;
	32	+ // Register for template and templatearg tag tokens
	33	+ manager.addTransform( this.onToken.bind(this),
	34	+ this.rank, 'any' );
	35	+
	36	+};
	37	+
	38	+
	39	+/**
	40	+ * Token handler
	41	+ *
	42	+ * Expands target and arguments (both keys and values) and either directly
	43	+ * calls or sets up the callback to _expandTemplate, which then fetches and
	44	+ * processes the template.
	45	+ */
	46	+AttributeExpander.prototype.onToken = function ( token, frame, cb ) {
	47	+ if ( token.constructor === TagTk && token.attributes ) {
	48	+ var expandData = {
	49	+ token: token,
	50	+ cb: cb
	51	+ };
	52	+ var atm = new AttributeTransformManager(
	53	+ this.manager,
	54	+ this._returnAttributes.bind( this, expandData )
	55	+ );
	56	+ if( atm.process( token.attribs ) ) {
	57	+ // Attributes were transformed synchronously
	58	+ this.manager.env.dp (
	59	+ 'sync attribs for ' + JSON.stringify( tplExpandData.target ),
	60	+ tplExpandData.expandedArgs
	61	+ );
	62	+ // All attributes are fully expanded synchronously (no IO was needed)
	63	+ return { token: token };
	64	+ } else {
	65	+ // Async attribute expansion is going on
	66	+ this.manager.env.dp( 'async return for ' + JSON.stringify( token ));
	67	+ expandData.async = true;
	68	+ return { async: true };
	69	+ }
	70	+ } else {
	71	+ return { token: token };
	72	+ }
	73	+};
	74	+
	75	+
	76	+/**
	77	+ * Callback for argument (including target) expansion in AttributeTransformManager
	78	+ */
	79	+AttributeExpander.prototype._returnAttributes = function ( expandData,
	80	+ attributes )
	81	+{
	82	+ this.manager.env.dp( 'AttributeExpander._returnAttributes: ' + JSON.stringify(attributes) );
	83	+ // Remove the target from the attributes
	84	+ expandData.token.attribs = attributes;
	85	+ if ( expandData.async ) {
	86	+ expandData.cb( [expandData.token], false );
	87	+ }
	88	+};
	89	+
	90	+if (typeof module == "object") {
	91	+ module.exports.AttributeExpander = AttributeExpander;
	92	+}
Property changes on: trunk/extensions/VisualEditor/modules/parser/ext.core.AttributeExpander.js
___________________________________________________________________
Added: svn:eol-style
1	93	+ native
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
—	—	@@ -24,6 +24,7 @@
25	25	.PostExpandParagraphHandler,
26	26	Sanitizer = require('./ext.core.Sanitizer.js').Sanitizer,
27	27	TemplateHandler = require('./ext.core.TemplateHandler.js').TemplateHandler,
	28	+ AttributeExpander = require('./ext.core.AttributeExpander.js').AttributeExpander,
28	29	Cite = require('./ext.Cite.js').Cite,
29	30	FauxHTML5 = require('./mediawiki.HTML5TreeBuilder.node.js').FauxHTML5,
30	31	DOMPostProcessor = require('./mediawiki.DOMPostProcessor.js').DOMPostProcessor,
—	—	@@ -176,6 +177,7 @@
177	178
178	179	// Register template expansion extension
179	180	new TemplateHandler( tokenExpander );
	181	+ new AttributeExpander( tokenExpander );
180	182
181	183	tokenExpander.listenForTokensFrom ( tokenPreProcessor );
182	184	// XXX: hack.
—	—	@@ -224,6 +226,7 @@
225	227	args, this.env
226	228	);
227	229	new TemplateHandler( tokenExpander );
	230	+ new AttributeExpander( tokenExpander );
228	231	tokenExpander.listenForTokensFrom ( tokenPreProcessor );
229	232
230	233	return new CachedTokenPipeline(
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
—	—	@@ -65,6 +65,7 @@
66	66	cb: cb,
67	67	origToken: token,
68	68	resultTokens: [],
	69	+ attribsAsync: true,
69	70	overallAsync: false,
70	71	expandDone: false
71	72	},
—	—	@@ -72,16 +73,30 @@
73	74	i = 0,
74	75	res;
75	76
76		~~- tplExpandData.target = token.attribs.shift().v;~~
77		~~- tplExpandData.expandedArgs = this._nameArgs( token.attribs );~~
	77	+ var attributes = [token.attribs.shift()].concat( this._nameArgs( token.attribs ) );
78	78
79		~~- // Attributes were transformed synchronously~~
80		~~- //this.manager.env.dp (~~
81		~~- // 'sync attribs for ' + JSON.stringify( tplExpandData.target ),~~
82		~~- // tplExpandData.expandedArgs~~
83		~~- //);~~
84		~~- // All attributes are fully expanded synchronously (no IO was needed)~~
85		~~- return this._expandTemplate ( tplExpandData );~~
	79	+ this.manager.env.dp( 'before AttributeTransformManager: ' +
	80	+ JSON.stringify( attributes, null, 2 ) );
	81	+ new AttributeTransformManager(
	82	+ this.manager,
	83	+ this._returnAttributes.bind( this, tplExpandData )
	84	+ ).process( attributes );
	85	+
	86	+ // Unblock finish
	87	+ if ( ! tplExpandData.attribsAsync ) {
	88	+ // Attributes were transformed synchronously
	89	+ this.manager.env.dp (
	90	+ 'sync attribs for ' + JSON.stringify( tplExpandData.target ),
	91	+ tplExpandData.expandedArgs
	92	+ );
	93	+ // All attributes are fully expanded synchronously (no IO was needed)
	94	+ return this._expandTemplate ( tplExpandData );
	95	+ } else {
	96	+ // Async attribute expansion is going on
	97	+ this.manager.env.dp( 'async return for ' + JSON.stringify( token ));
	98	+ tplExpandData.overallAsync = true;
	99	+ return { async: true };
	100	+ }
86	101	};
87	102
88	103	/**
—	—	@@ -103,6 +118,22 @@
104	119	return out;
105	120	};
106	121
	122	+/**
	123	+ * Callback for argument (including target) expansion in AttributeTransformManager
	124	+ */
	125	+TemplateHandler.prototype._returnAttributes = function ( tplExpandData,
	126	+ attributes )
	127	+{
	128	+ this.manager.env.dp( 'TemplateHandler._returnAttributes: ' + JSON.stringify(attributes) );
	129	+ // Remove the target from the attributes
	130	+ tplExpandData.attribsAsync = false;
	131	+ tplExpandData.target = attributes[0].v;
	132	+ attributes.shift();
	133	+ tplExpandData.expandedArgs = attributes;
	134	+ if ( tplExpandData.overallAsync ) {
	135	+ this._expandTemplate ( tplExpandData );
	136	+ }
	137	+};
107	138
108	139	/**
109	140	* Fetch, tokenize and token-transform a template after all arguments and the
—	—	@@ -214,10 +245,9 @@
215	246	// expandDone is set to true in our _onEnd handler.
216	247	if ( tplExpandData.overallAsync \|\|
217	248	! tplExpandData.expandDone ) {
	249	+ tplExpandData.overallAsync = true;
218	250	this.manager.env.dp( 'Async return from _expandTemplate for ' +
219		~~- JSON.stringify ( tplExpandData.target ) +~~
220		~~- JSON.stringify( tplExpandData, null, 2 ));~~
221		~~- tplExpandData.overallAsync = true;~~
	251	+ JSON.stringify ( tplExpandData.target ) );
222	252	return { async: true };
223	253	} else {
224	254	this.manager.env.dp( 'Sync return from _expandTemplate for ' +
—	—	@@ -288,13 +318,12 @@
289	319	// @fixme normalize name?
290	320	var self = this;
291	321	if ( title in this.manager.env.pageCache ) {
292		~~- callback( self.manager.env.pageCache[title], title );~~
293		~~- //// Unwind the stack~~
294		~~- //process.nextTick(~~
295		~~- // function () {~~
296		~~- // callback( self.manager.env.pageCache[title], title );~~
297		~~- // }~~
298		~~- //);~~
	322	+ // Unwind the stack
	323	+ process.nextTick(
	324	+ function () {
	325	+ callback( self.manager.env.pageCache[title], title );
	326	+ }
	327	+ );
299	328	} else if ( ! this.manager.env.fetchTemplates ) {
300	329	callback( 'Warning: Page/template fetching disabled, and no cache for ' +
301	330	title, title );
—	—	@@ -324,26 +353,51 @@
325	354	* Expand template arguments with tokens from the containing frame.
326	355	*/
327	356	TemplateHandler.prototype.onTemplateArg = function ( token, frame, cb ) {
328		~~- var argName = this.manager.env.tokensToString( token.attribs.shift().v ).trim(),~~
	357	+
	358	+ token.resultTokens = false;
	359	+
	360	+ new AttributeTransformManager (
	361	+ this.manager,
	362	+ this._returnArgAttributes.bind( this, token, cb, frame )
	363	+ ).process( token.attribs );
	364	+
	365	+ if ( token.resultTokens !== false ) {
	366	+ // synchronous return
	367	+ //console.log( 'synchronous attribute expand: ' + JSON.stringify( token.resultTokens ) );
	368	+
	369	+ return { tokens: token.resultTokens };
	370	+ } else {
	371	+ //console.log( 'asynchronous attribute expand: ' + JSON.stringify( token, null, 2 ) );
	372	+ // asynchronous return
	373	+ token.resultTokens = [];
	374	+ return { async: true };
	375	+ }
	376	+};
	377	+
	378	+TemplateHandler.prototype._returnArgAttributes = function ( token, cb, frame, attributes ) {
	379	+ //console.log( '_returnArgAttributes: ' + JSON.stringify( attributes ));
	380	+ var argName = this.manager.env.tokensToString( attributes[0].v ).trim(),
329	381	res;
330		-
331	382	if ( argName in this.manager.args ) {
332	383	// return tokens for argument
333	384	//console.log( 'templateArg found: ' + argName +
334	385	// ' vs. ' + JSON.stringify( this.manager.args ) );
335	386	res = this.manager.args[argName];
336	387	} else {
337		~~- var defaultValue = token.attribs[0];~~
	388	+ var defaultValue = (attributes[1] && ! attributes[1].k.length && attributes[1].v) \|\| false;
338	389	this.manager.env.dp( 'templateArg not found: ' + argName +
339	390	' vs. ' + JSON.stringify( defaultValue ) );
340		~~- if ( defaultValue && ! defaultValue.k.length && defaultValue.v.length ) {~~
341		~~- res = defaultValue.v;~~
	391	+ if ( defaultValue ) {
	392	+ res = defaultValue;
342	393	} else {
343	394	res = [ '{{{' + argName + '}}}' ];
344	395	}
345	396	}
346		~~- return { tokens: res };~~
347		-
	397	+ if ( token.resultTokens !== false ) {
	398	+ cb( res );
	399	+ } else {
	400	+ token.resultTokens = res;
	401	+ }
348	402	};
349	403
350	404

Status & tagging log

13:45, 9 February 2012 GWicke (talk | contribs) changed the status of r111031 [removed: new added: deferred]