r110406 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r110405‎ \| r110406 \| r110407 >
Date:	16:50, 31 January 2012
Author:	gwicke
Status:	deferred
Tags:
Comment:	A few more debug helpers including a --trace mode for light debugging. Some improvements to parser functions on the way to support the cite extensions. Preparation for generic template and template arg in attribute support. 222 parser tests now passing.
Modified paths:	/trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/parse.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
—	—	@@ -12,7 +12,7 @@
13	13	manager,
14	14	function ( tokens ) {
15	15	if ( isInclude ) {
16		~~- manager.env.dp( 'noinclude stripping', tokens );~~
	16	+ manager.env.tp( 'noinclude stripping' );
17	17	return {};
18	18	} else {
19	19	tokens.shift();
—	—	@@ -42,6 +42,7 @@
43	43	}
44	44	return { tokens: tokens };
45	45	} else {
	46	+ manager.env.tp( 'includeonly stripping' );
46	47	return {};
47	48	}
48	49	},
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
—	—	@@ -214,6 +214,11 @@
215	215	}
216	216	// Transform token with side effects
217	217	res = transformer.transform( res.token, cb, this, this.prevToken );
	218	+ // XXX: Sync transform:
	219	+ // res = transformer.transform( res.token, this, this.prevToken );
	220	+ // XXX: Async transform:
	221	+ // res = transformer.transform( res.token, this, cb );
	222	+
218	223	// if multiple tokens or null token: process returned tokens (in parent)
219	224	if ( !res.token \|\| // async implies tokens instead of token, so no
220	225	// need to check explicitly
—	—	@@ -725,7 +730,7 @@
726	731
727	732	// transform each argument (key and value), and handle asynchronous returns
728	733	for ( var i = 0, l = attributes.length; i < l; i++ ) {
729		~~- kv = { key: [], value: [] };~~
	734	+ var kv = { key: [], value: [] };
730	735	this.kvs.push( kv );
731	736
732	737	// Assume that the return is async, will be decremented in callback
Index: trunk/extensions/VisualEditor/modules/parser/parse.js
—	—	@@ -23,6 +23,11 @@
24	24	'boolean': true,
25	25	'default': false
26	26	},
	27	+ 'trace': {
	28	+ description: 'Trace mode (light debugging), implied by --debug',
	29	+ 'boolean': true,
	30	+ 'default': false
	31	+ },
27	32	'maxdepth': {
28	33	description: 'Maximum expansion depth',
29	34	'boolean': false,
—	—	@@ -52,6 +57,7 @@
53	58	fetchTemplates: argv.fetchTemplates,
54	59	// enable/disable debug output using this switch
55	60	debug: argv.debug,
	61	+ trace: argv.trace,
56	62	maxDepth: argv.maxdepth
57	63	} ),
58	64	parser = new ParserPipeline( env );
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
—	—	@@ -14,10 +14,10 @@
15	15
16	16	ParserFunctions.prototype['pf_#if'] = function ( target, argList, argDict ) {
17	17	if ( target.trim() !== '' ) {
18		~~- this.manager.env.dp('#if, first branch', argDict[1] );~~
	18	+ this.manager.env.dp('#if, first branch', target.trim(), argDict[1] );
19	19	return argDict[1] \|\| [];
20	20	} else {
21		~~- this.manager.env.dp('#if, second branch', argDict[2] );~~
	21	+ this.manager.env.dp('#if, second branch', target.trim(), argDict[2] );
22	22	return argDict[2] \|\| [];
23	23	}
24	24	};
—	—	@@ -25,10 +25,11 @@
26	26	ParserFunctions.prototype['pf_#switch'] = function ( target, argList, argDict ) {
27	27	this.manager.env.dp( 'switch enter: ' + target.trim() +
28	28	' looking in ', argDict );
29		~~- if ( target.trim() in argDict ) {~~
30		~~- this.manager.env.dp( 'switch found: ' + target.trim() +~~
31		~~- ' res=', argDict[target.trim()] );~~
32		~~- return argDict[target.trim()];~~
	29	+ target = target.trim();
	30	+ if ( target in argDict ) {
	31	+ this.manager.env.dp( 'switch found: ' + target +
	32	+ ' res=', argDict[target] );
	33	+ return argDict[target];
33	34	} else if ( '#default' in argDict ) {
34	35	return argDict['#default'];
35	36	} else {
—	—	@@ -149,9 +150,9 @@
150	151	m: function() { return (this.getMonth() < 9 ? '0' : '') + (this.getMonth() + 1); },
151	152	M: function() { return Date.replaceChars.shortMonths[this.getMonth()]; },
152	153	n: function() { return this.getMonth() + 1; },
153		~~- t: function() { var d = new Date(); return new Date(d.getFullYear(), d.getMonth(), 0).getDate() }, // Fixed now, gets #days of date~~
	154	+ t: function() { var d = new Date(); return new Date(d.getFullYear(), d.getMonth(), 0).getDate(); }, // Fixed now, gets #days of date
154	155	// Year
155		~~- L: function() { var year = this.getFullYear(); return (year % 400 == 0 \|\| (year % 100 != 0 && year % 4 == 0)); }, // Fixed now~~
	156	+ L: function() { var year = this.getFullYear(); return (year % 400 === 0 \|\| (year % 100 !== 0 && year % 4 === 0)); }, // Fixed now
156	157	o: function() { var d = new Date(this.valueOf()); d.setDate(d.getDate() - ((this.getDay() + 6) % 7) + 3); return d.getFullYear();}, //Fixed now
157	158	Y: function() { return this.getFullYear(); },
158	159	y: function() { return ('' + this.getFullYear()).substr(2); },
—	—	@@ -180,23 +181,46 @@
181	182	U: function() { return this.getTime() / 1000; }
182	183	};
183	184
	185	+ParserFunctions.prototype['pf_#ifexpr'] = function ( target, argList, argDict ) {
	186	+ this.manager.env.dp( '#ifexp: ' + JSON.stringify( argList ) );
	187	+ var res;
	188	+ try {
	189	+ var f = new Function ( 'return (' + target + ')' );
	190	+ res = f();
	191	+ } catch ( e ) {
	192	+ return [{type: 'TEXT', value: 'class="error" in expression ' + target }];
	193	+ }
	194	+ if ( res ) {
	195	+ return ( argList[0] && argList[0][1] ) \|\| [];
	196	+ } else {
	197	+ return ( argList[1] && argList[1][1] ) \|\| [];
	198	+ }
	199	+};
	200	+ParserFunctions.prototype['pf_#iferror'] = function ( target, argList, argDict ) {
	201	+ if ( target.indexOf( 'class="error"' ) >= 0 ) {
	202	+ return ( argList[0] && argList[0][1] ) \|\| [];
	203	+ } else {
	204	+ return ( argList[1] && argList[1][1] ) \|\| [];
	205	+ }
	206	+};
	207	+ParserFunctions.prototype['pf_#expr'] = function ( target, argList, argDict ) {
	208	+ var res;
	209	+ try {
	210	+ var f = new Function ( 'return (' + target + ')' );
	211	+ res = f();
	212	+ } catch ( e ) {
	213	+ return [{type: 'TEXT', value: 'class="error" in expression ' + target }];
	214	+ }
	215	+ return [{type: 'TEXT', value: res.toString()}];
	216	+};
184	217
185	218
	219	+
186	220	/**
187	221	* Stub section: Pick any of these and actually implement them!
188	222	*/
189	223
190	224	// FIXME
191		~~-ParserFunctions.prototype['pf_#ifexpr'] = function ( target, argList, argDict ) {~~
192		~~- this.manager.env.dp( '#ifexp: ' + JSON.stringify( argList ) );~~
193		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
194		~~-};~~
195		~~-ParserFunctions.prototype['pf_#iferror'] = function ( target, argList, argDict ) {~~
196		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
197		~~-};~~
198		~~-ParserFunctions.prototype['pf_#expr'] = function ( target, argList, argDict ) {~~
199		~~- return ( argList[0] && argList[0][1] ) \|\| [];~~
200		~~-};~~
201	225	ParserFunctions.prototype['pf_#ifexist'] = function ( target, argList, argDict ) {
202	226	return ( argList[0] && argList[0][1] ) \|\| [];
203	227	};
—	—	@@ -225,12 +249,28 @@
226	250	return [{type: 'TEXT', value: target}];
227	251	};
228	252	ParserFunctions.prototype['pf_urlencode'] = function ( target, argList, argDict ) {
	253	+ this.manager.env.tp( 'urlencode: ' + target );
	254	+ return [{type: 'TEXT', value: target.trim()}];
	255	+};
	256	+ParserFunctions.prototype['pf_anchorencode'] = function ( target, argList, argDict ) {
229	257	return [{type: 'TEXT', value: target}];
230	258	};
231	259	ParserFunctions.prototype['pf_namespace'] = function ( target, argList, argDict ) {
232	260	return [{type: 'TEXT', value: 'Main'}];
233	261	};
	262	+ParserFunctions.prototype['pf_protectionlevel'] = function ( target, argList, argDict ) {
	263	+ return [{type: 'TEXT', value: ''}];
	264	+};
	265	+ParserFunctions.prototype['pf_ns'] = function ( target, argList, argDict ) {
	266	+ return [{type: 'TEXT', value: target}];
	267	+};
234	268
	269	+ParserFunctions.prototype['pf_subjectspace'] = function ( target, argList, argDict ) {
	270	+ return [{type: 'TEXT', value: 'Main'}];
	271	+};
	272	+ParserFunctions.prototype['pf_talkspace'] = function ( target, argList, argDict ) {
	273	+ return [{type: 'TEXT', value: 'Talk'}];
	274	+};
235	275
236	276	if (typeof module == "object") {
237	277	module.exports.ParserFunctions = ParserFunctions;
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
—	—	@@ -629,7 +629,7 @@
630	630	extlink
631	631	= "["
632	632	& { return setFlag('extlink'); }
633		~~- target:url~~
	633	+ target:(url / tplarg / template)
634	634	space*
635	635	text:inlineline?
636	636	"]" {
—	—	@@ -722,7 +722,7 @@
723	723	var obj = {
724	724	type: 'SELFCLOSINGTAG',
725	725	name: 'template',
726		~~- attribs: [['data-target', JSON.stringify(target)]],~~
	726	+ attribs: [], //[['data-target', JSON.stringify(target)]],
727	727	orderedArgs: params,
728	728	//args: {},
729	729	target: target
—	—	@@ -748,6 +748,7 @@
749	749	// parser only recognizes known self-closing tags for now, so use an
750	750	// explicit end tag for now.
751	751	//console.log(pp(obj));
	752	+ //console.log( 'tokenizer template ' + JSON.stringify( target ));
752	753	return obj;
753	754	}
754	755
—	—	@@ -770,12 +771,13 @@
771	772	};
772	773	if (params && params.length) {
773	774	// HACK, not final.
774		~~- obj.attribs.push(['data-defaultvalue', params[0][1]]);~~
775		~~- obj.attribs.push(['data-json-args', JSON.stringify(params)]);~~
	775	+ //obj.attribs.push(['data-defaultvalue', params[0][1]]);
	776	+ //console.log(JSON.stringify(params, null, 2));
	777	+ //obj.attribs.push(['data-json-args', JSON.stringify(params)]);
776	778	obj.defaultvalue = params[0][1].length ? params[0][1] :
777	779	[{type: 'TEXT', value: ''}];
778	780	}
779		~~- //console.log( 'tokenizer tplarg ' + JSON.stringify( obj ));~~
	781	+ //console.log( 'tokenizer tplarg ' + JSON.stringify( obj, null, 2 ));
780	782	return obj;
781	783	}
782	784
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
—	—	@@ -88,6 +88,9 @@
89	89	*/
90	90	this.treeBuilder = new FauxHTML5.TreeBuilder();
91	91	this.treeBuilder.listenForTokensFrom( this.tokenPostProcessor );
	92	+ //this.tokenPostProcessor.on('chunk', function( c ) {
	93	+ // console.log( JSON.stringify( c, null, 2 ));
	94	+ //} );
92	95
93	96	/**
94	97	* Final processing on the HTML DOM.
—	—	@@ -233,7 +236,7 @@
234	237
235	238	ParserPipeline.prototype.cachePipeline = function ( inputType, pipelinePart, pipe ) {
236	239	var cache = this.pipelineCache[inputType][pipelinePart];
237		~~- if ( cache && cache.length < 5 ) {~~
	240	+ if ( cache && cache.length < 2 ) {
238	241	cache.push( pipe );
239	242	}
240	243	};
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
—	—	@@ -160,16 +160,17 @@
161	161	// check for parser functions
162	162
163	163	// First, check the target for loops
164		~~- var target = this.manager.env.tokensToString( tplExpandData.target );~~
	164	+ var target = this.manager.env.tokensToString( tplExpandData.target ).trim();
165	165
166	166	var args = this.manager.env.KVtoHash( tplExpandData.expandedArgs );
167	167
168	168	this.manager.env.dp( 'argHash: ', args );
169	169
170		~~- var prefix = target.split(':', 1)[0].toLowerCase();~~
	170	+ var prefix = target.split(':', 1)[0].toLowerCase().trim();
171	171	if ( prefix && 'pf_' + prefix in this.parserFunctions ) {
172	172	var funcArg = target.substr( prefix.length + 1 );
173		~~- this.manager.env.dp( 'entering prefix', funcArg, args );~~
	173	+ this.manager.env.tp( 'func prefix: ' + prefix + ' arg=' + funcArg );
	174	+ //this.manager.env.dp( 'entering prefix', funcArg, args );
174	175	res = this.parserFunctions[ 'pf_' + prefix ]( funcArg,
175	176	tplExpandData.expandedArgs, args );
176	177
—	—	@@ -185,6 +186,7 @@
186	187	//data.reset();
187	188	}
188	189	}
	190	+ this.manager.env.tp( 'template target: ' + target );
189	191
190	192	// now normalize the target before template processing
191	193	target = this.manager.env.normalizeTitle( target );
—	—	@@ -348,10 +350,11 @@
349	351	// Start a new request if none is outstanding
350	352	this.manager.env.dp( 'requestQueue: ', this.manager.env.requestQueue);
351	353	if ( this.manager.env.requestQueue[title] === undefined ) {
	354	+ this.manager.env.tp( 'Note: Starting new request for ' + title );
352	355	this.manager.env.requestQueue[title] = new TemplateRequest( this.manager, title );
353	356	}
354	357	// Append a listener to the request
355		~~- this.manager.env.requestQueue[title].addListener( 'src', callback );~~
	358	+ this.manager.env.requestQueue[title].once( 'src', callback );
356	359
357	360	}
358	361	};
—	—	@@ -368,7 +371,7 @@
369	372
370	373	token.resultTokens = false;
371	374
372		~~- new AttributeTransformManager(~~
	375	+ new AttributeTransformManager (
373	376	this.manager,
374	377	this._returnArgAttributes.bind( this, token, cb, frame )
375	378	).process( attributes );
—	—	@@ -471,9 +474,8 @@
472	475	src = '';
473	476	}
474	477	//console.log( 'Page ' + title + ': got ' + src );
475		~~- manager.env.dp( 'Success for ' + title + ' :' + body + ':' );~~
	478	+ manager.env.tp( 'Retrieved ' + title );
476	479	manager.env.pageCache[title] = src;
477		~~- manager.env.dp(data);~~
478	480	self.emit( 'src', src, title );
479	481	}
480	482	// XXX: handle other status codes
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
—	—	@@ -4,6 +4,7 @@
5	5	parserFunctions: {},
6	6	pageCache: {}, // @fixme use something with managed space
7	7	debug: false,
	8	+ trace: false,
8	9	wgScriptPath: "http://en.wikipedia.org/w",
9	10	wgScriptExtension: ".php",
10	11	fetchTemplates: false,
—	—	@@ -129,7 +130,7 @@
130	131	var tstring = JSON.stringify( token );
131	132	this.dp ( 'MWParserEnvironment.tokensToString, non-text token: ' +
132	133	tstring + JSON.stringify( tokens, null, 2 ) );
133		~~- out.push( tstring );~~
	134	+ //out.push( tstring );
134	135	}
135	136	}
136	137	//console.log( 'MWParserEnvironment.tokensToString result: ' + out.join('') );
—	—	@@ -150,6 +151,18 @@
151	152	}
152	153	};
153	154
	155	+/**
	156	+ * Simple debug helper, trace-only
	157	+ */
	158	+MWParserEnvironment.prototype.tp = function ( ) {
	159	+ if ( this.debug \|\| this.trace ) {
	160	+ if ( arguments.length > 1 ) {
	161	+ console.log( JSON.stringify( arguments, null, 2 ) );
	162	+ } else {
	163	+ console.log( arguments[0] );
	164	+ }
	165	+ }
	166	+};
154	167
155	168
156	169	if (typeof module == "object") {

Status & tagging log

16:50, 31 January 2012 GWicke (talk | contribs) changed the status of r110406 [removed: new added: deferred]