r108887 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r108886‎ \| r108887 \| r108888 >
Date:	00:58, 14 January 2012
Author:	gwicke
Status:	deferred
Tags:
Comment:	First template expansion tests start working, and a bug fix in DOMPostProcessor paragraph wrapper. 187 parser tests now passing.
Modified paths:	/trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMPostProcessor.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history) /trunk/extensions/VisualEditor/tests/parser/parserTests.js (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/tests/parser/parserTests.js
—	—	@@ -184,7 +184,8 @@
185	185
186	186	this.currentItem = undefined;
187	187
188		~~- return this;~~
	188	+ // Create a new parser environment
	189	+ this.env = new MWParserEnvironment({});
189	190	}
190	191
191	192
—	—	@@ -269,29 +270,17 @@
270	271	return undefined;
271	272	};
272	273
273		-
274		~~-ParserTests.prototype.normalizeTitle = function(name) {~~
275		~~- if (typeof name !== 'string') {~~
276		~~- throw new Error('nooooooooo not a string');~~
277		~~- }~~
278		~~- name = name.replace(/[\s_]+/g, '_');~~
279		~~- name = name.substr(0, 1).toUpperCase() + name.substr(1);~~
280		~~- if (name === '') {~~
281		~~- throw new Error('Invalid/empty title');~~
282		~~- }~~
283		~~- return name;~~
284		~~-};~~
285		-
286	274	ParserTests.prototype.fetchArticle = function(name) {
287	275	// very simple for now..
288		~~- var norm = normalizeTitle(name);~~
	276	+ var norm = this.env.normalizeTitle(name);
289	277	if (norm in this.articles) {
290	278	return this.articles[norm];
291	279	}
292	280	};
293	281
294	282	ParserTests.prototype.processArticle = function(item) {
295		~~- var norm = this.normalizeTitle(item.title);~~
	283	+ var norm = this.env.normalizeTitle(item.title);
	284	+ //console.log( 'processArticle ' + norm );
296	285	this.articles[norm] = item.text;
297	286	};
298	287
—	—	@@ -552,8 +541,8 @@
553	542	// }
554	543	//});
555	544
556		~~- var env = new MWParserEnvironment({});~~
557		~~- var parserPipeline = new ParserPipeline( env );~~
	545	+ this.env.pageCache = this.articles;
	546	+ var parserPipeline = new ParserPipeline( this.env );
558	547
559	548	var comments = [],
560	549	pt = this;
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
—	—	@@ -314,7 +314,7 @@
315	315	this._construct();
316	316	this._reset( args, env );
317	317	// FIXME: pass actual title?
318		~~- this.loopCheck = new LoopCheck ( null );~~
	318	+ this.loopCheck = new LoopCheck( null );
319	319	}
320	320
321	321	// Inherit from TokenTransformManager, and thus also from EventEmitter.
—	—	@@ -337,7 +337,10 @@
338	338	var child = pipe.last;
339	339	// We assume that the title was already checked against this.loopCheck
340	340	// before!
341		~~- child.loopCheck = new LoopCheck ( title, this.loopCheck );~~
	341	+ child.loopCheck = new LoopCheck (
	342	+ this.env.normalizeTitle( this.env.tokensToString ( title ) ),
	343	+ this.loopCheck
	344	+ );
342	345	// Same for depth!
343	346	child.depth = this.depth + 1;
344	347	return pipe;
—	—	@@ -926,6 +929,7 @@
927	930	LoopCheck.prototype.check = function ( title ) {
928	931	var elem = this;
929	932	do {
	933	+ //console.log( 'loop check: ' + title + ' vs ' + elem.title );
930	934	if ( elem.title === title ) {
931	935	// Loop detected
932	936	return true;
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
—	—	@@ -455,7 +455,7 @@
456	456
457	457	para
458	458	= s1:sol s2:sol c:inlineline {
459		~~- return s1.concat(s2, [{type: 'TAG', name: 'p'}], c);~~
	459	+ return s1.concat(s2, /* [{type: 'TAG', name: 'p'}],*/ c);
460	460	}
461	461
462	462	br = space* &newline { return {type: 'SELFCLOSINGTAG', name: 'br'} }
—	—	@@ -488,7 +488,7 @@
489	489	c = flatten(c);
490	490	for (var i = 0, l = c.length; i < l; i++) {
491	491	var ci = c[i];
492		~~- if (typeof ci == 'string') {~~
	492	+ if (typeof ci === 'string') {
493	493	if(ci !== '') {
494	494	text.push(ci);
495	495	}
—	—	@@ -686,13 +686,14 @@
687	687	}
688	688
689	689	template
690		~~- = "{{" target:template_target~~
	690	+ = "{{" target:template_param_text
691	691	params:(newline? "\|" newline? p:template_param { return p })*
692	692	"}}" {
	693	+ target = flatten( target );
693	694	var obj = {
694	695	type: 'SELFCLOSINGTAG',
695	696	name: 'template',
696		~~- attribs: [['data-target', target]],~~
	697	+ attribs: [['data-target', JSON.stringify(target)]],
697	698	orderedArgs: params,
698	699	args: {},
699	700	target: target
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMPostProcessor.js
—	—	@@ -46,7 +46,7 @@
47	47	var body = document.body,
48	48	newP = document.createElement('p'),
49	49	cnodes = body.childNodes,
50		~~- haveInlines = false,~~
	50	+ inParagraph = false,
51	51	deleted = 0;
52	52
53	53	function isElementContentWhitespace ( e ) {
—	—	@@ -57,22 +57,26 @@
58	58	var child = cnodes[i - deleted],
59	59	ctype = child.nodeType;
60	60	//console.log(child + ctype);
61		~~- if ((ctype === 3 && (haveInlines \|\| !isElementContentWhitespace(child))) \|\|~~
62		~~- (ctype !== Node.TEXT_NODE &&~~
63		~~- ctype !== Node.COMMENT_NODE &&~~
64		~~- !isBlock(child.nodeName))) {~~
65		~~- // text node~~
	61	+ if ((ctype === 3 && (inParagraph \|\| !isElementContentWhitespace( child ))) \|\|
	62	+ (ctype === Node.COMMENT_NODE && inParagraph ) \|\|
	63	+ (ctype !== Node.TEXT_NODE &&
	64	+ ctype !== Node.COMMENT_NODE &&
	65	+ !isBlock(child.nodeName))
	66	+ )
	67	+ {
	68	+ // wrap in paragraph
66	69	newP.appendChild(child);
67		~~- haveInlines = true;~~
	70	+ inParagraph = true;
68	71	deleted++;
69		~~- } else if (haveInlines) {~~
	72	+ } else if (inParagraph) {
70	73	body.insertBefore(newP, child);
	74	+ deleted--;
71	75	newP = document.createElement('p');
72		~~- haveInlines = false;~~
73		~~- }~~
	76	+ inParagraph = false;
	77	+ }
74	78	}
75	79
76		~~- if (haveInlines) {~~
	80	+ if (inParagraph) {
77	81	body.appendChild(newP);
78	82	}
79	83	};
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
—	—	@@ -156,6 +156,7 @@
157	157	tokenExpander.listenForTokensFrom ( tokenPreProcessor );
158	158	// XXX: hack.
159	159	tokenExpander.inputType = inputType;
	160	+ tokenPreProcessor.inputType = inputType;
160	161
161	162	return new CachedTokenPipeline(
162	163	this.cachePipeline.bind( this, 'text/wiki', 'input' ),
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
—	—	@@ -47,7 +47,7 @@
48	48	* processes the template.
49	49	*/
50	50	TemplateHandler.prototype.onTemplate = function ( token, cb ) {
51		~~- //console.log('onTemplate!');~~
	51	+ //console.log('onTemplate! ' + JSON.stringify( token, null, 2 ) );
52	52
53	53	this.parentCB = cb;
54	54
—	—	@@ -71,7 +71,8 @@
72	72	res,
73	73	kv;
74	74
75		~~- var attributes = [[{ type: 'TEXT', value: '' } , token.target ]].concat( token.orderedArgs );~~
	75	+ var attributes = [[[{ type: 'TEXT', value: '' }] , token.target ]]
	76	+ .concat( token.orderedArgs );
76	77
77	78	//console.log( 'before AttributeTransformManager: ' + JSON.stringify( attributes, null, 2 ) );
78	79	new AttributeTransformManager(
—	—	@@ -109,7 +110,20 @@
110	111	//console.log('TemplateHandler.expandTemplate: ' +
111	112	// JSON.stringify( templateTokenTransformData, null, 2 ) );
112	113	// First, check the target for loops
113		~~- this.manager.loopCheck.check( templateTokenTransformData.target );~~
	114	+ var target = this.manager.env.normalizeTitle(
	115	+ this.manager.env.tokensToString( templateTokenTransformData.target )
	116	+ );
	117	+ if( this.manager.loopCheck.check( target ) ) {
	118	+ // Loop detected, abort!
	119	+ return {
	120	+ tokens: [
	121	+ {
	122	+ type: 'TEXT',
	123	+ value: 'Template expansion loop detected!'
	124	+ }
	125	+ ]
	126	+ };
	127	+ }
114	128
115	129	// Create a new nested transformation pipeline for the input type
116	130	// (includes the tokenizer and synchronous stage-1 transforms for
—	—	@@ -117,8 +131,9 @@
118	132	// Returned pipe (for now):
119	133	// { first: tokenizer, last: AsyncTokenTransformManager }
120	134	this.inputPipeline = this.manager.newChildPipeline(
121		~~- this.manager.inputType,~~
122		~~- templateTokenTransformData.expandedArgs~~
	135	+ this.manager.inputType \|\| 'text/wiki',
	136	+ templateTokenTransformData.expandedArgs,
	137	+ templateTokenTransformData.target
123	138	);
124	139
125	140	// Hook up the inputPipeline output events to call back our parentCB.
—	—	@@ -128,7 +143,7 @@
129	144
130	145	// Resolve a possibly relative link
131	146	var templateName = this.manager.env.resolveTitle(
132		~~- this.manager.env.tokensToString( templateTokenTransformData.target ),~~
	147	+ target,
133	148	'Template'
134	149	);
135	150	this._fetchTemplateAndTitle( templateName, this._processTemplateAndTitle.bind( this ) );
—	—	@@ -176,7 +191,8 @@
177	192	TemplateHandler.prototype._onEnd = function( ) {
178	193	// Encapsulate the template in a single token, which contains all the
179	194	// information needed for the editor.
180		~~- var res =~~
	195	+ var res = this.resultTokens;
	196	+ /*
181	197	[{
182	198	type: 'TAG',
183	199	name: 'div',
—	—	@@ -188,6 +204,7 @@
189	205	this.resultTokens,
190	206	[{ type: 'ENDTAG', name: 'div' }]
191	207	);
	208	+ */
192	209	//console.log( 'TemplateHandler._onEnd: ' + JSON.stringify( res, null, 2 ) );
193	210
194	211	if ( this.isAsync ) {
—	—	@@ -219,7 +236,7 @@
220	237	// @fixme normalize name?
221	238	if (title in this.manager.env.pageCache) {
222	239	// @fixme should this be forced to run on next event?
223		~~- callback( this.env.manager.pageCache[title], title );~~
	240	+ callback( this.manager.env.pageCache[title], title );
224	241	} else {
225	242	// whee fun hack!
226	243	//console.log(title);
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
—	—	@@ -58,6 +58,19 @@
59	59	}
60	60	};
61	61
	62	+MWParserEnvironment.prototype.normalizeTitle = function( name ) {
	63	+ if (typeof name !== 'string') {
	64	+ throw new Error('nooooooooo not a string');
	65	+ }
	66	+ name = name.replace(/[\s_]+/g, '_');
	67	+ function upperFirst( s ) { return s.substr(0, 1).toUpperCase() + s.substr(1); }
	68	+ name = name.split(':').map( upperFirst ).join(':');
	69	+ //if (name === '') {
	70	+ // throw new Error('Invalid/empty title');
	71	+ //}
	72	+ return name;
	73	+};
	74	+
62	75	/**
63	76	* @fixme do this for real eh
64	77	*/
—	—	@@ -65,28 +78,30 @@
66	79	// hack!
67	80	if (name.indexOf(':') == -1 && typeof namespace ) {
68	81	// hack hack hack
69		~~- name = namespace + ':' + name;~~
	82	+ name = namespace + ':' + this.normalizeTitle( name );
70	83	}
71	84	return name;
72	85	};
73	86
74	87	MWParserEnvironment.prototype.tokensToString = function ( tokens ) {
75	88	var out = [];
	89	+ //console.log( 'MWParserEnvironment.tokensToString, tokens: ' + JSON.stringify( tokens ) );
76	90	// XXX: quick hack, track down non-array sources later!
77	91	if ( ! $.isArray( tokens ) ) {
78	92	tokens = [ tokens ];
79	93	}
80	94	for ( var i = 0, l = tokens.length; i < l; i++ ) {
81		~~- console.log( 'MWParserEnvironment.tokensToString: ' + token );~~
82	95	var token = tokens[i];
	96	+ //console.log( 'MWParserEnvironment.tokensToString, token: ' + JSON.stringify( token ) );
83	97	if ( token.type === 'TEXT' ) {
84	98	out.push( token.value );
85	99	} else {
86	100	var tstring = JSON.stringify( token );
87		~~- console.log ( 'MWParserEnvironment::tokensToString: ' + tstring );~~
88		~~- out.push( tstring );~~
	101	+ //console.log ( 'MWParserEnvironment.tokensToString, non-text token: ' + tstring );
	102	+ //out.push( tstring );
89	103	}
90	104	}
	105	+ //console.log( 'MWParserEnvironment.tokensToString result: ' + out.join('') );
91	106	return out.join('');
92	107	};
93	108

Status & tagging log

00:59, 14 January 2012 GWicke (talk | contribs) changed the status of r108887 [removed: new added: deferred]