r110495 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r110494‎ \| r110495 \| r110496 >
Date:	16:30, 1 February 2012
Author:	gwicke
Status:	deferred
Tags:
Comment:	Change token format to plain strings for text tokens, and specific objects for other tokens. This is only the first half of the conversion. The next step is to drop the type attribute on most tokens and match on the constructor in the token transform machinery.
Modified paths:	/trunk/extensions/VisualEditor/modules/parser/ext.Cite.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.QuoteTransformer.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.Sanitizer.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.tokenizer.peg.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/ext.core.QuoteTransformer.js
—	—	@@ -88,7 +88,7 @@
89	89	this.bolds.push(this.currentChunk);
90	90	break;
91	91	case 4:
92		~~- this.currentChunk.push( {type: 'TEXT', value: "'"} );~~
	92	+ this.currentChunk.push( "'" );
93	93	this._startNewChunk();
94	94	this.currentChunk.push(ctx);
95	95	this.bolds.push(this.currentChunk);
—	—	@@ -108,7 +108,7 @@
109	109	break;
110	110	default: // longer than 5, only use the last 5 ticks
111	111	var newvalue = token.value.substr(0, qlen - 5 );
112		~~- this.currentChunk.push ( {type: 'TEXT', value: newvalue} );~~
	112	+ this.currentChunk.push ( newvalue );
113	113	this._startNewChunk();
114	114	this.currentChunk.push(ctx);
115	115	this.italics.push(this.currentChunk);
—	—	@@ -153,9 +153,9 @@
154	154	var ctx = this.bolds[j][0];
155	155	//console.log("balancing!" + JSON.stringify(ctx.prevToken, null, 2));
156	156	if (ctx.prevToken) {
157		~~- if (ctx.prevToken.type === 'TEXT') {~~
158		~~- var lastchar = prevToken.value[ctx.prevToken.value.length - 1],~~
159		~~- secondtolastchar = ctx.prevToken.value[ctx.prevToken.value.length - 2];~~
	157	+ if (ctx.prevToken.constructor === String) {
	158	+ var lastchar = prevToken[ctx.prevToken.length - 1],
	159	+ secondtolastchar = ctx.prevToken[ctx.prevToken.length - 2];
160	160	if (lastchar === ' ' && firstspace === -1) {
161	161	firstspace = j;
162	162	} else if (lastchar !== ' ') {
—	—	@@ -215,7 +215,7 @@
216	216	// italic tags. In the process, one quote needs to be converted back to text.
217	217	QuoteTransformer.prototype.convertBold = function ( i ) {
218	218	var chunk = this.bolds[i],
219		~~- textToken = { type: 'TEXT', value: "'" };~~
	219	+ textToken = "'";
220	220	//console.log('convertbold!');
221	221	if ( chunk.pos ) {
222	222	this.chunks[chunk.pos - 1].push( textToken );
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js
—	—	@@ -61,7 +61,7 @@
62	62	if ( maybeAttribs && $.isArray( maybeAttribs ) ) {
63	63	for(var i = 0, length = maybeAttribs.length; i < length; i++) {
64	64	var att = maybeAttribs[i];
65		~~- atts.push({nodeName: att[0], nodeValue: att[1]});~~
	65	+ atts.push({nodeName: att.k, nodeValue: att.v});
66	66	}
67	67	}
68	68	return atts;
—	—	@@ -71,54 +71,55 @@
72	72	// html tree builder by emitting the token.
73	73	FauxHTML5.TreeBuilder.prototype.processToken = function (token) {
74	74
75		~~- switch (token.type) {~~
76		~~- case "TEXT":~~
77		~~- this.emit('token', {type: 'Characters', data: token.value});~~
78		~~- break;~~
79		~~- case "TAG":~~
80		~~- this.emit('token', {type: 'StartTag',~~
81		~~- name: token.name,~~
82		~~- data: this._att(token.attribs)});~~
83		~~- break;~~
84		~~- case "ENDTAG":~~
85		~~- this.emit('token', {type: 'EndTag',~~
86		~~- name: token.name,~~
87		~~- data: this._att(token.attribs)});~~
88		~~- break;~~
89		~~- case "SELFCLOSINGTAG":~~
90		~~- this.emit('token', {type: 'StartTag',~~
91		~~- name: token.name,~~
92		~~- data: this._att(token.attribs)});~~
93		~~- if ( HTML5.VOID_ELEMENTS.indexOf( token.name.toLowerCase() ) < 0 ) {~~
94		~~- // VOID_ELEMENTS are automagically treated as self-closing by~~
95		~~- // the tree builder~~
	75	+ if ( token.constructor === String ) {
	76	+ this.emit('token', {type: 'Characters', data: token});
	77	+ } else {
	78	+ switch (token.type) {
	79	+ case "TAG":
	80	+ this.emit('token', {type: 'StartTag',
	81	+ name: token.name,
	82	+ data: this._att(token.attribs)});
	83	+ break;
	84	+ case "ENDTAG":
96	85	this.emit('token', {type: 'EndTag',
97	86	name: token.name,
98	87	data: this._att(token.attribs)});
99		~~- }~~
100		~~- break;~~
101		~~- case "COMMENT":~~
102		~~- this.emit('token', {type: 'Comment',~~
103		~~- data: token.value});~~
104		~~- break;~~
105		~~- case "END":~~
106		~~- this.emit('end');~~
107		~~- this.emit('token', { type: 'EOF' } );~~
108		~~- this.document = this.parser.document;~~
109		~~- if ( ! this.document.body ) {~~
110		~~- // HACK: This should not be needed really.~~
111		~~- this.document.body = this.parser.document.getElementsByTagName('body')[0];~~
112		~~- }~~
113		~~- // Emit the document to consumers~~
114		~~- //this.emit('document', this.document);~~
115		~~- break;~~
116		~~- case "NEWLINE":~~
117		~~- //this.emit('end');~~
118		~~- //this.emit('token', {type: 'Characters', data: "\n"});~~
119		~~- break;~~
120		~~- default:~~
121		~~- console.log("Unhandled token: " + JSON.stringify(token));~~
122		~~- break;~~
	88	+ break;
	89	+ case "SELFCLOSINGTAG":
	90	+ this.emit('token', {type: 'StartTag',
	91	+ name: token.name,
	92	+ data: this._att(token.attribs)});
	93	+ if ( HTML5.VOID_ELEMENTS.indexOf( token.name.toLowerCase() ) < 0 ) {
	94	+ // VOID_ELEMENTS are automagically treated as self-closing by
	95	+ // the tree builder
	96	+ this.emit('token', {type: 'EndTag',
	97	+ name: token.name,
	98	+ data: this._att(token.attribs)});
	99	+ }
	100	+ break;
	101	+ case "COMMENT":
	102	+ this.emit('token', {type: 'Comment',
	103	+ data: token.value});
	104	+ break;
	105	+ case "END":
	106	+ this.emit('end');
	107	+ this.emit('token', { type: 'EOF' } );
	108	+ this.document = this.parser.document;
	109	+ if ( ! this.document.body ) {
	110	+ // HACK: This should not be needed really.
	111	+ this.document.body = this.parser.document.getElementsByTagName('body')[0];
	112	+ }
	113	+ // Emit the document to consumers
	114	+ //this.emit('document', this.document);
	115	+ break;
	116	+ case "NEWLINE":
	117	+ //this.emit('end');
	118	+ //this.emit('token', {type: 'Characters', data: "\n"});
	119	+ break;
	120	+ default:
	121	+ console.log("Unhandled token: " + JSON.stringify(token));
	122	+ break;
	123	+ }
123	124	}
124	125	};
125	126
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
—	—	@@ -151,9 +151,16 @@
152	152	* fully processed). The token type change case still needs to be covered
153	153	* though.
154	154	*/
	155	+TokenTransformManager.prototype._setTokenRank = function ( token ) {
	156	+};
155	157	TokenTransformManager.prototype._resetTokenRank = function ( res, transformer ) {
156	158	if ( res.token ) {
157	159	// reset rank after type or name change
	160	+
	161	+ // Convert String literal to String object
	162	+ if ( res.token.constructor === String && res.token.rank === undefined ) {
	163	+ res.token = new String( res.token );
	164	+ }
158	165	if ( transformer.rank < 1 ) {
159	166	res.token.rank = 0;
160	167	} else {
—	—	@@ -161,6 +168,11 @@
162	169	}
163	170	} else if ( res.tokens && transformer.rank > 2 ) {
164	171	for ( var i = 0; i < res.tokens.length; i++ ) {
	172	+ var token = res.tokens[i];
	173	+ // convert string literal to string object
	174	+ if ( token.constructor === String && token.rank === undefined ) {
	175	+ res.tokens[i] = new String( token );
	176	+ }
165	177	if ( res.tokens[i].rank === undefined ) {
166	178	// Do not run phase 0 on newly created tokens from
167	179	// phase 1.
—	—	@@ -229,10 +241,16 @@
230	242	break;
231	243	}
232	244	// track progress on token
	245	+ if ( res.token.rank === undefined && res.token.constructor === String ) {
	246	+ res.token = new String ( res.token );
	247	+ }
233	248	res.token.rank = transformer.rank;
234	249	}
235	250	if ( ! aborted ) {
236	251	// Mark token as fully processed.
	252	+ if ( res.token.rank === undefined && res.token.constructor === String ) {
	253	+ res.token = new String ( res.token );
	254	+ }
237	255	res.token.rank = phaseEndRank;
238	256	}
239	257	}
—	—	@@ -279,10 +297,16 @@
280	298	aborted = true;
281	299	break;
282	300	}
	301	+ if ( res.token.rank === undefined && res.token.constructor === String ) {
	302	+ res.token = new String ( res.token );
	303	+ }
283	304	res.token.rank = transformer.rank;
284	305	}
285	306	if ( ! aborted ) {
286	307	// mark token as completely processed
	308	+ if ( res.token.rank === undefined && res.token.constructor === String ) {
	309	+ res.token = new String ( res.token );
	310	+ }
287	311	res.token.rank = phaseEndRank; // need phase passed in!
288	312	}
289	313
—	—	@@ -418,6 +442,8 @@
419	443	AsyncTokenTransformManager.prototype.onChunk = function ( tokens ) {
420	444	// Set top-level callback to next transform phase
421	445	var res = this.transformTokens ( tokens, this.tokenCB );
	446	+ this.env.dp('AsyncTokenTransformManager onChunk res.async=' + res.async +
	447	+ ' tokens=' + JSON.stringify( tokens ) );
422	448
423	449	if ( ! this.tailAccumulator ) {
424	450	this.emit( 'chunk', res.tokens );
—	—	@@ -429,7 +455,6 @@
430	456	this.tailAccumulator = res.async;
431	457	this.tokenCB = res.async.getParentCB ( 'sibling' );
432	458	}
433		~~- this.env.dp('AsyncTokenTransformManager onChunk res.async=' + res.async);~~
434	459	//this.phase2TailCB( tokens, true );
435	460
436	461	// The next processed chunk should call back as a sibling to last
—	—	@@ -462,27 +487,29 @@
463	488	for ( var i = 0; i < tokensLength; i++ ) {
464	489	token = tokens[i];
465	490
466		~~- switch( token.type ) {~~
467		~~- case 'TAG':~~
468		~~- case 'ENDTAG':~~
469		~~- case 'SELFCLOSINGTAG':~~
470		~~- res = this._transformTagToken( token, cb, phaseEndRank );~~
471		~~- break;~~
472		~~- case 'TEXT':~~
473		~~- res = this._transformToken( token, cb, phaseEndRank, ts.text );~~
474		~~- break;~~
475		~~- case 'COMMENT':~~
476		~~- res = this._transformToken( token, cb, phaseEndRank, ts.comment);~~
477		~~- break;~~
478		~~- case 'NEWLINE':~~
479		~~- res = this._transformToken( token, cb, phaseEndRank, ts.newline );~~
480		~~- break;~~
481		~~- case 'END':~~
482		~~- res = this._transformToken( token, cb, phaseEndRank, ts.end );~~
483		~~- break;~~
484		~~- default:~~
485		~~- res = this._transformToken( token, cb, phaseEndRank, ts.martian );~~
486		~~- break;~~
	491	+ if ( token.constructor === String ) {
	492	+ res = this._transformToken( token, cb, phaseEndRank, ts.text );
	493	+ //console.log( 'transform string ' + token + ' res:' + JSON.stringify( res ) );
	494	+ } else {
	495	+ switch( token.type ) {
	496	+ case 'TAG':
	497	+ case 'ENDTAG':
	498	+ case 'SELFCLOSINGTAG':
	499	+ res = this._transformTagToken( token, cb, phaseEndRank );
	500	+ break;
	501	+ case 'COMMENT':
	502	+ res = this._transformToken( token, cb, phaseEndRank, ts.comment);
	503	+ break;
	504	+ case 'NEWLINE':
	505	+ res = this._transformToken( token, cb, phaseEndRank, ts.newline );
	506	+ break;
	507	+ case 'END':
	508	+ res = this._transformToken( token, cb, phaseEndRank, ts.end );
	509	+ break;
	510	+ default:
	511	+ res = this._transformToken( token, cb, phaseEndRank, ts.martian );
	512	+ break;
	513	+ }
487	514	}
488	515
489	516	if( res.tokens ) {
—	—	@@ -638,29 +665,31 @@
639	666
640	667	for ( var i = 0; i < tokensLength; i++ ) {
641	668	token = tokens[i];
	669	+
	670	+ if ( token.constructor === String ) {
	671	+ res = this._transformToken( token, cb, this.phaseEndRank,
	672	+ ts.text );
	673	+ } else {
642	674
643		~~- switch( token.type ) {~~
644		~~- case 'TAG':~~
645		~~- case 'ENDTAG':~~
646		~~- case 'SELFCLOSINGTAG':~~
647		~~- res = this._transformTagToken( token, cb, this.phaseEndRank );~~
648		~~- break;~~
649		~~- case 'TEXT':~~
650		~~- res = this._transformToken( token, cb, this.phaseEndRank,~~
651		~~- ts.text );~~
652		~~- break;~~
653		~~- case 'COMMENT':~~
654		~~- res = this._transformToken( token, cb, this.phaseEndRank, ts.comment );~~
655		~~- break;~~
656		~~- case 'NEWLINE':~~
657		~~- res = this._transformToken( token, cb, this.phaseEndRank, ts.newline );~~
658		~~- break;~~
659		~~- case 'END':~~
660		~~- res = this._transformToken( token, cb, this.phaseEndRank, ts.end );~~
661		~~- break;~~
662		~~- default:~~
663		~~- res = this._transformToken( token, cb, this.phaseEndRank, ts.martian );~~
664		~~- break;~~
	675	+ switch( token.type ) {
	676	+ case 'TAG':
	677	+ case 'ENDTAG':
	678	+ case 'SELFCLOSINGTAG':
	679	+ res = this._transformTagToken( token, cb, this.phaseEndRank );
	680	+ break;
	681	+ case 'COMMENT':
	682	+ res = this._transformToken( token, cb, this.phaseEndRank, ts.comment );
	683	+ break;
	684	+ case 'NEWLINE':
	685	+ res = this._transformToken( token, cb, this.phaseEndRank, ts.newline );
	686	+ break;
	687	+ case 'END':
	688	+ res = this._transformToken( token, cb, this.phaseEndRank, ts.end );
	689	+ break;
	690	+ default:
	691	+ res = this._transformToken( token, cb, this.phaseEndRank, ts.martian );
	692	+ break;
	693	+ }
665	694	}
666	695
667	696	if( res.tokens ) {
—	—	@@ -744,7 +773,7 @@
745	774	pipe.addListener( 'end',
746	775	this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) )
747	776	);
748		~~- pipe.process( attributes[i][0].concat([{type:'END'}]) );~~
	777	+ pipe.process( attributes[i].k.concat([{type:'END'}]) );
749	778
750	779	// transform the value
751	780	pipe = this.manager.getAttributePipeline( this.manager.args );
—	—	@@ -754,11 +783,11 @@
755	784	pipe.addListener( 'end',
756	785	this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
757	786	);
758		~~- //console.log('starting attribute transform of ' + JSON.stringify( attributes[i][1] ) );~~
759		~~- pipe.process( attributes[i][1].concat([{type:'END'}]) );~~
	787	+ //console.log('starting attribute transform of ' + JSON.stringify( attributes[i].v ) );
	788	+ pipe.process( attributes[i].v.concat([{type:'END'}]) );
760	789	}
761	790	this.outstanding--;
762		~~- if ( this.outstanding == 0 ) {~~
	791	+ if ( this.outstanding === 0 ) {
763	792	this._returnAttributes();
764	793	}
765	794	};
—	—	@@ -768,7 +797,7 @@
769	798	var out = [];
770	799	for ( var i = 0, l = this.kvs.length; i < l; i++ ) {
771	800	var kv = this.kvs[i];
772		~~- out.push( [kv.key, kv.value] );~~
	801	+ out.push( new KV(kv.key, kv.value) );
773	802	}
774	803
775	804	// and call the callback with the result
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
—	—	@@ -56,19 +56,16 @@
57	57	};
58	58
59	59	ParserFunctions.prototype['pf_lc'] = function ( target, argList, argDict ) {
60		~~- return [{type: 'TEXT', value: target.toLowerCase()}];~~
	60	+ return [ target.toLowerCase() ];
61	61	};
62	62
63	63	ParserFunctions.prototype['pf_uc'] = function ( target, argList, argDict ) {
64		~~- return [{type: 'TEXT', value: target.toUpperCase()}];~~
	64	+ return [ target.toUpperCase() ];
65	65	};
66	66
67	67	ParserFunctions.prototype['pf_ucfirst'] = function ( target, argList, argDict ) {
68	68	if ( target ) {
69		~~- return [{~~
70		~~- type: 'TEXT',~~
71		~~- value: target[0].toUpperCase() + target.substr(1)~~
72		~~- }];~~
	69	+ return [ target[0].toUpperCase() + target.substr(1) ];
73	70	} else {
74	71	return [];
75	72	}
—	—	@@ -76,10 +73,7 @@
77	74
78	75	ParserFunctions.prototype['pf_lcfirst'] = function ( target, argList, argDict ) {
79	76	if ( target ) {
80		~~- return [{~~
81		~~- type: 'TEXT',~~
82		~~- value: target[0].toLowerCase() + target.substr(1)~~
83		~~- }];~~
	77	+ return [ target[0].toLowerCase() + target.substr(1) ];
84	78	} else {
85	79	return [];
86	80	}
—	—	@@ -93,20 +87,19 @@
94	88	// Based on http://jacwright.com/projects/javascript/date_format/ for now, MIT
95	89	// licensed.
96	90	ParserFunctions.prototype['pf_#time'] = function ( target, argList, argDict ) {
97		~~- //return [{type: 'TEXT', value: 'January 22, 2012'}];~~
98	91	var res,
99	92	tpl = target.trim();
100	93	//try {
101	94	// var date = new Date( this.manager.env.tokensToString( argList[0][1] ) );
102		~~- // res = [{type: 'TEXT', value: date.format( target ) }];~~
	95	+ // res = [ date.format( target ) ];
103	96	//} catch ( e ) {
104	97	// this.manager.env.dp( 'ERROR: #time ' + e );
105	98
106	99	try {
107		~~- res = [{type: 'TEXT', value: new Date().format ( tpl ) }];~~
	100	+ res = [ new Date().format ( tpl ) ];
108	101	} catch ( e2 ) {
109	102	this.manager.env.dp( 'ERROR: #time ' + e2 );
110		~~- res = [{type: 'TEXT', value: new Date().toString() }];~~
	103	+ res = [ new Date().toString() ];
111	104	}
112	105	//}
113	106	return res;
—	—	@@ -130,6 +123,7 @@
131	124	return returnStr;
132	125	};
133	126
	127	+// XXX: support localization
134	128	Date.replaceChars = {
135	129	shortMonths: ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'],
136	130	longMonths: ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'],
—	—	@@ -190,7 +184,7 @@
191	185	var f = new Function ( 'return (' + target + ')' );
192	186	res = f();
193	187	} catch ( e ) {
194		~~- return [{type: 'TEXT', value: 'class="error" in expression ' + target }];~~
	188	+ return [ 'class="error" in expression ' + target ];
195	189	}
196	190	if ( res ) {
197	191	return ( argList[0] && argList[0][1] ) \|\| [];
—	—	@@ -211,9 +205,9 @@
212	206	var f = new Function ( 'return (' + target + ')' );
213	207	res = f();
214	208	} catch ( e ) {
215		~~- return [{type: 'TEXT', value: 'class="error" in expression ' + target }];~~
	209	+ return [ 'class="error" in expression ' + target ];
216	210	}
217		~~- return [{type: 'TEXT', value: res.toString()}];~~
	211	+ return [ res.toString() ];
218	212	};
219	213
220	214
—	—	@@ -227,51 +221,51 @@
228	222	return ( argList[0] && argList[0][1] ) \|\| [];
229	223	};
230	224	ParserFunctions.prototype['pf_formatnum'] = function ( target, argList, argDict ) {
231		~~- return [{type: 'TEXT', value: target}];~~
	225	+ return [ target ];
232	226	};
233	227	ParserFunctions.prototype['pf_currentpage'] = function ( target, argList, argDict ) {
234		~~- return [{type: 'TEXT', value: target}];~~
	228	+ return [ target ];
235	229	};
236	230	ParserFunctions.prototype['pf_pagename'] = function ( target, argList, argDict ) {
237		~~- return [{type: 'TEXT', value: target}];~~
	231	+ return [ target ];
238	232	};
239	233	ParserFunctions.prototype['pf_pagesize'] = function ( target, argList, argDict ) {
240		~~- return [{type: 'TEXT', value: '100'}];~~
	234	+ return [ '100' ];
241	235	};
242	236	ParserFunctions.prototype['pf_pagename'] = function ( target, argList, argDict ) {
243		~~- return [{type: 'TEXT', value: target}];~~
	237	+ return [ target ];
244	238	};
245	239	ParserFunctions.prototype['pf_fullpagename'] = function ( target, argList, argDict ) {
246		~~- return [{type: 'TEXT', value: target}];~~
	240	+ return [target];
247	241	};
248	242	ParserFunctions.prototype['pf_fullpagenamee'] = function ( target, argList, argDict ) {
249		~~- return [{type: 'TEXT', value: target}];~~
	243	+ return [target];
250	244	};
251	245	ParserFunctions.prototype['pf_fullurl'] = function ( target, argList, argDict ) {
252		~~- return [{type: 'TEXT', value: target}];~~
	246	+ return [target];
253	247	};
254	248	ParserFunctions.prototype['pf_urlencode'] = function ( target, argList, argDict ) {
255	249	this.manager.env.tp( 'urlencode: ' + target );
256		~~- return [{type: 'TEXT', value: target.trim()}];~~
	250	+ return [target.trim()];
257	251	};
258	252	ParserFunctions.prototype['pf_anchorencode'] = function ( target, argList, argDict ) {
259		~~- return [{type: 'TEXT', value: target}];~~
	253	+ return [target];
260	254	};
261	255	ParserFunctions.prototype['pf_namespace'] = function ( target, argList, argDict ) {
262		~~- return [{type: 'TEXT', value: 'Main'}];~~
	256	+ return ['Main'];
263	257	};
264	258	ParserFunctions.prototype['pf_protectionlevel'] = function ( target, argList, argDict ) {
265		~~- return [{type: 'TEXT', value: ''}];~~
	259	+ return [''];
266	260	};
267	261	ParserFunctions.prototype['pf_ns'] = function ( target, argList, argDict ) {
268		~~- return [{type: 'TEXT', value: target}];~~
	262	+ return [target];
269	263	};
270	264
271	265	ParserFunctions.prototype['pf_subjectspace'] = function ( target, argList, argDict ) {
272		~~- return [{type: 'TEXT', value: 'Main'}];~~
	266	+ return ['Main'];
273	267	};
274	268	ParserFunctions.prototype['pf_talkspace'] = function ( target, argList, argDict ) {
275		~~- return [{type: 'TEXT', value: 'Talk'}];~~
	269	+ return ['Talk'];
276	270	};
277	271
278	272	if (typeof module == "object") {
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
—	—	@@ -75,10 +75,10 @@
76	76	};
77	77
78	78	var pushList = function ( listName, itemName ) {
79		~~- out.push({type: 'TAG', name: listName});~~
80		~~- out.push({type: 'TAG', name: itemName});~~
81		~~- endtags.push({type: 'ENDTAG', name: listName});~~
82		~~- endtags.push({type: 'ENDTAG', name: itemName});~~
	79	+ out.push( new TagTk( listName ));
	80	+ out.push( new TagTk( itemName ));
	81	+ endtags.push( new EndTagTk( listName ));
	82	+ endtags.push( new EndTagTk( itemName ));
83	83	};
84	84
85	85	var popTags = function ( n ) {
—	—	@@ -103,8 +103,8 @@
104	104	if (changeLen === 0) {
105	105	var itemToken = endtags.pop();
106	106	out.push(itemToken);
107		~~- out.push({type: 'TAG', name: itemToken.name});~~
108		~~- endtags.push({type: 'ENDTAG', name: itemToken.name});~~
	107	+ out.push(new TagTk( itemToken.name ));
	108	+ endtags.push(new EndTagTk( itemToken.name ));
109	109	} else if ( bs.length == bn.length
110	110	&& changeLen == 1
111	111	&& isDlDd( bs[prefixLen], bn[prefixLen] ) ) {
—	—	@@ -115,16 +115,16 @@
116	116	} else {
117	117	var newName = 'dd';
118	118	}
119		~~- out.push({type: 'TAG', name: newName});~~
120		~~- endtags.push({type: 'ENDTAG', name: newName});~~
	119	+ out.push(new TagTk( newName ));
	120	+ endtags.push(new EndTagTk( newName ));
121	121	} else {
122	122	popTags(bs.length - prefixLen);
123	123
124	124	if (prefixLen > 0 && bn.length == prefixLen ) {
125	125	var itemToken = endtags.pop();
126	126	out.push(itemToken);
127		~~- out.push({type: 'TAG', name: itemToken.name});~~
128		~~- endtags.push({type: 'ENDTAG', name: itemToken.name});~~
	127	+ out.push(new TagTk( itemToken.name ));
	128	+ endtags.push(new EndTagTk( itemToken.name ));
129	129	}
130	130
131	131	for(var i = prefixLen; i < bn.length; i++) {
—	—	@@ -264,7 +264,7 @@
265	265	// texts)
266	266	//console.log( pp( flatten ( e ) ) );
267	267	cache = {};
268		~~- __parseArgs[2]( [ { type: 'END' } ] );~~
	268	+ __parseArgs[2]( [ new EOFTk( ) ] );
269	269	return []; //flatten(e);
270	270	}
271	271
—	—	@@ -350,7 +350,7 @@
351	351	optionalSpaceToken
352	352	= s:space* {
353	353	if ( s.length ) {
354		~~- return [{type: 'TEXT', value: s.join('')}];~~
	354	+ return [s.join('')];
355	355	} else {
356	356	return [];
357	357	}
—	—	@@ -363,7 +363,7 @@
364	364	// was actually preceded by a newline
365	365	cn:( c:comment n:newline? {
366	366	if ( n !== '' ) {
367		~~- return [c, {type: 'TEXT', value: n}];~~
	367	+ return [c, n];
368	368	} else {
369	369	return [c];
370	370	}
—	—	@@ -376,9 +376,9 @@
377	377	var niToken = [];
378	378	if ( ni !== '') {
379	379	if ( ni[0] === '/' ) {
380		~~- niToken = [{type: 'ENDTAG', name: ni[1]}];~~
	380	+ niToken = [new EndTagTk( ni[1] )];
381	381	} else {
382		~~- niToken = [{type: 'TAG', name: ni[1]}];~~
	382	+ niToken = [new TagTk( ni[1] )];
383	383	}
384	384	}
385	385
—	—	@@ -391,20 +391,25 @@
392	392	newline
393	393	= '\n' / '\r\n'
394	394
395		~~-newlineToken = newline { return [{type: 'NEWLINE'}] }~~
	395	+newlineToken = newline { return [new NlTk()] }
396	396
397	397	eolf = newline / eof
398	398
399	399	toplevelblock
400	400	= & { blockStart = pos; return true; } b:block {
401	401	b = flatten(b);
402		~~- var bs = b[0];~~
403	402	if ( b.length ) {
	403	+ var bs = b[0];
	404	+ if ( bs.constructor === String && bs.attribs === undefined ) {
	405	+ b[0] = new String( bs );
	406	+ bs = b[0];
	407	+ }
404	408	//dp('toplevelblock:' + pp(b));
405	409	if (bs.attribs === undefined) {
406	410	bs.attribs = [];
407	411	}
408		~~- bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]);~~
	412	+ bs.attribs.push(new KV('data-sourcePos', blockStart + ':' + pos));
	413	+ //console.log( 'toplevelblock: ' + pp( bs ));
409	414	}
410	415	// XXX: only run this for lines that actually need it!
411	416	//b.push({type: 'NEWLINE'});
—	—	@@ -461,10 +466,10 @@
462	467
463	468	para
464	469	= s1:sol s2:sol c:inlineline {
465		~~- return s1.concat(s2, /* [{type: 'TAG', name: 'p'}],*/ c);~~
	470	+ return s1.concat(s2, /* [new TagTk('p')],*/ c);
466	471	}
467	472
468		~~-br = space* &newline { return {type: 'SELFCLOSINGTAG', name: 'br'} }~~
	473	+br = space* &newline { return new SelfclosingTagTk( 'br' ) }
469	474
470	475	// Syntax stops to limit inline expansion defending on syntactic context
471	476	inline_breaks
—	—	@@ -504,14 +509,14 @@
505	510	}
506	511	} else {
507	512	if (text.length) {
508		~~- out.push({ type: "TEXT", value: text.join('') });~~
	513	+ out.push( text.join('') );
509	514	text = [];
510	515	}
511	516	out.push(ci);
512	517	}
513	518	}
514	519	if (text.length) {
515		~~- out.push({ type: 'TEXT', value: text.join('') });~~
	520	+ out.push( text.join('') );
516	521	}
517	522	//dp('inline out:' + pp(out));
518	523	return out;
—	—	@@ -531,14 +536,14 @@
532	537	}
533	538	} else {
534	539	if (text.length) {
535		~~- out.push({type: 'TEXT', value: text.join('')});~~
	540	+ out.push( text.join('') );
536	541	text = [];
537	542	}
538	543	out.push(ci);
539	544	}
540	545	}
541	546	if (text.length) {
542		~~- out.push({type: 'TEXT', value: text.join('')});~~
	547	+ out.push( text.join('') );
543	548	}
544	549	//dp('inlineline out:' + pp(out));
545	550	return out;
—	—	@@ -550,7 +555,7 @@
551	556	/ & '{' ( & '{{{{{' template / tplarg / template )
552	557	/// & '{' ( tplarg / template )
553	558	// Eat three opening brackets as text.
554		~~- / '[[[' { return { type: 'TEXT', value: '[[[' } }~~
	559	+ / '[[[' { return '[[[' }
555	560	/ & '[' ( wikilink / extlink )
556	561	/ & "'" quote
557	562
—	—	@@ -563,7 +568,7 @@
564	569	& { return setFlag('h'); }
565	570	c:inlineline
566	571	e:'='+
567		- spc:(sp:space+ { return {type: 'TEXT', value: sp.join('') } } / comment)*
	572	+ spc:(sp:space+ { return sp.join('') } / comment)*
568	573	&eolf
569	574	{
570	575	clearFlag('h');
—	—	@@ -571,24 +576,24 @@
572	577	// convert surplus equals into text
573	578	if(s.length > level) {
574	579	var extras = s.substr(0, s.length - level);
575		~~- if(c[0].type == 'TEXT') {~~
576		~~- c[0].value = extras + c[0].value;~~
	580	+ if(c[0].constructor === String) {
	581	+ c[0] = extras + c[0];
577	582	} else {
578		~~- c.unshift({type: 'TEXT', value: extras});~~
	583	+ c.unshift( extras );
579	584	}
580	585	}
581	586	if(e.length > level) {
582	587	var extras = e.substr(0, e.length - level),
583	588	lastElem = c[c.length - 1];
584		~~- if(lastElem.type == 'TEXT') {~~
585		~~- lastElem.value = lastElem.value + extras;~~
	589	+ if(lastElem.constructor === String) {
	590	+ lastElem += extras;
586	591	} else {
587		~~- c.push({type: 'TEXT', value: extras});~~
	592	+ c.push( extras );
588	593	}
589	594	}
590	595
591		~~- return [{type: 'TAG', name: 'h' + level}]~~
592		~~- .concat(c, [{type: 'ENDTAG', name: 'h' + level}, spc]);~~
	596	+ return [new TagTk( 'h' + level )]
	597	+ .concat(c, [new EndTagTk( 'h' + level ), spc]);
593	598	}
594	599	/ & { dp('nomatch exit h'); clearFlag('h'); return false } { return null }
595	600	) { return r }
—	—	@@ -596,12 +601,12 @@
597	602
598	603	pre_indent
599	604	= l:pre_indent_line ls:(sol pre_indent_line)* {
600		~~- return [{type: 'TAG', name: 'pre'}]~~
	605	+ return [new TagTk( 'pre' )]
601	606	.concat( [l], ls
602		~~- , [{type: 'ENDTAG', name: 'pre'}]);~~
	607	+ , [new EndTagTk( 'pre' )]);
603	608	}
604	609	pre_indent_line = space l:inlineline {
605		~~- return [{type: 'TEXT', value: '\n'}].concat(l);~~
	610	+ return [ '\n' ].concat(l);
606	611	}
607	612
608	613
—	—	@@ -618,11 +623,9 @@
619	624
620	625	urllink
621	626	= target:url {
622		~~- return [ { type: 'TAG',~~
623		~~- name: 'a',~~
624		~~- attribs: [['href', target]] }~~
625		~~- , {type: 'TEXT', value: target}~~
626		~~- , {type: 'ENDTAG', name: 'a'}~~
	627	+ return [ new TagTk( 'a', [new KV('href', target)] )
	628	+ , target
	629	+ , new EndTagTk( 'a' )
627	630	];
628	631	}
629	632
—	—	@@ -636,20 +639,16 @@
637	640	clearFlag('extlink');
638	641	if ( text == '' ) {
639	642	// XXX: Link numbering should be implemented in post-processor.
640		~~- text = [{type: 'TEXT', value: "[" + linkCount + "]"}];~~
	643	+ text = [ "[" + linkCount + "]" ];
641	644	linkCount++;
642	645	}
643	646	return [
644		~~- {~~
645		~~- type: 'TAG',~~
646		~~- name: 'a',~~
647		~~- attribs: [~~
648		~~- ['href', target],~~
649		~~- ['data-type', 'external']~~
650		~~- ],~~
651		~~- }~~
	647	+ new TagTk( 'a', [
	648	+ new KV('href', target),
	649	+ new KV('data-type', 'external')
	650	+ ] ),
652	651	].concat( text
653		~~- , [{type: 'ENDTAG', name: 'a'}]);~~
	652	+ , [ new EndTagTk( 'a' )]);
654	653	}
655	654	/ "[" & { clearFlag('extlink'); return false; }
656	655
—	—	@@ -719,42 +718,16 @@
720	719	params:(newline? "\|" newline? p:template_param { return p })*
721	720	"}}" {
722	721	target = flatten( target );
723		~~- var obj = {~~
724		~~- type: 'SELFCLOSINGTAG',~~
725		~~- name: 'template',~~
726		~~- attribs: [], //[['data-target', JSON.stringify(target)]],~~
727		~~- orderedArgs: params,~~
728		~~- //args: {},~~
729		~~- target: target~~
730		~~- };~~
731		~~- // XXX: this is kind of broken, as arg keys need to be expanded~~
732		~~- //if (params && params.length) {~~
733		~~- // var position = 1;~~
734		~~- // for ( var i = 0, l = params.length; i < l; i++ ) {~~
735		~~- // var param = params[i];~~
736		~~- // if ( param[0] === null ) {~~
737		~~- // obj.args[position] = param[1];~~
738		~~- // position++;~~
739		~~- // } else {~~
740		~~- // // Last value wins for duplicates.~~
741		~~- // obj.args[param[0]] = param[1];~~
742		~~- // }~~
743		~~- // }~~
744		~~- // // HACK: temporarily also push the args into an attribute~~
745		~~- // // (just for debugging)~~
746		~~- // obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);~~
747		~~- //}~~
748		~~- // Should actually use a self-closing tag here, but the Node HTML5~~
749		~~- // parser only recognizes known self-closing tags for now, so use an~~
750		~~- // explicit end tag for now.~~
751		~~- //console.log(pp(obj));~~
	722	+ var obj = new SelfclosingTagTk( 'template' );
	723	+ obj.orderedArgs = params;
	724	+ obj.target = target;
752	725	//console.log( 'tokenizer template ' + JSON.stringify( target ));
753	726	return obj;
754	727	}
755	728
756	729	// XXX: support template and args in target!
757	730	//template_target
758		~~-// = h:( !"}}" x:([^\|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }~~
	731	+// = h:( !"}}" x:([^\|\n]) { return x } )* { return h.join('') }
759	732
760	733	tplarg
761	734	= "{{{"
—	—	@@ -762,20 +735,17 @@
763	736	params:( newline? "\|" newline? p:template_param { return p })*
764	737	"}}}" {
765	738	name = flatten( name );
766		~~- var obj = {~~
767		~~- type: 'SELFCLOSINGTAG',~~
768		~~- name: 'templatearg',~~
769		~~- attribs: [],~~
770		~~- argname: name,~~
771		~~- defaultvalue: []~~
772		~~- };~~
	739	+ var obj = new SelfclosingTagTk( 'templatearg', [] );
	740	+ obj.argname= name;
773	741	if (params && params.length) {
774	742	// HACK, not final.
775		~~- //obj.attribs.push(['data-defaultvalue', params[0][1]]);~~
	743	+ //obj.attribs.push(new KV('data-defaultvalue', params[0][1]));
776	744	//console.log(JSON.stringify(params, null, 2));
777		~~- //obj.attribs.push(['data-json-args', JSON.stringify(params)]);~~
778		~~- obj.defaultvalue = params[0][1].length ? params[0][1] :~~
779		~~- [{type: 'TEXT', value: ''}];~~
	745	+ //obj.attribs.push(new KV('data-json-args', JSON.stringify(params)));
	746	+ obj.defaultvalue = params[0].v.length ? params[0].v :
	747	+ [ '' ];
	748	+ } else {
	749	+ obj.defaultvalue = [];
780	750	}
781	751	//console.log( 'tokenizer tplarg ' + JSON.stringify( obj, null, 2 ));
782	752	return obj;
—	—	@@ -785,14 +755,14 @@
786	756	= name:template_param_name space* "=" space* c:template_param_text? {
787	757	//console.log( 'named template_param matched' + pp([name, flatten( c )]) );
788	758	if ( c !== '' ) {
789		~~- return [name, flatten( c )];~~
	759	+ return new KV(name, flatten( c ));
790	760	} else {
791		~~- return [name, []];~~
	761	+ return new KV(name, []);
792	762	}
793	763	} / c:template_param_text {
794		~~- return [[], flatten( c ) ];~~
	764	+ return new KV([], flatten( c ) );
795	765	}
796		~~- / & [\|}] { return [[], []]; }~~
	766	+ / & [\|}] { return new KV([], []); }
797	767
798	768
799	769	// FIXME: handle template args and templates in key! (or even parser functions?)
—	—	@@ -827,27 +797,24 @@
828	798	// class. Can we work out a static negative class instead?
829	799	// XXX: Exclude uppercase chars from non-latin languages too!
830	800	trail:(! [A-Z \t(),.:-] tc:text_char { return tc })* {
831		~~- var obj = {~~
832		~~- type: 'TAG',~~
833		~~- name: 'a',~~
834		~~- attribs: [~~
835		~~- ['data-type', 'internal']~~
836		~~- ]~~
837		~~- },~~
	801	+ var obj = new TagTk( 'a',
	802	+ [
	803	+ new KV('data-type', 'internal')
	804	+ ] ),
838	805	textTokens = [];
839		~~- obj.attribs.push(['href', target]);~~
	806	+ obj.attribs.push( new KV('href', target) );
840	807	if (lcontent && lcontent.length) {
841	808	textTokens = lcontent;
842	809	if (trail) {
843		~~- textTokens.push( { type: 'TEXT', value: trail.join('') } );~~
	810	+ textTokens.push( trail.join('') );
844	811	}
845	812	} else {
846	813	if (trail) {
847	814	target += trail.join('');
848	815	}
849		~~- textTokens = [{type: 'TEXT', value: target}];~~
	816	+ textTokens = [ target ];
850	817	}
851		~~- return [obj].concat(textTokens, [{type: 'ENDTAG', name: 'a'}]);~~
	818	+ return [obj].concat(textTokens, [new EndTagTk( 'a' )]);
852	819	}
853	820
854	821	link_target
—	—	@@ -872,18 +839,16 @@
873	840	link_end = "]]"
874	841
875	842	/* Generic quote production for italic and bold, further processed in a token
876		~~- * stream transformation in doQuotes. Relies on NEWLINE tokens being emitted~~
	843	+ * stream transformation in doQuotes. Relies on NlTk tokens being emitted
877	844	* for each line of text to balance quotes per line.
878	845	*
879	846	* We are not using a simple pair rule here as we need to support mis-nested
880	847	* bolds/italics and MediaWiki's special heuristics for apostrophes, which are
881	848	* all not context free. */
882	849	quote = "''" x:"'"* {
883		~~- return {~~
884		~~- type: 'TAG',~~
885		~~- name : 'mw-quote', // Will be consumed in token transforms~~
886		~~- value: "''" + x.join('')~~
887		~~- }~~
	850	+ var res = new TagTk( 'mw-quote' ); // Will be consumed in token transforms
	851	+ res.value = "''" + x.join('');
	852	+ return res;
888	853	}
889	854
890	855	/* XXX: Extension tags can require a change in the tokenizer mode, which
—	—	@@ -901,16 +866,16 @@
902	867	= "<pre"
903	868	attribs:generic_attribute*
904	869	">"
905		~~- ts:(t1:[^<]+ { return {type:'TEXT',value:t1.join('')} }~~
	870	+ ts:(t1:[^<]+ { return t1.join('') }
906	871	/ nowiki
907		~~- / !"</pre>" t2:. {return {type:'TEXT',value:t2}})+~~
	872	+ / !"</pre>" t2:. { return t2 })+
908	873	("</pre>" / eof) {
909	874	// return nowiki tags as well?
910	875	//console.log('inpre');
911		~~- return [ {type: 'TAG', name: 'pre', attribs: attribs} ]~~
912		~~- .concat(ts, [{type: 'ENDTAG', name: 'pre'}]);~~
	876	+ return [ new TagTk( 'pre', attribs ) ]
	877	+ .concat(ts, [ new EndTagTk( 'pre' ) ]);
913	878	}
914		~~- / "</pre>" { return {type: 'TEXT', value: "</pre>"}; }~~
	879	+ / "</pre>" { return "</pre>"; }
915	880
916	881	nowiki
917	882	= "<nowiki>" nc:nowiki_content "</nowiki>" {
—	—	@@ -919,9 +884,9 @@
920	885	}
921	886	/ "<nowiki>" {
922	887	//console.log('nowiki fallback');
923		~~- return [{type: 'TEXT', value: '<nowiki>'}];~~
	888	+ return ['<nowiki>'];
924	889	}
925		~~- / "</nowiki>" { return [{type: 'TEXT', value: '</nowiki>'}]; }~~
	890	+ / "</nowiki>" { return ['</nowiki>']; }
926	891
927	892	nowiki_content
928	893	= ts:( t:[^<]+ { return t.join('') }
—	—	@@ -932,7 +897,7 @@
933	898	/ (!("</nowiki>" / "</pre>") c:. {return c})
934	899	)* {
935	900	// return nowiki tags as well?
936		~~- return [{type: 'TEXT', value: ts.join('')}];~~
	901	+ return [ts.join('')];
937	902	}
938	903
939	904	// See http://dev.w3.org/html5/spec/Overview.html#syntax-tag-name and
—	—	@@ -946,13 +911,13 @@
947	912	// abort match if tag is not block-level
948	913	return null;
949	914	}
950		~~- var res = {name: name, attribs: attribs};~~
	915	+ var res;
951	916	if ( end != '' ) {
952		~~- res.type = 'ENDTAG';~~
	917	+ res = new EndTagTk( name, attribs );
953	918	} else if ( selfclose != '' ) {
954		~~- res.type = 'SELFCLOSINGTAG';~~
	919	+ res = new SelfclosingTagTk( name, attribs );
955	920	} else {
956		~~- res.type = 'TAG';~~
	921	+ res = new TagTk( name, attribs );
957	922	}
958	923	return [res];
959	924	}
—	—	@@ -974,15 +939,16 @@
975	940	space*
976	941	selfclose:"/"?
977	942	">" {
978		~~- var res = {name: name.join(''), attribs: attribs};~~
	943	+ name = name.join('');
	944	+ var res;
979	945	if ( end != '' ) {
980		~~- res.type = 'ENDTAG';~~
	946	+ res = new EndTagTk( name, attribs );
981	947	} else if ( selfclose != '' ) {
982		~~- res.type = 'SELFCLOSINGTAG';~~
	948	+ res = new SelfclosingTagTk( name, attribs );
983	949	} else {
984		~~- res.type = 'TAG';~~
	950	+ res = new TagTk( name, attribs );
985	951	}
986		~~- res.attribs.push(['data-sourceTagPos', (tagStartPos - 1) + ":" + pos]);~~
	952	+ res.attribs.push(new KV('data-sourceTagPos', (tagStartPos - 1) + ":" + pos));
987	953	return res;
988	954	}
989	955
—	—	@@ -993,9 +959,9 @@
994	960	v:generic_attribute_value { return v })?
995	961	{
996	962	if ( value !== '' ) {
997		~~- return [name, value];~~
	963	+ return new KV( name, value );
998	964	} else {
999		~~- return [name,''];~~
	965	+ return new KV( name, '' );
1000	966	}
1001	967	}
1002	968
—	—	@@ -1024,9 +990,9 @@
1025	991	/* Lists */
1026	992	lists = e:(dtdd / li) es:(sol (dtdd / li))*
1027	993	{
1028		~~- return annotateList( [ { type: 'TAG', name: 'list'} ]~~
	994	+ return annotateList( [ new TagTk( 'list' ) ]
1029	995	.concat(flatten([e].concat(es))
1030		~~- ,[{ type: 'ENDTAG', name: 'list' }]));~~
	996	+ ,[ new EndTagTk( 'list' ) ]));
1031	997	}
1032	998
1033	999	li = bullets:list_char+
—	—	@@ -1035,10 +1001,9 @@
1036	1002	{
1037	1003	if ( c == '' )
1038	1004	c = [];
1039		~~- return [ { type: 'TAG',~~
1040		~~- name: 'listItem',~~
1041		~~- bullets: bullets }~~
1042		~~- , c ];~~
	1005	+ var li = new TagTk( 'listItem' );
	1006	+ li.bullets = bullets;
	1007	+ return [ li, c ];
1043	1008	}
1044	1009
1045	1010	dtdd
—	—	@@ -1055,17 +1020,18 @@
1056	1021	// Convert trailing space into
1057	1022	// XXX: This should be moved to a serializer
1058	1023	//var clen = c.length;
1059		~~- //if (clen && c[clen - 1].type === 'TEXT') {~~
	1024	+ //if (clen && c[clen - 1].constructor === String) {
1060	1025	// var val = c[clen - 1].value;
1061	1026	// if(val.length && val[val.length - 1] == ' ') {
1062	1027	// c[clen - 1].value = val.substr(0, val.length - 1) + "\u00a0";
1063	1028	// }
1064	1029	//}
1065	1030
1066		~~- return [ { type: 'TAG', name: 'listItem', bullets: bullets + ";" } ]~~
1067		~~- .concat( c~~
1068		~~- ,[{ type: 'TAG', name: 'listItem', bullets: bullets + ":" } ]~~
1069		~~- , d );~~
	1031	+ var li = new TagTk( 'listItem' );
	1032	+ li.bullets = bullets + ";";
	1033	+ var li2 = new TagTk( 'listItem' );
	1034	+ li2.bullets = bullets + ":";
	1035	+ return [ li ].concat( c, [ li2 ], d );
1070	1036	}
1071	1037	// Fall-back case to clear the colon flag
1072	1038	/ & { return true; } { syntaxFlags['colon'] = 0; return null; }
—	—	@@ -1111,19 +1077,19 @@
1112	1078	space*
1113	1079	te:table_end_tag? // can occur somewhere in the middle of the line too
1114	1080	{
1115		~~- var tok = [{type: 'TAG', name: 'table'}];~~
	1081	+ var toks = [ new TagTk( 'table' ) ];
1116	1082	if ( ta )
1117		~~- tok[0].attribs = ta;~~
	1083	+ toks[0].attribs = ta;
1118	1084	if ( te )
1119		~~- tok = tok.concat(te);~~
1120		~~- return tok;~~
	1085	+ toks = toks.concat( te );
	1086	+ return toks;
1121	1087	}
1122	1088
1123	1089	table_caption_tag
1124	1090	= "\|+"
1125	1091	c:inline* {
1126		~~- return [{type: 'TAG', name: 'caption'}]~~
1127		~~- .concat( c, [{type: 'ENDTAG', name: 'caption'}]);~~
	1092	+ return [ new TagTk( 'caption' )]
	1093	+ .concat( c, [ new EndTagTk( 'caption' ) ]);
1128	1094	}
1129	1095
1130	1096
—	—	@@ -1138,7 +1104,7 @@
1139	1105	// We rely on our tree builder to close the row as needed. This is
1140	1106	// needed to support building tables from fragment templates with
1141	1107	// individual cells or rows.
1142		~~- var trToken = [{type: 'TAG', name: 'tr', attribs: a}];~~
	1108	+ var trToken = [ new TagTk( 'tr', a ) ];
1143	1109	if ( !td ) {
1144	1110	return trToken;
1145	1111	} else {
—	—	@@ -1167,8 +1133,8 @@
1168	1134	a = [];
1169	1135	}
1170	1136	//dp("table data result: " + pp(td) + ", attribts: " + pp(a));
1171		~~- return [{ type: 'TAG', name: 'td', attribs: a}]~~
1172		~~- .concat( td, [{type: 'ENDTAG', name: 'td'}] );~~
	1137	+ return [ new TagTk( 'td', a )]
	1138	+ .concat( td, [ new EndTagTk ( 'td' ) ] );
1173	1139	}
1174	1140
1175	1141	table_heading_tags
—	—	@@ -1184,13 +1150,13 @@
1185	1151	if ( a == '' ) {
1186	1152	a = [];
1187	1153	}
1188		~~- return [{type: 'TAG', name: 'th', attribs: a}]~~
1189		~~- .concat( c, [{type: 'ENDTAG', name: 'th'}] );~~
	1154	+ return [ new TagTk( 'th', a ) ]
	1155	+ .concat( c, [new EndTagTk( 'th' )] );
1190	1156	}
1191	1157
1192	1158	table_end_tag
1193	1159	= "\|}" {
1194		~~- var tok = [{type: 'ENDTAG', name: 'table'}];~~
	1160	+ var tok = [new EndTagTk( 'table' )];
1195	1161	return tok;
1196	1162	}
1197	1163
—	—	@@ -1209,25 +1175,24 @@
1210	1176	* when those work as intended! */
1211	1177	table
1212	1178	= tas:table_start space* c:table_caption? b:table_body? te:table_end {
1213		~~- var res = {type: 'TAG', name: 'table'}~~
	1179	+ var res = new TagTk( 'table' );
1214	1180	var body = b !== '' ? b : [];
1215	1181	dp("body: " + pp(body));
1216	1182	if (tas.length > 0) {
1217	1183	// FIXME: actually parse and build structure
1218		~~- //res.attribs = [['data-unparsed', tas.join('')]];~~
	1184	+ //res.attribs = [new KV('data-unparsed', tas.join(''))];
1219	1185	res.attribs = tas;
1220	1186	}
1221	1187
1222	1188	if (c != '') {
1223		~~- var caption = [{type: 'TAG', name: 'caption'}]~~
1224		~~- .concat(c, [{type: 'ENDTAG', name: 'caption'}], te);~~
	1189	+ var caption = [ new TagTk( 'caption' ) ]
	1190	+ .concat(c, [new EndTagTk( 'caption' )], te);
1225	1191	} else {
1226	1192	var caption = [];
1227	1193	}
1228	1194	//dp(pp(res));
1229	1195
1230		~~- return [res].concat(caption, body,~~
1231		~~- [{type: 'ENDTAG', name: 'table'}]);~~
	1196	+ return [res].concat(caption, body, [new EndTagTk( 'table' )]);
1232	1197	}
1233	1198
1234	1199	table_start
—	—	@@ -1263,8 +1228,8 @@
1264	1229	table_firstrow
1265	1230	= td:(table_data / table_heading)+ {
1266	1231	//dp('firstrow: ' + pp(td));
1267		~~- return [{ type: 'TAG', name: 'tr' }]~~
1268		~~- .concat(td, [{type: 'ENDTAG', name: 'tr'}]);~~
	1232	+ return [ new TagTk( 'tr' )]
	1233	+ .concat(td, [ new EndTagTk( 'tr' )]);
1269	1234	}
1270	1235
1271	1236	table_row
—	—	@@ -1274,8 +1239,8 @@
1275	1240	a:(as:generic_attribute+ space* "\|" !"\|" { return as } )?
1276	1241	space*
1277	1242	td:(table_data / table_heading)* {
1278		~~- return n.concat([{type: 'TAG', name: 'tr'}]~~
1279		~~- , td, [{type: 'ENDTAG', name: 'tr'}]);~~
	1243	+ return n.concat([ new TagTk( 'tr' )]
	1244	+ , td, [ new EndTagTk( 'tr' )]);
1280	1245	}
1281	1246
1282	1247	table_data
—	—	@@ -1293,8 +1258,8 @@
1294	1259	a = [];
1295	1260	}
1296	1261	//dp("table data result: " + pp(td) + ", attribts: " + pp(a));
1297		~~- return n.concat( [{ type: 'TAG', name: 'td', attribs: a}]~~
1298		~~- , td, [{type: 'ENDTAG', name: 'td'}] );~~
	1262	+ return n.concat( [ new TagTk( 'td', a ) ]
	1263	+ , td, [ new EndTagTk( 'td' ) ] );
1299	1264	}
1300	1265
1301	1266	table_heading
—	—	@@ -1304,8 +1269,8 @@
1305	1270	if ( a == '' ) {
1306	1271	a = [];
1307	1272	}
1308		~~- return n.concat( [{type: 'TAG', name: 'th', attribs: a}]~~
1309		~~- , c, [{type: 'ENDTAG', name: 'th'}]);~~
	1273	+ return n.concat( [ new TagTk( 'th', a )]
	1274	+ , c, [ new EndTagTk( 'th' ) ]);
1310	1275	}
1311	1276
1312	1277	thtd_attribs
Index: trunk/extensions/VisualEditor/modules/parser/ext.Cite.js
—	—	@@ -178,11 +178,8 @@
179	179	// XXX: Add round-trip info here?
180	180	]
181	181	},
	182	+ '[' + bits.join(' ') + ']',
182	183	{
183		~~- type: 'TEXT',~~
184		~~- value: '[' + bits.join(' ') + ']'~~
185		~~- },~~
186		~~- {~~
187	184	type: 'ENDTAG',
188	185	name: 'a'
189	186	},
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.tokenizer.peg.js
—	—	@@ -10,13 +10,16 @@
11	11	path = require('path'),
12	12	fs = require('fs'),
13	13	$ = require('jquery'),
14		~~- events = require('events');~~
	14	+ events = require('events'),
	15	+ defines = require('./mediawiki.parser.defines.js');
15	16
16	17	function PegTokenizer() {
17	18	var pegSrcPath = path.join( __dirname, 'pegTokenizer.pegjs.txt' );
18	19	this.src = fs.readFileSync( pegSrcPath, 'utf8' );
19	20	}
20	21
	22	+
	23	+
21	24	// Inherit from EventEmitter
22	25	PegTokenizer.prototype = new events.EventEmitter();
23	26	PegTokenizer.prototype.constructor = PegTokenizer;
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.Sanitizer.js
—	—	@@ -34,7 +34,7 @@
35	35	}
36	36	var hrefKV = this.manager.env.lookupKV( token.attribs, 'href' );
37	37	if ( hrefKV !== null ) {
38		~~- var bits = hrefKV[1].match( /(.?\/\/)([^\/]+)(\/?.)/ );~~
	38	+ var bits = hrefKV.v.match( /(.?\/\/)([^\/]+)(\/?.)/ );
39	39	if ( bits ) {
40	40	proto = bits[1];
41	41	host = bits[2];
—	—	@@ -42,10 +42,10 @@
43	43	} else {
44	44	proto = '';
45	45	host = '';
46		~~- path = hrefKV[1];~~
	46	+ path = hrefKV.v;
47	47	}
48	48	host = this._stripIDNs( host );
49		~~- hrefKV[1] = proto + host + path;~~
	49	+ hrefKV.v = proto + host + path;
50	50	}
51	51	return { token: token };
52	52	};
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
—	—	@@ -16,7 +16,8 @@
17	17	qs = require('querystring'),
18	18	ParserFunctions = require('./ext.core.ParserFunctions.js').ParserFunctions,
19	19	AttributeTransformManager = require('./mediawiki.TokenTransformManager.js')
20		~~- .AttributeTransformManager;~~
	20	+ .AttributeTransformManager,
	21	+ defines = require('./mediawiki.parser.defines.js');
21	22
22	23
23	24	function TemplateHandler ( manager ) {
—	—	@@ -70,11 +71,9 @@
71	72	},
72	73	transformCB,
73	74	i = 0,
74		~~- kvs = [],~~
75		~~- res,~~
76		~~- kv;~~
	75	+ res;
77	76
78		~~- var attributes = [[[{ type: 'TEXT', value: '' }] , token.target ]]~~
	77	+ var attributes = [ {k: [''] , v: token.target} ]
79	78	.concat( this._nameArgs( token.orderedArgs ) );
80	79
81	80	//this.manager.env.dp( 'before AttributeTransformManager: ' +
—	—	@@ -109,8 +108,8 @@
110	109	out = [];
111	110	for ( var i = 0, l = orderedArgs.length; i < l; i++ ) {
112	111	// FIXME: Also check for whitespace-only named args!
113		~~- if ( ! orderedArgs[i][0].length ) {~~
114		~~- out.push( [[{ type: 'TEXT', value: n }], orderedArgs[i][1]]);~~
	112	+ if ( ! orderedArgs[i].k.length ) {
	113	+ out.push( {k: [ n.toString() ], v: orderedArgs[i].v } );
115	114	n++;
116	115	} else {
117	116	out.push( orderedArgs[i] );
—	—	@@ -129,7 +128,7 @@
130	129	this.manager.env.dp( 'TemplateHandler._returnAttributes: ' + JSON.stringify(attributes) );
131	130	// Remove the target from the attributes
132	131	tplExpandData.attribsAsync = false;
133		~~- tplExpandData.target = attributes[0][1];~~
	132	+ tplExpandData.target = attributes[0].v;
134	133	attributes.shift();
135	134	tplExpandData.expandedArgs = attributes;
136	135	if ( tplExpandData.overallAsync ) {
—	—	@@ -195,23 +194,10 @@
196	195	if( checkRes ) {
197	196	// Loop detected or depth limit exceeded, abort!
198	197	res = [
199		~~- {~~
200		~~- type: 'TEXT',~~
201		~~- value: checkRes~~
202		~~- },~~
203		~~- {~~
204		~~- type: 'TAG',~~
205		~~- name: 'a',~~
206		~~- attrib: [['href', target]]~~
207		~~- },~~
208		~~- {~~
209		~~- type: 'TEXT',~~
210		~~- value: target~~
211		~~- },~~
212		~~- {~~
213		~~- type: 'ENDTAG',~~
214		~~- name: 'a'~~
215		~~- }~~
	198	+ checkRes,
	199	+ new TagTk( 'a', [{k: 'href', v: target}] ),
	200	+ target,
	201	+ new EndTagTk( 'a' )
216	202	];
217	203	if ( tplExpandData.overallAsync ) {
218	204	return tplExpandData.cb( res, false );
—	—	@@ -367,7 +353,7 @@
368	354	*/
369	355	TemplateHandler.prototype.onTemplateArg = function ( token, cb, frame ) {
370	356
371		~~- var attributes = [[token.argname, token.defaultvalue]];~~
	357	+ var attributes = [{k: token.argname, v: token.defaultvalue}];
372	358
373	359	token.resultTokens = false;
374	360
—	—	@@ -391,8 +377,8 @@
392	378
393	379	TemplateHandler.prototype._returnArgAttributes = function ( token, cb, frame, attributes ) {
394	380	//console.log( '_returnArgAttributes: ' + JSON.stringify( attributes ));
395		~~- var argName = this.manager.env.tokensToString( attributes[0][0] ).trim(),~~
396		~~- defaultValue = attributes[0][1],~~
	381	+ var argName = this.manager.env.tokensToString( attributes[0].k ).trim(),
	382	+ defaultValue = attributes[0].v,
397	383	res;
398	384	if ( argName in this.manager.args ) {
399	385	// return tokens for argument
—	—	@@ -405,7 +391,7 @@
406	392	if ( defaultValue.length ) {
407	393	res = defaultValue;
408	394	} else {
409		~~- res = [{ type: 'TEXT', value: '{{{' + argName + '}}}' }];~~
	395	+ res = [ '{{{' + argName + '}}}' ];
410	396	}
411	397	}
412	398	if ( token.resultTokens !== false ) {
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js
—	—	@@ -74,7 +74,7 @@
75	75	//console.log( 'PostExpandParagraphHandler.onAny' );
76	76	this.tokens.push( token );
77	77	if ( token.type === 'COMMENT' \|\|
78		~~- ( token.type === 'TEXT' && token.value.match( /^[\t ]+$/ ) )~~
	78	+ ( token.constructor === String && token.match( /^[\t ]+$/ ) )
79	79	)
80	80	{
81	81	// Continue with collection..
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
—	—	@@ -25,7 +25,7 @@
26	26	var kv;
27	27	for ( var i = 0, l = kvs.length; i < l; i++ ) {
28	28	kv = kvs[i];
29		~~- if ( kv[0] === key ) {~~
	29	+ if ( kv.k === key ) {
30	30	// found, return it.
31	31	return kv;
32	32	}
—	—	@@ -42,9 +42,9 @@
43	43	var res = {};
44	44	for ( var i = 0, l = kvs.length; i < l; i++ ) {
45	45	var kv = kvs[i],
46		~~- key = this.tokensToString( kv[0] ).trim();~~
	46	+ key = this.tokensToString( kv.k ).trim();
47	47	if( res[key] === undefined ) {
48		~~- res[key] = kv[1];~~
	48	+ res[key] = kv.v;
49	49	}
50	50	}
51	51	//console.log( 'KVtoHash: ' + JSON.stringify( res ));
—	—	@@ -122,8 +122,8 @@
123	123	JSON.stringify( token ) );
124	124	continue;
125	125	}
126		~~- if ( token.type === 'TEXT' ) {~~
127		~~- out.push( token.value );~~
	126	+ if ( token.constructor === String ) {
	127	+ out.push( token );
128	128	} else if ( token.type === 'COMMENT' \|\| token.type === 'NEWLINE' ) {
129	129	// strip comments and newlines
130	130	} else {

Follow-up revisions

Revision	Commit summary	Author	Date
r110657	Fix nowiki tokenization regression introduced r110495	gwicke	13:10, 3 February 2012

Status & tagging log

17:04, 1 February 2012 GWicke (talk | contribs) changed the status of r110495 [removed: old added: deferred]
16:31, 1 February 2012 GWicke (talk | contribs) changed the status of r110495 [removed: new added: old]