r110936 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r110935‎ | r110936 | r110937 >
Date:15:10, 8 February 2012
Author:gwicke
Status:deferred
Tags:
Comment:
Enable support for general preprocessor functionality in attribute keys and
values. This includes comments, templates and template arguments.

This also replaces the specialized expansion logic in the TemplateHandler. The
removal of link validation lets one more parser test fail for now. External
link target validation will need to be implemented in the token stream handler
for links. This is noted as TODO in
https://www.mediawiki.org/wiki/Future/Parser_development#Token_stream_transforms.
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.Sanitizer.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
——@@ -486,7 +486,28 @@
487487
488488 for ( var i = 0; i < tokensLength; i++ ) {
489489 token = tokens[i];
 490+
 491+ if ( token.attribs ) {
 492+ // TODO: Transform token attributes (that need it) before general token processing.
 493+ // Sync return -> continue as-is.
 494+ // Async return -> return with async
 495+ // per-item callback: function ( res ) {
 496+ var atm = new AttributeTransformManager (
 497+ this,
 498+ this._returnAttributes.bind( this, token, cb )
 499+ );
490500
 501+ if ( atm.process( token.attribs ) ) {
 502+ // async attribute expansion ongoing
 503+ activeAccum = accum;
 504+ accum = new TokenAccumulator( this, activeAccum.getParentCB( 'sibling' ) );
 505+ cb = accum.getParentCB( 'child' );
 506+ // Only transform token once attributes are fully processed
 507+ // _returnAttributes will call transformTokens if sync flag is not set
 508+ continue;
 509+ }
 510+ }
 511+
491512 switch ( token.constructor ) {
492513 case String:
493514 res = this._transformToken( token, phaseEndRank, ts.text, cb );
——@@ -545,6 +566,8 @@
546567 accum = new TokenAccumulator( this, activeAccum.getParentCB( 'sibling' ) );
547568 cb = accum.getParentCB( 'child' );
548569 }
 570+
 571+
549572 }
550573
551574 // Return finished tokens directly to caller, and indicate if further
——@@ -590,6 +613,36 @@
591614 }
592615 };
593616
 617+AsyncTokenTransformManager.prototype._returnAttributes
 618+ = function ( token, cb, attributes, async )
 619+{
 620+ this.env.dp( 'AsyncTokenTransformManager._returnAttributes: ' +
 621+ JSON.stringify( token, null, 2 ) +
 622+ JSON.stringify( attributes, null, 2 ) );
 623+ token.attribs = attributes;
 624+
 625+
 626+ if ( async ) {
 627+ // Attributes were expanded asynchronously, so now we still need to
 628+ // process the token.
 629+ var res = this.transformTokens( [token], cb );
 630+ if ( res.async ) {
 631+ // Token transformation is again asynchronous. There is already a
 632+ // TokenAccumulator associated with this token, so we only need to
 633+ // be concerned with synchronously returned tokens.
 634+ if ( res.tokens ) {
 635+ // Pass on already-returned tokens to the cb
 636+ cb( res.tokens, true );
 637+ }
 638+ } else if ( res.token ) {
 639+ cb ( [res.token], false );
 640+ } else {
 641+ cb ( res.tokens, false );
 642+ }
 643+ }
 644+};
 645+
 646+
594647 /**
595648 * Callback for the end event emitted from the tokenizer.
596649 * Either signals the end of input to the tail of an ongoing asynchronous
——@@ -764,34 +817,52 @@
765818 for ( var i = 0, l = attributes.length; i < l; i++ ) {
766819 var kv = { key: [], value: [] };
767820 this.kvs.push( kv );
 821+ var cur = attributes[i];
768822
769 - // Assume that the return is async, will be decremented in callback
770 - this.outstanding += 2;
 823+ if ( cur.k.constructor !== String ) {
 824+ // Assume that the return is async, will be decremented in callback
 825+ this.outstanding++;
771826
772 - // transform the key
773 - pipe = this.manager.getAttributePipeline( this.manager.args );
774 - pipe.addListener( 'chunk',
775 - this.onChunk.bind( this, this._returnAttributeKey.bind( this, i ) )
776 - );
777 - pipe.addListener( 'end',
778 - this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) )
779 - );
780 - pipe.process( attributes[i].k.concat([{type:'END'}]) );
 827+ // transform the key
 828+ pipe = this.manager.getAttributePipeline( this.manager.args );
 829+ pipe.addListener( 'chunk',
 830+ this.onChunk.bind( this, this._returnAttributeKey.bind( this, i ) )
 831+ );
 832+ pipe.addListener( 'end',
 833+ this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) )
 834+ );
 835+ pipe.process( attributes[i].k.concat([{type:'END'}]) );
 836+ } else {
 837+ kv.key = cur.k;
 838+ }
781839
782 - // transform the value
783 - pipe = this.manager.getAttributePipeline( this.manager.args );
784 - pipe.addListener( 'chunk',
785 - this.onChunk.bind( this, this._returnAttributeValue.bind( this, i ) )
786 - );
787 - pipe.addListener( 'end',
788 - this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
789 - );
790 - //console.log('starting attribute transform of ' + JSON.stringify( attributes[i].v ) );
791 - pipe.process( attributes[i].v.concat([{type:'END'}]) );
 840+ if ( cur.v.constructor !== String ) {
 841+ // Assume that the return is async, will be decremented in callback
 842+ this.outstanding++;
 843+
 844+ // transform the value
 845+ pipe = this.manager.getAttributePipeline( this.manager.args );
 846+ pipe.addListener( 'chunk',
 847+ this.onChunk.bind( this, this._returnAttributeValue.bind( this, i ) )
 848+ );
 849+ pipe.addListener( 'end',
 850+ this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
 851+ );
 852+ //console.log('starting attribute transform of ' + JSON.stringify( attributes[i].v ) );
 853+ pipe.process( cur.v.concat([{type:'END'}]) );
 854+ } else {
 855+ kv.value = cur.v;
 856+ }
792857 }
793858 this.outstanding--;
794859 if ( this.outstanding === 0 ) {
795860 this._returnAttributes();
 861+ // synchronous / done
 862+ return false;
 863+ } else {
 864+ // async, will call back
 865+ this.async = true;
 866+ return true;
796867 }
797868 };
798869
——@@ -805,7 +876,7 @@
806877
807878 // and call the callback with the result
808879 //console.log('AttributeTransformManager._returnAttributes: ' + out );
809 - this.callback( out );
 880+ this.callback( out, this.async );
810881 };
811882
812883 /**
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
——@@ -80,9 +80,11 @@
8181 };
8282
8383 ParserFunctions.prototype['pf_#tag'] = function ( target, argList, argDict ) {
84 - return [ new TagTk( target ),
85 - argList[0].v,
86 - new EndTagTk( target ) ];
 84+ // XXX: handle things like {{#tag:nowiki|{{{input1|[[shouldnotbelink]]}}}}}
 85+ // https://www.mediawiki.org/wiki/Future/Parser_development#Token_stream_transforms
 86+ return [ new TagTk( target ) ]
 87+ .concat( argList[0].v,
 88+ [ new EndTagTk( target ) ] );
8789 };
8890
8991 // A first approximation, anyway..
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
——@@ -365,7 +365,7 @@
366366 // Returns either a list of tokens, or a plain string (if nothing is to be
367367 // processed).
368368 preprocessor_text
369 - = r:( t:[^'<~[{\n\r\t|!\]} &=]+ { return t.join(''); }
 369+ = r:( t:[^<~[{\n\r\t|!\]} &=]+ { return t.join(''); }
370370 / directive
371371 / !inline_breaks text_char )+ {
372372 return flatten_string ( r );
——@@ -790,10 +790,11 @@
791791 = "{{" target:template_param_text
792792 params:(newline? "|" newline? p:template_param { return p })*
793793 "}}" {
794 - target = flatten( target );
795 - var obj = new SelfclosingTagTk( 'template' );
796 - obj.orderedArgs = params;
797 - obj.target = target;
 794+ // Insert target as first positional attribute, so that it can be
 795+ // generically expanded. The TemplateHandler then needs to shift it out
 796+ // again.
 797+ params.unshift( { k: '', v: flatten( target ) } );
 798+ var obj = new SelfclosingTagTk( 'template', params );
798799 //console.log( 'tokenizer template ' + JSON.stringify( target ));
799800 return obj;
800801 }
——@@ -808,18 +809,8 @@
809810 params:( newline? "|" newline? p:template_param { return p })*
810811 "}}}" {
811812 name = flatten( name );
812 - var obj = new SelfclosingTagTk( 'templatearg', [] );
813 - obj.argname= name;
814 - if (params && params.length) {
815 - // HACK, not final.
816 - //obj.attribs.push(new KV('data-defaultvalue', params[0][1]));
817 - //console.log(JSON.stringify(params, null, 2));
818 - //obj.attribs.push(new KV('data-json-args', JSON.stringify(params)));
819 - obj.defaultvalue = params[0].v.length ? params[0].v :
820 - [ '' ];
821 - } else {
822 - obj.defaultvalue = [];
823 - }
 813+ params.unshift( { k: '', v: name } );
 814+ var obj = new SelfclosingTagTk( 'templatearg', params );
824815 //console.log( 'tokenizer tplarg ' + JSON.stringify( obj, null, 2 ));
825816 return obj;
826817 }
——@@ -863,7 +854,8 @@
864855 wikilink
865856 = "[["
866857 ! url
867 - target:link_target
 858+ //target:link_target
 859+ target:preprocessor_text
868860 lcontent:( "|" lt:link_text { return lt } )*
869861 "]]"
870862 // XXX In real MediaWiki, this is a language-dependent positive character
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.Sanitizer.js
——@@ -13,6 +13,7 @@
1414
1515
1616 function Sanitizer ( manager ) {
 17+ this.manager = manager;
1718 this.register( manager );
1819 }
1920
——@@ -33,7 +34,11 @@
3435 return { token: token };
3536 }
3637 var hrefKV = this.manager.env.lookupKV( token.attribs, 'href' );
 38+ // FIXME: Should the flattening of attributes to text be performed as part
 39+ // of the AttributeTransformManager processing? This certainly is not the
 40+ // right place!
3741 if ( hrefKV !== null ) {
 42+ hrefKV.v = this.manager.env.tokensToString( hrefKV.v );
3843 var bits = hrefKV.v.match( /(.*?\/\/)([^\/]+)(\/?.*)/ );
3944 if ( bits ) {
4045 proto = bits[1];
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
——@@ -65,7 +65,6 @@
6666 cb: cb,
6767 origToken: token,
6868 resultTokens: [],
69 - attribsAsync: true,
7069 overallAsync: false,
7170 expandDone: false
7271 },
——@@ -73,68 +72,37 @@
7473 i = 0,
7574 res;
7675
77 - var attributes = [ {k: [''] , v: token.target} ]
78 - .concat( this._nameArgs( token.orderedArgs ) );
 76+ tplExpandData.target = token.attribs.shift().v;
 77+ tplExpandData.expandedArgs = this._nameArgs( token.attribs );
7978
80 - //this.manager.env.dp( 'before AttributeTransformManager: ' +
81 - // JSON.stringify( attributes, null, 2 ) );
82 - new AttributeTransformManager(
83 - this.manager,
84 - this._returnAttributes.bind( this, tplExpandData )
85 - ).process( attributes );
86 -
87 - // Unblock finish
88 - if ( ! tplExpandData.attribsAsync ) {
89 - // Attributes were transformed synchronously
90 - this.manager.env.dp (
91 - 'sync attribs for ' + JSON.stringify( tplExpandData.target ),
92 - tplExpandData.expandedArgs
93 - );
94 - // All attributes are fully expanded synchronously (no IO was needed)
95 - return this._expandTemplate ( tplExpandData );
96 - } else {
97 - // Async attribute expansion is going on
98 - this.manager.env.dp( 'async return for ' + JSON.stringify( token ));
99 - tplExpandData.overallAsync = true;
100 - return { async: true };
101 - }
 79+ // Attributes were transformed synchronously
 80+ //this.manager.env.dp (
 81+ // 'sync attribs for ' + JSON.stringify( tplExpandData.target ),
 82+ // tplExpandData.expandedArgs
 83+ //);
 84+ // All attributes are fully expanded synchronously (no IO was needed)
 85+ return this._expandTemplate ( tplExpandData );
10286 };
10387
10488 /**
10589 * Create positional (number) keys for arguments without explicit keys
10690 */
107 -TemplateHandler.prototype._nameArgs = function ( orderedArgs ) {
 91+TemplateHandler.prototype._nameArgs = function ( attribs ) {
10892 var n = 1,
10993 out = [];
110 - for ( var i = 0, l = orderedArgs.length; i < l; i++ ) {
 94+ for ( var i = 0, l = attribs.length; i < l; i++ ) {
11195 // FIXME: Also check for whitespace-only named args!
112 - if ( ! orderedArgs[i].k.length ) {
113 - out.push( {k: [ n.toString() ], v: orderedArgs[i].v } );
 96+ if ( ! attribs[i].k.length ) {
 97+ out.push( {k: [ n.toString() ], v: attribs[i].v } );
11498 n++;
11599 } else {
116 - out.push( orderedArgs[i] );
 100+ out.push( attribs[i] );
117101 }
118102 }
119103 this.manager.env.dp( '_nameArgs: ' + JSON.stringify( out ) );
120104 return out;
121105 };
122106
123 -/**
124 - * Callback for argument (including target) expansion in AttributeTransformManager
125 - */
126 -TemplateHandler.prototype._returnAttributes = function ( tplExpandData,
127 - attributes )
128 -{
129 - this.manager.env.dp( 'TemplateHandler._returnAttributes: ' + JSON.stringify(attributes) );
130 - // Remove the target from the attributes
131 - tplExpandData.attribsAsync = false;
132 - tplExpandData.target = attributes[0].v;
133 - attributes.shift();
134 - tplExpandData.expandedArgs = attributes;
135 - if ( tplExpandData.overallAsync ) {
136 - this._expandTemplate ( tplExpandData );
137 - }
138 -};
139107
140108 /**
141109 * Fetch, tokenize and token-transform a template after all arguments and the
——@@ -246,9 +214,10 @@
247215 // expandDone is set to true in our _onEnd handler.
248216 if ( tplExpandData.overallAsync ||
249217 ! tplExpandData.expandDone ) {
250 - tplExpandData.overallAsync = true;
251218 this.manager.env.dp( 'Async return from _expandTemplate for ' +
252 - JSON.stringify ( tplExpandData.target ) );
 219+ JSON.stringify ( tplExpandData.target ) +
 220+ JSON.stringify( tplExpandData, null, 2 ));
 221+ tplExpandData.overallAsync = true;
253222 return { async: true };
254223 } else {
255224 this.manager.env.dp( 'Sync return from _expandTemplate for ' +
——@@ -319,12 +288,13 @@
320289 // @fixme normalize name?
321290 var self = this;
322291 if ( title in this.manager.env.pageCache ) {
323 - // Unwind the stack
324 - process.nextTick(
325 - function () {
326 - callback( self.manager.env.pageCache[title], title );
327 - }
328 - );
 292+ callback( self.manager.env.pageCache[title], title );
 293+ //// Unwind the stack
 294+ //process.nextTick(
 295+ // function () {
 296+ // callback( self.manager.env.pageCache[title], title );
 297+ // }
 298+ //);
329299 } else if ( ! this.manager.env.fetchTemplates ) {
330300 callback( 'Warning: Page/template fetching disabled, and no cache for ' +
331301 title, title );
——@@ -354,53 +324,26 @@
355325 * Expand template arguments with tokens from the containing frame.
356326 */
357327 TemplateHandler.prototype.onTemplateArg = function ( token, frame, cb ) {
358 -
359 - var attributes = [{k: token.argname, v: token.defaultvalue}];
360 -
361 - token.resultTokens = false;
362 -
363 - new AttributeTransformManager (
364 - this.manager,
365 - this._returnArgAttributes.bind( this, token, cb, frame )
366 - ).process( attributes );
367 -
368 - if ( token.resultTokens !== false ) {
369 - // synchronous return
370 - //console.log( 'synchronous attribute expand: ' + JSON.stringify( token.resultTokens ) );
371 -
372 - return { tokens: token.resultTokens };
373 - } else {
374 - //console.log( 'asynchronous attribute expand: ' + JSON.stringify( token, null, 2 ) );
375 - // asynchronous return
376 - token.resultTokens = [];
377 - return { async: true };
378 - }
379 -};
380 -
381 -TemplateHandler.prototype._returnArgAttributes = function ( token, cb, frame, attributes ) {
382 - //console.log( '_returnArgAttributes: ' + JSON.stringify( attributes ));
383 - var argName = this.manager.env.tokensToString( attributes[0].k ).trim(),
384 - defaultValue = attributes[0].v,
 328+ var argName = this.manager.env.tokensToString( token.attribs.shift().v ).trim(),
385329 res;
 330+
386331 if ( argName in this.manager.args ) {
387332 // return tokens for argument
388333 //console.log( 'templateArg found: ' + argName +
389334 // ' vs. ' + JSON.stringify( this.manager.args ) );
390335 res = this.manager.args[argName];
391336 } else {
 337+ var defaultValue = token.attribs[0];
392338 this.manager.env.dp( 'templateArg not found: ' + argName +
393 - ' vs. ' + JSON.stringify( this.manager.args ) );
394 - if ( defaultValue.length ) {
395 - res = defaultValue;
 339+ ' vs. ' + JSON.stringify( defaultValue ) );
 340+ if ( defaultValue && ! defaultValue.k.length && defaultValue.v.length ) {
 341+ res = defaultValue.v;
396342 } else {
397343 res = [ '{{{' + argName + '}}}' ];
398344 }
399345 }
400 - if ( token.resultTokens !== false ) {
401 - cb( res );
402 - } else {
403 - token.resultTokens = res;
404 - }
 346+ return { tokens: res };
 347+
405348 };
406349
407350

Status & tagging log