r108662 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r108661‎ | r108662 | r108663 >
Date:19:48, 11 January 2012
Author:gwicke
Status:deferred
Tags:
Comment:
Results of early template expansion debugging. Still disabled by default, but
getting closer.
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
@@ -206,7 +206,8 @@
207207 if ( ts ) {
208208 for ( i = 0, l = ts.length; i < l; i++ ) {
209209 transformer = ts[i];
210 - if ( res.token.rank && transformer.rank <= res.token.rank ) {
 210+ if ( res.token.rank && transformer.rank < res.token.rank ) {
 211+ //console.log( 'SKIPPING' + JSON.stringify( token, null, 2 ) + JSON.stringify( transformer, null, 2 ) );
211212 // skip transformation, was already applied.
212213 continue;
213214 }
@@ -311,6 +312,8 @@
312313 this.childFactories = childFactories;
313314 this._construct();
314315 this._reset( args, env );
 316+ // FIXME: pass actual title?
 317+ this.loopCheck = new LoopCheck ( null );
315318 }
316319
317320 // Inherit from TokenTransformManager, and thus also from EventEmitter.
@@ -333,7 +336,7 @@
334337 var child = pipe.last;
335338 // We assume that the title was already checked against this.loopCheck
336339 // before!
337 - child.loopCheck = new LoopCheck ( this.loopCheck, title );
 340+ child.loopCheck = new LoopCheck ( title, this.loopCheck );
338341 // Same for depth!
339342 child.depth = this.depth + 1;
340343 return pipe;
@@ -382,6 +385,16 @@
383386 }
384387 };
385388
 389+/**
 390+ * Simplified wrapper that processes all tokens passed in
 391+ */
 392+AsyncTokenTransformManager.prototype.process = function ( tokens ) {
 393+ if ( ! $.isArray ( tokens ) ) {
 394+ tokens = [tokens];
 395+ }
 396+ this.onChunk( tokens );
 397+ this.onEndEvent();
 398+};
386399
387400 /**
388401 * Transform and expand tokens. Transformed token chunks will be emitted in
@@ -422,6 +435,7 @@
423436 token,
424437 ts = this.transformers;
425438
 439+
426440 for ( var i = 0; i < tokensLength; i++ ) {
427441 token = tokens[i];
428442
@@ -555,6 +569,15 @@
556570 SyncTokenTransformManager.prototype.constructor = SyncTokenTransformManager;
557571
558572
 573+SyncTokenTransformManager.prototype.process = function ( tokens ) {
 574+ if ( ! $.isArray ( tokens ) ) {
 575+ tokens = [tokens];
 576+ }
 577+ this.onChunk( tokens );
 578+ this.onEndEvent();
 579+};
 580+
 581+
559582 /**
560583 * Global in-order and synchronous traversal on token stream. Emits
561584 * transformed chunks of tokens in the 'chunk' event.
@@ -649,13 +672,14 @@
650673 * @param {Object} Containing TokenTransformManager
651674 */
652675 function AttributeTransformManager ( manager, callback ) {
 676+ this.manager = manager;
653677 this.callback = callback;
654678 this.outstanding = 0;
655679 this.kvs = [];
656 - this.pipe = manager.newAttributePipeline( manager.args );
 680+ this.pipe = manager.getAttributePipeline( manager.args );
657681 }
658682
659 -AttributeTransformManager.prototype.transformAttributes = function ( attributes ) {
 683+AttributeTransformManager.prototype.process = function ( attributes ) {
660684 // Potentially need to use multiple pipelines to support concurrent async expansion
661685 //this.pipe.process(
662686 var pipe,
@@ -664,30 +688,28 @@
665689 // transform each argument (key and value), and handle asynchronous returns
666690 for ( var i = 0, l = attributes.length; i < l; i++ ) {
667691 kv = { key: [], value: [] };
668 - kvs.push( kv );
 692+ this.kvs.push( kv );
669693
670 - ref = { frame: templateExpandData, index: i };
671 -
672694 // Assume that the return is async, will be decremented in callback
673695 this.outstanding += 2;
674696
675697 // transform the key
676 - pipe = manager.getAttributePipeline( manager.args );
677 - pipe.last.addListener( 'chunk',
678 - this.onChunk.bind( this, this._returnAttributeKey, ref )
 698+ pipe = this.manager.getAttributePipeline( this.manager.args );
 699+ pipe.addListener( 'chunk',
 700+ this.onChunk.bind( this, this._returnAttributeKey.bind( this, i ) )
679701 );
680 - pipe.last.addListener( 'end',
681 - this.onEnd.bind( this, this._returnAttributeKey, ref )
 702+ pipe.addListener( 'end',
 703+ this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) )
682704 );
683705 pipe.process( attributes[i][0] );
684706
685707 // transform the value
686 - pipe = manager.getAttributePipeline( manager.args );
687 - pipe.last.addListener( 'chunk',
688 - this.onChunk.bind( this, this._returnAttributeValue, ref )
 708+ pipe = this.manager.getAttributePipeline( this.manager.args );
 709+ pipe.addListener( 'chunk',
 710+ this.onChunk.bind( this, this._returnAttributeValue.bind( this, i ) )
689711 );
690 - pipe.last.addListener( 'end',
691 - this.onEnd.bind( this, this._returnAttributeKey, ref )
 712+ pipe.addListener( 'end',
 713+ this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
692714 );
693715 pipe.process( attributes[i][1] );
694716 }
@@ -697,7 +719,8 @@
698720 // convert attributes
699721 var out = [];
700722 for ( var i = 0, l = this.kvs.length; i < l; i++ ) {
701 - out.push( [this.kvs.key, this.kvs.value] );
 723+ var kv = this.kvs[i];
 724+ out.push( [kv.key, kv.value] );
702725 }
703726
704727 // and call the callback with the result
@@ -708,14 +731,14 @@
709732 * Collect chunks returned from the pipeline
710733 */
711734 AttributeTransformManager.prototype.onChunk = function ( callback, ref, chunk ) {
712 - callback.call( this, ref, chunk, true );
 735+ callback.call( ref, chunk, true );
713736 };
714737
715738 /**
716739 * Empty the pipeline by returning to the parent
717740 */
718741 AttributeTransformManager.prototype.onEnd = function ( callback, ref ) {
719 - callback.call(this, ref, [], false );
 742+ callback.call( ref, [], false );
720743 };
721744
722745
@@ -723,16 +746,12 @@
724747 * Callback for async argument value expansions
725748 */
726749 AttributeTransformManager.prototype._returnAttributeValue = function ( ref, tokens, notYetDone ) {
727 - var frame = ref.frame;
728 - this.kvs[ref.index].value.push( tokens );
 750+ this.kvs[ref].value.push( tokens );
729751 if ( ! notYetDone ) {
730 - frame.outstanding--;
731 - if ( frame.outstanding === 0 ) {
 752+ this.outstanding--;
 753+ if ( this.outstanding === 0 ) {
732754 // this calls back to frame.cb, so no return here.
733 - this.outstanding--;
734 - if ( this.outstanding === 0 ) {
735 - this._returnAttributes();
736 - }
 755+ this._returnAttributes();
737756 }
738757 }
739758 };
@@ -741,16 +760,12 @@
742761 * Callback for async argument key expansions
743762 */
744763 AttributeTransformManager.prototype._returnAttributeKey = function ( ref, tokens, notYetDone ) {
745 - var frame = ref.frame;
746 - this.kvs[ref.index].key.push( tokens );
 764+ this.kvs[ref].key.push( tokens );
747765 if ( ! notYetDone ) {
748 - frame.outstanding--;
749 - if ( frame.outstanding === 0 ) {
 766+ this.outstanding--;
 767+ if ( this.outstanding === 0 ) {
750768 // this calls back to frame.cb, so no return here.
751 - this.outstanding--;
752 - if ( this.outstanding === 0 ) {
753 - this._returnAttributes();
754 - }
 769+ this._returnAttributes();
755770 }
756771 }
757772 };
@@ -878,7 +893,7 @@
879894 * @class
880895 * @constructor
881896 */
882 -function LoopCheck ( parent, title ) {
 897+function LoopCheck ( title, parent ) {
883898 if ( parent ) {
884899 this.parent = parent;
885900 } else {
@@ -909,4 +924,5 @@
910925 if (typeof module == "object") {
911926 module.exports.AsyncTokenTransformManager = AsyncTokenTransformManager;
912927 module.exports.SyncTokenTransformManager = SyncTokenTransformManager;
 928+ module.exports.AttributeTransformManager = AttributeTransformManager;
913929 }
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
@@ -693,7 +693,9 @@
694694 type: 'SELFCLOSINGTAG',
695695 name: 'template',
696696 attribs: [['data-target', target]],
697 - args: {}
 697+ orderedArgs: params,
 698+ args: {},
 699+ target: target
698700 };
699701 if (params && params.length) {
700702 var position = 1;
@@ -720,13 +722,13 @@
721723
722724 // XXX: support template and args in target!
723725 template_target
724 - = h:( !"}}" x:([^|\n]) { return x } )* { return h.join(''); }
 726+ = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }
725727
726728 template_param
727729 = name:template_param_name space* "=" space* c:template_param_text {
728 - return [name, c];
 730+ return [{ type: 'TEXT', value: name }, { type: 'TEXT', value: c }];
729731 } / c:template_param_text {
730 - return [null, c];
 732+ return [[], { type: 'TEXT', value: c }];
731733 }
732734
733735 tplarg
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
@@ -16,6 +16,7 @@
1717 PegTokenizer = require('./mediawiki.tokenizer.peg.js').PegTokenizer,
1818 TokenTransformManager = require('./mediawiki.TokenTransformManager.js'),
1919 QuoteTransformer = require('./ext.core.QuoteTransformer.js').QuoteTransformer,
 20+ TemplateHandler = require('./ext.core.TemplateHandler.js').TemplateHandler,
2021 Cite = require('./ext.Cite.js').Cite,
2122 FauxHTML5 = require('./mediawiki.HTML5TreeBuilder.node.js').FauxHTML5,
2223 DOMPostProcessor = require('./mediawiki.DOMPostProcessor.js').DOMPostProcessor,
@@ -58,15 +59,16 @@
5960
6061 // Create an input pipeline for the given input type.
6162 this.inputPipeline = this.makeInputPipeline ( inputType );
 63+ this.inputPipeline.atTopLevel = true;
6264
6365
6466 this.tokenPostProcessor = new TokenTransformManager.SyncTokenTransformManager ( env );
65 - this.tokenPostProcessor.listenForTokensFrom ( this.inputPipeline.last );
 67+ this.tokenPostProcessor.listenForTokensFrom ( this.inputPipeline );
6668
6769
6870 // Add token transformations..
69 - var qt = new QuoteTransformer( this.tokenPostProcessor );
70 -
 71+ new QuoteTransformer( this.tokenPostProcessor );
 72+
7173 //var citeExtension = new Cite( this.tokenTransformer );
7274
7375
@@ -126,7 +128,7 @@
127129 switch ( inputType ) {
128130 case 'text/wiki':
129131 if ( this.pipelineCache['text/wiki'].input.length ) {
130 - return this.pipelineCache['text/wiki'].attribute.pop();
 132+ return this.pipelineCache['text/wiki'].input.pop();
131133 } else {
132134 var wikiTokenizer = new PegTokenizer();
133135
@@ -147,7 +149,13 @@
148150 },
149151 args, this.env
150152 );
 153+
 154+ // Register template expansion extension
 155+ //new TemplateHandler( tokenExpander );
 156+
151157 tokenExpander.listenForTokensFrom ( tokenPreProcessor );
 158+ // XXX: hack.
 159+ tokenExpander.inputType = inputType;
152160
153161 return new CachedTokenPipeline(
154162 this.cachePipeline.bind( this, 'text/wiki', 'input' ),
@@ -158,6 +166,7 @@
159167 break;
160168
161169 default:
 170+ console.trace();
162171 throw "ParserPipeline.makeInputPipeline: Unsupported input type " + inputType;
163172 }
164173 };
@@ -179,7 +188,13 @@
180189 */
181190 var tokenPreProcessor = new TokenTransformManager.SyncTokenTransformManager ( this.env );
182191 var tokenExpander = new TokenTransformManager.AsyncTokenTransformManager (
183 - this.makeInputPipeline.bind( this ), args, this.env );
 192+ {
 193+ 'input': this.makeInputPipeline.bind( this ),
 194+ 'attributes': this.makeAttributePipeline.bind( this )
 195+ },
 196+ args, this.env
 197+ );
 198+ //new TemplateHandler( tokenExpander );
184199 tokenExpander.listenForTokensFrom ( tokenPreProcessor );
185200
186201 return new CachedTokenPipeline(
@@ -211,7 +226,7 @@
212227 ParserPipeline.prototype.parse = function ( ) {
213228 // Set the pipeline in motion by feeding the first element with the given
214229 // arguments.
215 - this.inputPipeline.first.process.apply( this.inputPipeline.first , arguments );
 230+ this.inputPipeline.process.apply( this.inputPipeline , arguments );
216231 };
217232
218233 // XXX: Lame hack: set document property. Instead, emit events
@@ -254,7 +269,17 @@
255270 CachedTokenPipeline.prototype = new events.EventEmitter();
256271 CachedTokenPipeline.prototype.constructor = CachedTokenPipeline;
257272
 273+
258274 /**
 275+ * Feed input tokens to the first pipeline stage
 276+ */
 277+CachedTokenPipeline.prototype.process = function ( chunk ) {
 278+ //console.log( 'CachedTokenPipeline::process: ' + chunk );
 279+ this.first.process( chunk );
 280+};
 281+
 282+
 283+/**
259284 * Forward chunks to our listeners
260285 */
261286 CachedTokenPipeline.prototype.forwardChunk = function ( chunk ) {
@@ -270,9 +295,11 @@
271296 // first, forward the event
272297 this.emit( 'end' );
273298 // now recycle self
274 - this.removeAllListeners( 'end' );
275 - this.removeAllListeners( 'chunk' );
276 - this.returnToCacheCB ( this );
 299+ if ( ! this.atTopLevel ) {
 300+ this.removeAllListeners( 'end' );
 301+ this.removeAllListeners( 'chunk' );
 302+ this.returnToCacheCB ( this );
 303+ }
277304 };
278305
279306
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
@@ -9,11 +9,13 @@
1010 * @author Gabriel Wicke <gwicke@wikimedia.org>
1111 * @author Brion Vibber <brion@wikimedia.org>
1212 */
13 -var $ = require('jquery');
 13+var $ = require('jquery'),
 14+ AttributeTransformManager = require('./mediawiki.TokenTransformManager.js').AttributeTransformManager;
1415
1516
16 -function TemplateHandler () {
 17+function TemplateHandler ( manager ) {
1718 this.reset();
 19+ this.register( manager );
1820 }
1921
2022 TemplateHandler.prototype.reset = function () {
@@ -45,6 +47,7 @@
4648 * processes the template.
4749 */
4850 TemplateHandler.prototype.onTemplate = function ( token, cb ) {
 51+ console.log('onTemplate!');
4952
5053 // check for 'subst:'
5154 // check for variable magic names
@@ -54,7 +57,7 @@
5558 // create a new temporary frame for argument and title expansions
5659 var templateTokenTransformData = {
5760 args: {},
58 - env: frame.env,
 61+ manager: this.manager,
5962 outstanding: 0,
6063 cb: cb,
6164 origToken: token,
@@ -66,13 +69,14 @@
6770 res,
6871 kv;
6972
70 - var attributes = [['', token.target]].concat( token.args );
 73+ var attributes = [['', token.target]].concat( token.orderedArgs );
7174
72 - new AttributeTransformer( this.manager,
 75+ new AttributeTransformManager( this.manager,
7376 this._returnAttributes.bind( this, templateTokenTransformData )
7477 ).process( attributes );
7578
76 - if ( templateExpandData.outstanding === 0 ) {
 79+ if ( templateTokenTransformData.outstanding === 0 ) {
 80+ console.log( 'direct call');
7781 return this._expandTemplate ( templateTokenTransformData );
7882 } else {
7983 templateTokenTransformData.isAsync = true;
@@ -81,6 +85,7 @@
8286 };
8387
8488 TemplateHandler.prototype._returnAttributes = function ( templateTokenTransformData, attributes ) {
 89+ console.log( '_returnAttributes' + JSON.stringify(attributes) );
8590 // Remove the target from the attributes
8691 templateTokenTransformData.target = attributes[0][1];
8792 attributes.shift();
@@ -95,6 +100,8 @@
96101 * target were expanded in frame.
97102 */
98103 TemplateHandler.prototype._expandTemplate = function ( templateTokenTransformData ) {
 104+ console.log('TemplateHandler.expandTemplate: ' +
 105+ JSON.stringify( templateTokenTransformData, null, 2 ) );
99106 // First, check the target for loops
100107 this.manager.loopCheck.check( templateTokenTransformData.target );
101108
@@ -103,16 +110,19 @@
104111 // 'text/wiki' input).
105112 // Returned pipe (for now):
106113 // { first: tokenizer, last: AsyncTokenTransformManager }
107 - this.inputPipeline = this.manager.newChildPipeline( inputType, args );
 114+ this.inputPipeline = this.manager.newChildPipeline(
 115+ this.manager.inputType,
 116+ templateTokenTransformData.expandedArgs
 117+ );
108118
109119 // Hook up the AsyncTokenTransformManager output events to call back our
110120 // parentCB.
111 - this.inputPipeline.last.addListener( 'chunk', this._onChunk.bind ( this ) );
112 - this.inputPipeline.last.addListener( 'end', this._onEnd.bind ( this ) );
 121+ this.inputPipeline.addListener( 'chunk', this._onChunk.bind ( this ) );
 122+ this.inputPipeline.addListener( 'end', this._onEnd.bind ( this ) );
113123
114124
115125 // Resolve a possibly relative link
116 - var templateName = this.env.resolveTitle( this.target, 'Template' );
 126+ var templateName = this.manager.env.resolveTitle( this.manager.env.tokensToString( this.target ), 'Template' );
117127 this._fetchTemplateAndTitle( templateName, this._processTemplateAndTitle.bind( this ) );
118128
119129 // Set up a pipeline:
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
@@ -70,7 +70,28 @@
7171 return name;
7272 };
7373
 74+MWParserEnvironment.prototype.tokensToString = function ( tokens ) {
 75+ var out = [];
 76+ // XXX: quick hack, track down non-array sources later!
 77+ if ( ! $.isArray( tokens ) ) {
 78+ tokens = [ tokens ];
 79+ }
 80+ for ( var i = 0, l = tokens.length; i < l; i++ ) {
 81+ console.log( 'MWParserEnvironment.tokensToString: ' + token );
 82+ var token = tokens[i];
 83+ if ( token.type === 'TEXT' ) {
 84+ out.push( token.value );
 85+ } else {
 86+ var tstring = JSON.stringify( token );
 87+ console.log ( 'MWParserEnvironment::tokensToString: ' + tstring );
 88+ out.push( tstring );
 89+ }
 90+ }
 91+ return out.join('');
 92+};
7493
 94+
 95+
7596 if (typeof module == "object") {
7697 module.exports.MWParserEnvironment = MWParserEnvironment;
7798 }

Status & tagging log