r108827 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r108826‎ | r108827 | r108828 >
Date:18:48, 13 January 2012
Author:gwicke
Status:deferred
Tags:
Comment:
Template expansion now enabled and somewhat working, but template fetching
still fails all the time.
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
——@@ -207,7 +207,8 @@
208208 for ( i = 0, l = ts.length; i < l; i++ ) {
209209 transformer = ts[i];
210210 if ( res.token.rank && transformer.rank < res.token.rank ) {
211 - //console.log( 'SKIPPING' + JSON.stringify( token, null, 2 ) + JSON.stringify( transformer, null, 2 ) );
 211+ //console.log( 'SKIPPING' + JSON.stringify( token, null, 2 ) +
 212+ // '\ntransform:\n' + JSON.stringify( transformer, null, 2 ) );
212213 // skip transformation, was already applied.
213214 continue;
214215 }
——@@ -404,15 +405,15 @@
405406 * @param {Array} chunk of tokens
406407 */
407408 AsyncTokenTransformManager.prototype.onChunk = function ( tokens ) {
408 - //console.log('TokenTransformManager onChunk');
409409 // Set top-level callback to next transform phase
410410 var res = this.transformTokens ( tokens, this.tokenCB );
411411 this.tailAccumulator = res.async;
412 - this.emit( 'chunk', tokens );
 412+ //console.log('AsyncTokenTransformManager onChunk ', tokens);
413413 //this.phase2TailCB( tokens, true );
414414 if ( res.async ) {
415415 this.tokenCB = res.async.getParentCB ( 'sibling' );
416416 }
 417+ this.emit( 'chunk', res.tokens );
417418 };
418419
419420 /**
——@@ -422,10 +423,10 @@
423424 */
424425 AsyncTokenTransformManager.prototype.transformTokens = function ( tokens, parentCB ) {
425426
426 - //console.log('_transformPhase01: ' + JSON.stringify(tokens) );
 427+ //console.log('AsyncTokenTransformManager.transformTokens: ' + JSON.stringify(tokens) );
427428
428429 var res,
429 - phaseEndRank = 2, // parametrize!
 430+ phaseEndRank = 2, // XXX: parametrize!
430431 // Prepare a new accumulator, to be used by async children (if any)
431432 localAccum = [],
432433 accum = new TokenAccumulator( parentCB ),
——@@ -478,7 +479,7 @@
479480 // If there is no accumulator yet, then directly return the
480481 // token to the parent. Collect them in localAccum for this
481482 // purpose.
482 - localAccum.push(res.token);
 483+ localAccum.push( res.token );
483484 }
484485 } else {
485486 // re-process token.
——@@ -486,6 +487,7 @@
487488 i--;
488489 }
489490 } else if ( res.async ) {
 491+ //console.log( 'tokens returned' );
490492 // The child now switched to activeAccum, we have to create a new
491493 // accumulator for the next potential child.
492494 activeAccum = accum;
——@@ -511,15 +513,15 @@
512514 * (everything is done), or a truish value if not yet done.
513515 */
514516 AsyncTokenTransformManager.prototype._returnTokens = function ( tokens, notYetDone ) {
515 - // FIXME: store frame in object?
516 - this.emit('chunk', tokens);
517517 //tokens = this._transformPhase2( this.frame, tokens, this.parentCB );
518518 //console.log('AsyncTokenTransformManager._returnTokens, after _transformPhase2.');
519519
520 - //this.emit( 'chunk', tokens );
 520+ this.emit( 'chunk', tokens );
521521
522522 if ( ! notYetDone ) {
523 - console.log('AsyncTokenTransformManager._returnTokens done.');
 523+ //console.log('AsyncTokenTransformManager._returnTokens done. tokens:' +
 524+ // JSON.stringify( tokens, null, 2 ) + ', listeners: ' +
 525+ // JSON.stringify( this.listeners( 'chunk' ), null, 2 ) );
524526 // signal our done-ness to consumers.
525527 this.emit( 'end' );
526528 // and reset internal state.
——@@ -557,9 +559,10 @@
558560 * @constructor
559561 * @param {Object} environment.
560562 */
561 -function SyncTokenTransformManager ( env ) {
 563+function SyncTokenTransformManager ( env, phaseEndRank ) {
562564 // both inherited
563565 this._construct();
 566+ this.phaseEndRank = phaseEndRank;
564567 this.args = {}; // no arguments at the top level
565568 this.env = env;
566569 }
——@@ -586,8 +589,9 @@
587590 * @param {Array} Token chunk.
588591 */
589592 SyncTokenTransformManager.prototype.onChunk = function ( tokens ) {
 593+ //console.log('SyncTokenTransformManager.onChunk: ' +
 594+ // JSON.stringify( tokens, null, 2 ) );
590595 var res,
591 - phaseEndRank = 3,
592596 localAccum = [],
593597 localAccumLength = 0,
594598 tokensLength = tokens.length,
——@@ -603,23 +607,23 @@
604608 case 'TAG':
605609 case 'ENDTAG':
606610 case 'SELFCLOSINGTAG':
607 - res = this._transformTagToken( token, cb, phaseEndRank );
 611+ res = this._transformTagToken( token, cb, this.phaseEndRank );
608612 break;
609613 case 'TEXT':
610 - res = this._transformToken( token, cb, phaseEndRank,
 614+ res = this._transformToken( token, cb, this.phaseEndRank,
611615 ts.text );
612616 break;
613617 case 'COMMENT':
614 - res = this._transformToken( token, cb, phaseEndRank, ts.comment );
 618+ res = this._transformToken( token, cb, this.phaseEndRank, ts.comment );
615619 break;
616620 case 'NEWLINE':
617 - res = this._transformToken( token, cb, phaseEndRank, ts.newline );
 621+ res = this._transformToken( token, cb, this.phaseEndRank, ts.newline );
618622 break;
619623 case 'END':
620 - res = this._transformToken( token, cb, phaseEndRank, ts.end );
 624+ res = this._transformToken( token, cb, this.phaseEndRank, ts.end );
621625 break;
622626 default:
623 - res = this._transformToken( token, cb, phaseEndRank, ts.martian );
 627+ res = this._transformToken( token, cb, this.phaseEndRank, ts.martian );
624628 break;
625629 }
626630
——@@ -630,7 +634,7 @@
631635 tokensLength = tokens.length;
632636 i--; // continue at first inserted token
633637 } else if ( res.token ) {
634 - if ( res.token.rank === phaseEndRank ) {
 638+ if ( res.token.rank === this.phaseEndRank ) {
635639 // token is done.
636640 localAccum.push(res.token);
637641 this.prevToken = res.token;
——@@ -684,6 +688,7 @@
685689 //this.pipe.process(
686690 var pipe,
687691 ref;
 692+ //console.log( 'AttributeTransformManager.process: ' + JSON.stringify( attributes ) );
688693
689694 // transform each argument (key and value), and handle asynchronous returns
690695 for ( var i = 0, l = attributes.length; i < l; i++ ) {
——@@ -711,6 +716,7 @@
712717 pipe.addListener( 'end',
713718 this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
714719 );
 720+ //console.log('starting attribute transform of ' + JSON.stringify( attributes[i][1] ) );
715721 pipe.process( attributes[i][1] );
716722 }
717723 };
——@@ -724,21 +730,22 @@
725731 }
726732
727733 // and call the callback with the result
 734+ //console.log('AttributeTransformManager._returnAttributes: ' + out );
728735 this.callback( out );
729736 };
730737
731738 /**
732739 * Collect chunks returned from the pipeline
733740 */
734 -AttributeTransformManager.prototype.onChunk = function ( callback, ref, chunk ) {
735 - callback.call( ref, chunk, true );
 741+AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) {
 742+ cb( chunk, true );
736743 };
737744
738745 /**
739746 * Empty the pipeline by returning to the parent
740747 */
741 -AttributeTransformManager.prototype.onEnd = function ( callback, ref ) {
742 - callback.call( ref, [], false );
 748+AttributeTransformManager.prototype.onEnd = function ( cb ) {
 749+ cb( [], false );
743750 };
744751
745752
——@@ -746,7 +753,8 @@
747754 * Callback for async argument value expansions
748755 */
749756 AttributeTransformManager.prototype._returnAttributeValue = function ( ref, tokens, notYetDone ) {
750 - this.kvs[ref].value.push( tokens );
 757+ //console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens ) );
 758+ this.kvs[ref].value = this.kvs[ref].value.concat( tokens );
751759 if ( ! notYetDone ) {
752760 this.outstanding--;
753761 if ( this.outstanding === 0 ) {
——@@ -760,7 +768,7 @@
761769 * Callback for async argument key expansions
762770 */
763771 AttributeTransformManager.prototype._returnAttributeKey = function ( ref, tokens, notYetDone ) {
764 - this.kvs[ref].key.push( tokens );
 772+ this.kvs[ref].key = this.kvs[ref].key.concat( tokens );
765773 if ( ! notYetDone ) {
766774 this.outstanding--;
767775 if ( this.outstanding === 0 ) {
——@@ -818,32 +826,38 @@
819827 this.outstanding--;
820828 }
821829
 830+ //console.log( 'TokenAccumulator._returnTokens' );
822831 if ( reference === 'child' ) {
 832+ tokens = tokens.concat( this.accum );
 833+ //console.log('TokenAccumulator._returnTokens: ' +
 834+ // JSON.stringify( tokens, null, 2 )
 835+ // );
 836+ this.accum = [];
823837 // XXX: Use some marker to avoid re-transforming token chunks several
824838 // times?
825 - res = this.transformTokens( tokens, this.parentCB );
 839+ //res = this.transformTokens( tokens, this.parentCB );
826840
827 - if ( res.async ) {
828 - // new asynchronous expansion started, chain of accumulators
829 - // created
830 - if ( this.outstanding === 0 ) {
831 - // Last accum in chain should only wait for child
832 - res.async.outstanding--;
833 - cb = this.parentCB;
834 - } else {
835 - cb = this.parentCB;
836 - // set own callback to new sibling, the end of accumulator chain
837 - this.parentCB = res.async.getParentCB( 'sibling' );
838 - }
839 - }
840 - if ( ! notYetDone ) {
841 - // Child is done, return accumulator from sibling. Siblings
842 - // process tokens themselves, so we concat those to the result of
843 - // processing tokens from the child.
844 - tokens = res.tokens.concat( this.accum );
845 - this.accum = [];
846 - }
847 - this.cb( res.tokens, res.async );
 841+ //if ( res.async ) {
 842+ // // new asynchronous expansion started, chain of accumulators
 843+ // // created
 844+ // if ( this.outstanding === 0 ) {
 845+ // // Last accum in chain should only wait for child
 846+ // res.async.outstanding--;
 847+ // cb = this.parentCB;
 848+ // } else {
 849+ // cb = this.parentCB;
 850+ // // set own callback to new sibling, the end of accumulator chain
 851+ // this.parentCB = res.async.getParentCB( 'sibling' );
 852+ // }
 853+ //}
 854+ //if ( ! notYetDone ) {
 855+ // // Child is done, return accumulator from sibling. Siblings
 856+ // // process tokens themselves, so we concat those to the result of
 857+ // // processing tokens from the child.
 858+ // tokens = res.tokens.concat( this.accum );
 859+ // this.accum = [];
 860+ //}
 861+ this.parentCB( tokens, false );
848862 return null;
849863 } else {
850864 // sibling
——@@ -869,7 +883,8 @@
870884 * Mark the sibling as done (normally at the tail of a chain).
871885 */
872886 TokenAccumulator.prototype.siblingDone = function () {
873 - this._returnTokens01 ( 'sibling', [], false );
 887+ //console.log( 'TokenAccumulator.siblingDone: ' );
 888+ this._returnTokens ( 'sibling', [], false );
874889 };
875890
876891
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
——@@ -151,7 +151,7 @@
152152 );
153153
154154 // Register template expansion extension
155 - //new TemplateHandler( tokenExpander );
 155+ new TemplateHandler( tokenExpander );
156156
157157 tokenExpander.listenForTokensFrom ( tokenPreProcessor );
158158 // XXX: hack.
——@@ -194,7 +194,7 @@
195195 },
196196 args, this.env
197197 );
198 - //new TemplateHandler( tokenExpander );
 198+ new TemplateHandler( tokenExpander );
199199 tokenExpander.listenForTokensFrom ( tokenPreProcessor );
200200
201201 return new CachedTokenPipeline(
——@@ -250,12 +250,16 @@
251251 /************************ CachedTokenPipeline ********************************/
252252
253253 /**
254 - * Manage a part of a pipeline, that emits 'end' and 'chunk' events from its
255 - * last stage.
 254+ * Wrap a part of a pipeline. The last member of the pipeline is supposed to
 255+ * emit 'end' and 'chunk' events, while the first is supposed to support a
 256+ * process() method that sets the pipeline in motion.
256257 *
257258 * @class
258259 * @constructor
259 - * @param {
 260+ * @param {Function} returnToCacheCB: Callback to return the
 261+ * CachedTokenPipeline to a cache when processing has finished
 262+ * @param {Object} first: First stage of the pipeline
 263+ * @param {Object} last: Last stage of the pipeline
260264 */
261265 function CachedTokenPipeline ( returnToCacheCB, first, last ) {
262266 this.returnToCacheCB = returnToCacheCB;
——@@ -283,6 +287,10 @@
284288 * Forward chunks to our listeners
285289 */
286290 CachedTokenPipeline.prototype.forwardChunk = function ( chunk ) {
 291+ //console.log( 'CachedTokenPipeline.forwardChunk: ' +
 292+ // JSON.stringify( chunk, null, 2 )
 293+ // );
 294+
287295 this.emit( 'chunk', chunk );
288296 };
289297
——@@ -292,6 +300,8 @@
293301 * the given pipeline stage and returns it to a cache.
294302 */
295303 CachedTokenPipeline.prototype.forwardEndAndRecycleSelf = function ( ) {
 304+ //console.log( 'CachedTokenPipeline.forwardEndAndRecycleSelf: ' +
 305+ // JSON.stringify( this.listeners( 'chunk' ), null, 2 ) );
296306 // first, forward the event
297307 this.emit( 'end' );
298308 // now recycle self
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
——@@ -19,7 +19,7 @@
2020 }
2121
2222 TemplateHandler.prototype.reset = function () {
23 - this.outstanding = 0;
 23+ this.resultTokens = [];
2424 };
2525
2626 // constants
——@@ -47,7 +47,9 @@
4848 * processes the template.
4949 */
5050 TemplateHandler.prototype.onTemplate = function ( token, cb ) {
51 - console.log('onTemplate!');
 51+ //console.log('onTemplate!');
 52+
 53+ this.parentCB = cb;
5254
5355 // check for 'subst:'
5456 // check for variable magic names
——@@ -58,7 +60,7 @@
5961 var templateTokenTransformData = {
6062 args: {},
6163 manager: this.manager,
62 - outstanding: 0,
 64+ outstanding: 1, // Avoid premature finish
6365 cb: cb,
6466 origToken: token,
6567 isAsync: false
——@@ -69,14 +71,18 @@
7072 res,
7173 kv;
7274
73 - var attributes = [['', token.target]].concat( token.orderedArgs );
 75+ var attributes = [[{ type: 'TEXT', value: '' } , token.target ]].concat( token.orderedArgs );
7476
75 - new AttributeTransformManager( this.manager,
76 - this._returnAttributes.bind( this, templateTokenTransformData )
 77+ //console.log( 'before AttributeTransformManager: ' + JSON.stringify( attributes, null, 2 ) );
 78+ new AttributeTransformManager(
 79+ this.manager,
 80+ this._returnAttributes.bind( this, templateTokenTransformData )
7781 ).process( attributes );
7882
 83+ // Unblock finish
 84+ templateTokenTransformData.outstanding--;
7985 if ( templateTokenTransformData.outstanding === 0 ) {
80 - console.log( 'direct call');
 86+ //console.log( 'direct call');
8187 return this._expandTemplate ( templateTokenTransformData );
8288 } else {
8389 templateTokenTransformData.isAsync = true;
——@@ -85,7 +91,7 @@
8692 };
8793
8894 TemplateHandler.prototype._returnAttributes = function ( templateTokenTransformData, attributes ) {
89 - console.log( '_returnAttributes' + JSON.stringify(attributes) );
 95+ //console.log( 'TemplateHandler._returnAttributes: ' + JSON.stringify(attributes) );
9096 // Remove the target from the attributes
9197 templateTokenTransformData.target = attributes[0][1];
9298 attributes.shift();
——@@ -100,8 +106,8 @@
101107 * target were expanded in frame.
102108 */
103109 TemplateHandler.prototype._expandTemplate = function ( templateTokenTransformData ) {
104 - console.log('TemplateHandler.expandTemplate: ' +
105 - JSON.stringify( templateTokenTransformData, null, 2 ) );
 110+ //console.log('TemplateHandler.expandTemplate: ' +
 111+ // JSON.stringify( templateTokenTransformData, null, 2 ) );
106112 // First, check the target for loops
107113 this.manager.loopCheck.check( templateTokenTransformData.target );
108114
——@@ -115,14 +121,16 @@
116122 templateTokenTransformData.expandedArgs
117123 );
118124
119 - // Hook up the AsyncTokenTransformManager output events to call back our
120 - // parentCB.
 125+ // Hook up the inputPipeline output events to call back our parentCB.
121126 this.inputPipeline.addListener( 'chunk', this._onChunk.bind ( this ) );
122127 this.inputPipeline.addListener( 'end', this._onEnd.bind ( this ) );
123128
124129
125130 // Resolve a possibly relative link
126 - var templateName = this.manager.env.resolveTitle( this.manager.env.tokensToString( this.target ), 'Template' );
 131+ var templateName = this.manager.env.resolveTitle(
 132+ this.manager.env.tokensToString( templateTokenTransformData.target ),
 133+ 'Template'
 134+ );
127135 this._fetchTemplateAndTitle( templateName, this._processTemplateAndTitle.bind( this ) );
128136
129137 // Set up a pipeline:
——@@ -144,6 +152,13 @@
145153 // recursion depth check
146154 // fetch from DB or interwiki
147155 // infinte loop check
 156+
 157+ // Always asynchronous..
 158+ if ( this.isAsync ) {
 159+ return {};
 160+ } else {
 161+ return this.result;
 162+ }
148163 };
149164
150165
——@@ -161,14 +176,27 @@
162177 TemplateHandler.prototype._onEnd = function( ) {
163178 // Encapsulate the template in a single token, which contains all the
164179 // information needed for the editor.
165 - var res = {
166 - type: 'container',
167 - tokens: this.resultTokens, // The editor needs HTML serialization instead
168 - args: this.manager.args, // Here, the editor needs wikitext.
169 - attribs: this.origToken.attribs // Hmm..
 180+ var res =
 181+ [{
 182+ type: 'TAG',
 183+ name: 'div',
 184+ attribs: [['data-source', 'template']],
 185+ args: this.manager.args // Here, the editor needs wikitext.
 186+ }].concat(
 187+ // XXX: Mark source in attribute on result tokens, so that
 188+ // the visual editor can detect structures from templates!
 189+ this.resultTokens,
 190+ [{ type: 'ENDTAG', name: 'div' }]
 191+ );
 192+ //console.log( 'TemplateHandler._onEnd: ' + JSON.stringify( res, null, 2 ) );
170193
171 - };
172 - this.parentCB( res, false );
 194+ if ( this.isAsync ) {
 195+ this.parentCB( res, false );
 196+ this.reset();
 197+ } else {
 198+ this.result = { tokens: res };
 199+ this.reset();
 200+ }
173201 };
174202
175203
——@@ -178,6 +206,7 @@
179207 */
180208 TemplateHandler.prototype._processTemplateAndTitle = function( src, title ) {
181209 // Feed the pipeline. XXX: Support different formats.
 210+ //console.log( 'TemplateHandler._processTemplateAndTitle: ' + src );
182211 this.inputPipeline.process ( src );
183212 };
184213
——@@ -188,13 +217,13 @@
189218 */
190219 TemplateHandler.prototype._fetchTemplateAndTitle = function( title, callback ) {
191220 // @fixme normalize name?
192 - if (title in this.pageCache) {
 221+ if (title in this.manager.env.pageCache) {
193222 // @fixme should this be forced to run on next event?
194 - callback( this.pageCache[title], title );
 223+ callback( this.env.manager.pageCache[title], title );
195224 } else {
196225 // whee fun hack!
197 - console.log(title);
198 - console.log(this.pageCache);
 226+ //console.log(title);
 227+ //console.log(this.manager.env.pageCache);
199228 $.ajax({
200229 url: this.manager.env.wgScriptPath + '/api' + this.manager.env.wgScriptExtension,
201230 data: {
——@@ -202,7 +231,7 @@
203232 action: 'query',
204233 prop: 'revisions',
205234 rvprop: 'content',
206 - titles: name
 235+ titles: title
207236 },
208237 success: function(data, xhr) {
209238 var src = null, title = null;
——@@ -213,13 +242,15 @@
214243 }
215244 });
216245 if (typeof src !== 'string') {
217 - callback(null, null, 'Page not found');
 246+ //console.log( 'Page ' + title + 'not found!' );
 247+ callback( 'Page ' + title + ' not found' );
218248 } else {
219249 callback(src, title);
220250 }
221251 },
222252 error: function(msg) {
223 - callback(null, null, 'Page/template fetch failure');
 253+ //console.log( 'Page/template fetch failure for title ' + title );
 254+ callback('Page/template fetch failure for title ' + title);
224255 },
225256 dataType: 'json',
226257 cache: false // @fixme caching, versions etc?

Status & tagging log