r109653 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r109652‎ | r109653 | r109654 >
Date:22:02, 20 January 2012
Author:gwicke
Status:deferred
Tags:
Comment:
* NoInclude and IncludeOnly improvements
* Tokenizer support for templates and template args in template arguments and titles
* Async attribute expansion fixes
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js (added) (history)
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js (deleted) (history)
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/ext.core.NoOnlyInclude.js
——@@ -1,39 +0,0 @@
2 -/**
3 - * Simple noinclude / onlyinclude implementation. Strips all tokens in
4 - * noinclude sections.
5 - *
6 - * @author Gabriel Wicke <gwicke@wikimedia.org>
7 - */
8 -
9 -var TokenCollector = require( './ext.util.TokenCollector.js' ).TokenCollector;
10 -
11 -function NoInclude( manager ) {
12 - new TokenCollector(
13 - manager,
14 - function ( tokens ) {
15 - manager.env.dp( 'noinclude stripping', tokens );
16 - return {};
17 - }, // just strip it all..
18 - true, // match the end-of-input if </noinclude> is missing
19 - 0.01, // very early in stage 1, to avoid any further processing.
20 - 'tag',
21 - 'noinclude'
22 - );
23 -}
24 -
25 -function OnlyInclude( manager ) {
26 - new TokenCollector(
27 - manager,
28 - function ( ) { return {} }, // just strip it all..
29 - true, // match the end-of-input if </noinclude> is missing
30 - 0.01, // very early in stage 1, to avoid any further processing.
31 - 'tag',
32 - 'onlyinclude'
33 - );
34 -}
35 -
36 -
37 -if (typeof module == "object") {
38 - module.exports.NoInclude = NoInclude;
39 - module.exports.OnlyInclude = OnlyInclude;
40 -}
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
——@@ -0,0 +1,59 @@
 2+/**
 3+ * Simple noinclude / onlyinclude implementation. Strips all tokens in
 4+ * noinclude sections.
 5+ *
 6+ * @author Gabriel Wicke <gwicke@wikimedia.org>
 7+ */
 8+
 9+var TokenCollector = require( './ext.util.TokenCollector.js' ).TokenCollector;
 10+
 11+function NoInclude( manager, isInclude ) {
 12+ new TokenCollector(
 13+ manager,
 14+ function ( tokens ) {
 15+ if ( isInclude ) {
 16+ manager.env.dp( 'noinclude stripping', tokens );
 17+ return {};
 18+ } else {
 19+ tokens.shift();
 20+ if ( tokens.length &&
 21+ tokens[tokens.length - 1].type !== 'END' ) {
 22+ tokens.pop();
 23+ }
 24+ return { tokens: tokens };
 25+ }
 26+ }, // just strip it all..
 27+ true, // match the end-of-input if </noinclude> is missing
 28+ 0.01, // very early in stage 1, to avoid any further processing.
 29+ 'tag',
 30+ 'noinclude'
 31+ );
 32+}
 33+
 34+function IncludeOnly( manager, isInclude ) {
 35+ new TokenCollector(
 36+ manager,
 37+ function ( tokens ) {
 38+ if ( isInclude ) {
 39+ tokens.shift();
 40+ if ( tokens.length &&
 41+ tokens[tokens.length - 1].type !== 'END' ) {
 42+ tokens.pop();
 43+ }
 44+ return { tokens: tokens };
 45+ } else {
 46+ return {};
 47+ }
 48+ },
 49+ true, // match the end-of-input if </noinclude> is missing
 50+ 0.01, // very early in stage 1, to avoid any further processing.
 51+ 'tag',
 52+ 'includeonly'
 53+ );
 54+}
 55+
 56+
 57+if (typeof module == "object") {
 58+ module.exports.NoInclude = NoInclude;
 59+ module.exports.IncludeOnly = IncludeOnly;
 60+}
Property changes on: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js
___________________________________________________________________
Added: svn:eol-style
161 + native
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
——@@ -664,7 +664,7 @@
665665 }
666666 }
667667 this.env.dp( 'SyncTokenTransformManager.onChunk: emitting ' +
668 - JSON.stringify( localAccum ) );
 668+ JSON.stringify( localAccum, null, 2 ) );
669669 this.emit( 'chunk', localAccum );
670670 };
671671
——@@ -698,7 +698,7 @@
699699 function AttributeTransformManager ( manager, callback ) {
700700 this.manager = manager;
701701 this.callback = callback;
702 - this.outstanding = 0;
 702+ this.outstanding = 1;
703703 this.kvs = [];
704704 //this.pipe = manager.getAttributePipeline( manager.args );
705705 }
——@@ -726,7 +726,7 @@
727727 pipe.addListener( 'end',
728728 this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) )
729729 );
730 - pipe.process( attributes[i][0] );
 730+ pipe.process( attributes[i][0].concat([{type:'END'}]) );
731731
732732 // transform the value
733733 pipe = this.manager.getAttributePipeline( this.manager.args );
——@@ -737,8 +737,12 @@
738738 this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) )
739739 );
740740 //console.log('starting attribute transform of ' + JSON.stringify( attributes[i][1] ) );
741 - pipe.process( attributes[i][1] );
 741+ pipe.process( attributes[i][1].concat([{type:'END'}]) );
742742 }
 743+ this.outstanding--;
 744+ if ( this.outstanding == 0 ) {
 745+ this._returnAttributes();
 746+ }
743747 };
744748
745749 AttributeTransformManager.prototype._returnAttributes = function ( ) {
——@@ -758,6 +762,9 @@
759763 * Collect chunks returned from the pipeline
760764 */
761765 AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) {
 766+ if ( chunk.length && chunk[chunk.length - 1].type === 'END' ) {
 767+ chunk.pop();
 768+ }
762769 cb( chunk, true );
763770 };
764771
——@@ -773,7 +780,8 @@
774781 * Callback for async argument value expansions
775782 */
776783 AttributeTransformManager.prototype._returnAttributeValue = function ( ref, tokens, notYetDone ) {
777 - //console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens ) );
 784+ //console.log( 'check _returnAttributeValue: ' + JSON.stringify( tokens ) +
 785+ // ' notYetDone:' + notYetDone );
778786 this.kvs[ref].value = this.kvs[ref].value.concat( tokens );
779787 if ( ! notYetDone ) {
780788 this.outstanding--;
——@@ -788,6 +796,8 @@
789797 * Callback for async argument key expansions
790798 */
791799 AttributeTransformManager.prototype._returnAttributeKey = function ( ref, tokens, notYetDone ) {
 800+ //console.log( 'check _returnAttributeKey: ' + JSON.stringify( tokens ) +
 801+ // ' notYetDone:' + notYetDone );
792802 this.kvs[ref].key = this.kvs[ref].key.concat( tokens );
793803 if ( ! notYetDone ) {
794804 this.outstanding--;
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
——@@ -480,7 +480,11 @@
481481 / & { return syntaxFlags['extlink']; } "]" { return true; }
482482 / & { return syntaxFlags['linkdesc']; } link_end { return true; }
483483 / & { return syntaxFlags['h']; } '='+ space* newline { return true; }
484 - / & { return syntaxFlags['template']; } ('|' / '}}') { return true; }
 484+ / & { return syntaxFlags['template']; } ('|' / '}}' ) { return true; }
 485+ / & { return syntaxFlags['equal']; } '=' {
 486+ //console.log( 'equal stop!' );
 487+ return true;
 488+ }
485489
486490 inline
487491 = c:(urltext / (! inline_breaks (inline_element / . )))+ {
——@@ -539,6 +543,7 @@
540544 = //& { dp('inline_element enter' + input.substr(pos, 10)); return true; }
541545 & '<' ( comment / xmlish_tag )
542546 / & '{' ( & '{{{{{' template / tplarg / template )
 547+ /// & '{' ( tplarg / template )
543548 // Eat three opening brackets as text.
544549 / '[[[' { return { type: 'TEXT', value: '[[[' } }
545550 / & '[' ( wikilink / extlink )
——@@ -711,25 +716,26 @@
712717 name: 'template',
713718 attribs: [['data-target', JSON.stringify(target)]],
714719 orderedArgs: params,
715 - args: {},
 720+ //args: {},
716721 target: target
717722 };
718 - if (params && params.length) {
719 - var position = 1;
720 - for ( var i = 0, l = params.length; i < l; i++ ) {
721 - var param = params[i];
722 - if ( param[0] === null ) {
723 - obj.args[position] = param[1];
724 - position++;
725 - } else {
726 - // Last value wins for duplicates.
727 - obj.args[param[0]] = param[1];
728 - }
729 - }
730 - // HACK: temporarily also push the args into an attribute
731 - // (just for debugging)
732 - obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
733 - }
 723+ // XXX: this is kind of broken, as arg keys need to be expanded
 724+ //if (params && params.length) {
 725+ // var position = 1;
 726+ // for ( var i = 0, l = params.length; i < l; i++ ) {
 727+ // var param = params[i];
 728+ // if ( param[0] === null ) {
 729+ // obj.args[position] = param[1];
 730+ // position++;
 731+ // } else {
 732+ // // Last value wins for duplicates.
 733+ // obj.args[param[0]] = param[1];
 734+ // }
 735+ // }
 736+ // // HACK: temporarily also push the args into an attribute
 737+ // // (just for debugging)
 738+ // obj.attribs.push(['data-json-args', JSON.stringify(obj.args)]);
 739+ //}
734740 // Should actually use a self-closing tag here, but the Node HTML5
735741 // parser only recognizes known self-closing tags for now, so use an
736742 // explicit end tag for now.
——@@ -738,18 +744,14 @@
739745 }
740746
741747 // XXX: support template and args in target!
742 -template_target
743 - = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }
 748+//template_target
 749+// = h:( !"}}" x:([^|\n]) { return x } )* { return { type: 'TEXT', value: h.join('') } }
744750
745 -template_param
746 - = name:template_param_name space* "=" space* c:template_param_text {
747 - return [[{ type: 'TEXT', value: name }], flatten( c )];
748 - } / c:template_param_text {
749 - return [[], flatten( c ) ];
750 - }
751 -
752751 tplarg
753 - = "{{{" name:template_param_text params:("|" p:template_param { return p })* "}}}" {
 752+ = "{{{"
 753+ name:template_param_text
 754+ params:( newline? "|" newline? p:template_param { return p })*
 755+ "}}}" {
754756 name = flatten( name );
755757 var obj = {
756758 type: 'SELFCLOSINGTAG',
——@@ -764,21 +766,39 @@
765767 obj.attribs.push(['data-json-args', JSON.stringify(params)]);
766768 obj.defaultvalue = params[0][1];
767769 }
768 - //console.log( 'tokenizer templatearg ' + JSON.stringify( obj ));
 770+ console.log( 'tokenizer tplarg ' + JSON.stringify( obj ));
769771 return obj;
770772 }
771773
 774+template_param
 775+ = name:template_param_name space* "=" space* c:template_param_text {
 776+ //console.log( 'named template_param matched' + pp([name, flatten( c )]) );
 777+ return [name, flatten( c )];
 778+ } / c:template_param_text {
 779+ return [[], flatten( c ) ];
 780+ }
 781+
 782+
772783 // FIXME: handle template args and templates in key! (or even parser functions?)
773784 template_param_name
774 - = h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
 785+ = & { return setFlag( 'equal' ) }
 786+ tpt:template_param_text
 787+ & { clearFlag( 'equal' ); return true; }
 788+ {
 789+ //console.log( 'template param name matched: ' + pp( tpt ) );
 790+ return tpt;
 791+ }
775792
 793+ / & { return clearFlag( 'equal' ) }
 794+ //= h:( !"}}" x:([^=|\n]) { return x } )* { return h.join(''); }
 795+
776796 template_param_text
777797 = & { return setFlag('template') }
778 - il:inline+ {
 798+ il:inline {
779799 clearFlag('template');
780800 return il;
781801 }
782 - / & { clearFlag('template'); return false; }
 802+ / & { return clearFlag('template'); }
783803
784804 // TODO: handle link prefixes as in al[[Razi]]
785805 wikilink
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js
——@@ -16,9 +16,9 @@
1717 PegTokenizer = require('./mediawiki.tokenizer.peg.js').PegTokenizer,
1818 TokenTransformManager = require('./mediawiki.TokenTransformManager.js'),
1919
20 - NoOnlyInclude = require('./ext.core.NoOnlyInclude.js'),
21 - OnlyInclude = NoOnlyInclude.OnlyInclude,
22 - NoInclude = NoOnlyInclude.NoInclude,
 20+ NoIncludeOnly = require('./ext.core.NoIncludeOnly.js'),
 21+ IncludeOnly = NoIncludeOnly.IncludeOnly,
 22+ NoInclude = NoIncludeOnly.NoInclude,
2323 QuoteTransformer = require('./ext.core.QuoteTransformer.js').QuoteTransformer,
2424 PostExpandParagraphHandler = require('./ext.core.PostExpandParagraphHandler.js')
2525 .PostExpandParagraphHandler,
——@@ -65,7 +65,7 @@
6666 };
6767
6868 // Create an input pipeline for the given input type.
69 - this.inputPipeline = this.makeInputPipeline ( inputType );
 69+ this.inputPipeline = this.makeInputPipeline ( inputType, {}, true );
7070 this.inputPipeline.atTopLevel = true;
7171
7272
——@@ -137,7 +137,7 @@
138138 * accepts the wiki text this way. The last stage of the input pipeline is
139139 * always an AsyncTokenTransformManager, which emits its output in events.
140140 */
141 -ParserPipeline.prototype.makeInputPipeline = function ( inputType, args ) {
 141+ParserPipeline.prototype.makeInputPipeline = function ( inputType, args, isNoInclude ) {
142142 switch ( inputType ) {
143143 case 'text/wiki':
144144 //console.log( 'makeInputPipeline ' + JSON.stringify( args ) );
——@@ -158,8 +158,9 @@
159159 var tokenPreProcessor = new TokenTransformManager.SyncTokenTransformManager ( this.env );
160160 tokenPreProcessor.listenForTokensFrom ( wikiTokenizer );
161161
 162+ new IncludeOnly( tokenPreProcessor, ! isNoInclude );
162163 // Add noinclude transform for now
163 - new NoInclude( tokenPreProcessor );
 164+ new NoInclude( tokenPreProcessor, ! isNoInclude );
164165
165166 var tokenExpander = new TokenTransformManager.AsyncTokenTransformManager (
166167 {
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
——@@ -75,8 +75,8 @@
7676 var attributes = [[[{ type: 'TEXT', value: '' }] , token.target ]]
7777 .concat( this._nameArgs( token.orderedArgs ) );
7878
79 - //console.log( 'before AttributeTransformManager: ' +
80 - // JSON.stringify( attributes, null, 2 ) );
 79+ this.manager.env.dp( 'before AttributeTransformManager: ' +
 80+ JSON.stringify( attributes, null, 2 ) );
8181 new AttributeTransformManager(
8282 this.manager,
8383 this._returnAttributes.bind( this, tplExpandData )
——@@ -85,7 +85,10 @@
8686 // Unblock finish
8787 if ( ! tplExpandData.attribsAsync ) {
8888 // Attributes were transformed synchronously
89 - this.manager.env.dp( 'sync attribs for ' + JSON.stringify( token ));
 89+ this.manager.env.dp (
 90+ 'sync attribs for ' + JSON.stringify( tplExpandData.target ),
 91+ tplExpandData.expandedArgs
 92+ );
9093 // All attributes are fully expanded synchronously (no IO was needed)
9194 return this._expandTemplate ( tplExpandData );
9295 } else {
——@@ -110,7 +113,7 @@
111114 out.push( orderedArgs[i] );
112115 }
113116 }
114 - //console.log( '_nameArgs: ' + JSON.stringify( out ) );
 117+ this.manager.env.dp( '_nameArgs: ' + JSON.stringify( out ) );
115118 return out;
116119 };
117120
——@@ -190,6 +193,7 @@
191194 this.manager.env.KVtoHash( tplExpandData.expandedArgs ),
192195 tplExpandData.target
193196 );
 197+ this.manager.env.dp( 'argHash:', this.manager.env.KVtoHash( tplExpandData.expandedArgs ) );
194198
195199 // Hook up the inputPipeline output events to our handlers
196200 inputPipeline.addListener( 'chunk', this._onChunk.bind ( this, tplExpandData ) );
——@@ -359,8 +363,8 @@
360364 // ' vs. ' + JSON.stringify( this.manager.args ) );
361365 res = this.manager.args[argName];
362366 } else {
363 - //console.log( 'templateArg not found: ' + argName +
364 - // ' vs. ' + JSON.stringify( this.manager.args ) );
 367+ console.log( 'templateArg not found: ' + argName +
 368+ ' vs. ' + JSON.stringify( this.manager.args ) );
365369 if ( token.attribs.length > 1 ) {
366370 res = defaultValue;
367371 } else {

Status & tagging log