r109691 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r109690‎ | r109691 | r109692 >
Date:20:38, 21 January 2012
Author:gwicke
Status:deferred (Comments)
Tags:
Comment:
Implement a few parser functions. 220 parser tests now passing.
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js (added) (history)
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/parse.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js
——@@ -444,7 +444,7 @@
445445 phaseEndRank = 2, // XXX: parametrize!
446446 // Prepare a new accumulator, to be used by async children (if any)
447447 localAccum = [],
448 - accum = new TokenAccumulator( parentCB ),
 448+ accum = new TokenAccumulator( this, parentCB ),
449449 cb = accum.getParentCB( 'child' ),
450450 activeAccum = null,
451451 tokensLength = tokens.length,
——@@ -506,7 +506,7 @@
507507 // The child now switched to activeAccum, we have to create a new
508508 // accumulator for the next potential child.
509509 activeAccum = accum;
510 - accum = new TokenAccumulator( activeAccum.getParentCB( 'sibling' ) );
 510+ accum = new TokenAccumulator( this, activeAccum.getParentCB( 'sibling' ) );
511511 cb = accum.getParentCB( 'child' );
512512 }
513513 }
——@@ -831,7 +831,8 @@
832832 * @param {Object} next TokenAccumulator to link to
833833 * @param {Array} (optional) tokens, init accumulator with tokens or []
834834 */
835 -function TokenAccumulator ( parentCB ) {
 835+function TokenAccumulator ( manager, parentCB ) {
 836+ this.manager = manager;
836837 this.parentCB = parentCB;
837838 this.accum = [];
838839 // Wait for child and sibling by default
——@@ -870,7 +871,7 @@
871872 //console.log( 'TokenAccumulator._returnTokens' );
872873 if ( reference === 'child' ) {
873874 tokens = tokens.concat( this.accum );
874 - console.log('TokenAccumulator._returnTokens child: ' +
 875+ this.manager.env.dp('TokenAccumulator._returnTokens child: ' +
875876 JSON.stringify( tokens, null, 2 ) +
876877 ' outstanding: ' + this.outstanding
877878 );
——@@ -883,12 +884,12 @@
884885 tokens = this.accum.concat( tokens );
885886 // A sibling will transform tokens, so we don't have to do this
886887 // again.
887 - console.log( 'TokenAccumulator._returnTokens: sibling done and parentCB ' +
 888+ this.manager.env.dp( 'TokenAccumulator._returnTokens: sibling done and parentCB ' +
888889 JSON.stringify( tokens ) );
889890 this.parentCB( tokens, false );
890891 return null;
891892 } else if ( this.outstanding === 1 && notYetDone ) {
892 - console.log( 'TokenAccumulator._returnTokens: sibling done and parentCB but notYetDone ' +
 893+ this.manager.env.dp( 'TokenAccumulator._returnTokens: sibling done and parentCB but notYetDone ' +
893894 JSON.stringify( tokens ) );
894895 // Sibling is not yet done, but child is. Return own parentCB to
895896 // allow the sibling to go direct, and call back parent with
——@@ -897,7 +898,7 @@
898899 return this.parentCB( tokens, true);
899900 } else {
900901 this.accum = this.accum.concat( tokens );
901 - console.log( 'TokenAccumulator._returnTokens: sibling done, but not overall. notYetDone=' +
 902+ this.manager.env.dp( 'TokenAccumulator._returnTokens: sibling done, but not overall. notYetDone=' +
902903 notYetDone + ', this.outstanding=' + this.outstanding +
903904 ', this.accum=' +
904905 JSON.stringify( this.accum, null, 2 ) );
——@@ -955,7 +956,7 @@
956957 */
957958 LoopAndDepthCheck.prototype.check = function ( title ) {
958959 // XXX: set limit really low for testing!
959 - if ( this.depth > 5 ) {
 960+ if ( this.depth > 40 ) {
960961 // too deep
961962 //console.log( 'Loopcheck: ' + JSON.stringify( this, null, 2 ) );
962963 return 'Expansion depth limit exceeded at ';
Index: trunk/extensions/VisualEditor/modules/parser/parse.js
——@@ -20,7 +20,7 @@
2121 wgScriptExtension: ".php",
2222 fetchTemplates: true,
2323 // enable/disable debug output using this switch
24 - debug: true
 24+ debug: false
2525 } ),
2626 parser = new ParserPipeline( env );
2727
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
——@@ -0,0 +1,145 @@
 2+/**
 3+ * Some parser functions
 4+ */
 5+
 6+function ParserFunctions ( manager ) {
 7+ this.manager = manager;
 8+}
 9+
 10+ParserFunctions.prototype.fun = {};
 11+
 12+ParserFunctions.prototype['pf_#if'] = function ( target, argList, argDict ) {
 13+ if ( target.trim() !== '' ) {
 14+ this.manager.env.dp('#if, first branch', argDict[1] );
 15+ return argDict[1] || [];
 16+ } else {
 17+ this.manager.env.dp('#if, second branch', argDict[2] );
 18+ return argDict[2] || [];
 19+ }
 20+};
 21+
 22+ParserFunctions.prototype['pf_#switch'] = function ( target, argList, argDict ) {
 23+ this.manager.env.dp( 'switch enter: ' + target.trim() +
 24+ ' looking in ', argDict );
 25+ if ( target.trim() in argDict ) {
 26+ this.manager.env.dp( 'switch found: ' + target.trim() +
 27+ ' res=', argDict[target.trim()] );
 28+ return argDict[target.trim()];
 29+ } else if ( '#default' in argDict ) {
 30+ return argDict['#default'];
 31+ } else {
 32+ var lastKV = argList[argList.length - 1];
 33+ if ( lastKV && ! lastKV[0].length ) {
 34+ return lastKV[1];
 35+ } else {
 36+ return [];
 37+ }
 38+ }
 39+};
 40+
 41+// #ifeq
 42+ParserFunctions.prototype['pf_#ifeq'] = function ( target, argList, argDict ) {
 43+ if ( ! argList.length ) {
 44+ return [];
 45+ } else {
 46+ if ( target.trim() === this.manager.env.tokensToString( argList[0][1] ).trim() ) {
 47+ return ( argList[1] && argList[1][1]) || [];
 48+ } else {
 49+ return ( argList[1] && argList[1][1]) || [];
 50+ }
 51+ }
 52+};
 53+
 54+ParserFunctions.prototype['pf_lc'] = function ( target, argList, argDict ) {
 55+ return [{type: 'TEXT', value: target.toLowerCase()}];
 56+};
 57+ParserFunctions.prototype['pf_uc'] = function ( target, argList, argDict ) {
 58+ return [{type: 'TEXT', value: target.toUpperCase()}];
 59+};
 60+ParserFunctions.prototype['pf_ucfirst'] = function ( target, argList, argDict ) {
 61+ if ( target ) {
 62+ return [{
 63+ type: 'TEXT',
 64+ value: target[0].toUpperCase() + target.substr(1)
 65+ }];
 66+ } else {
 67+ return [];
 68+ }
 69+};
 70+ParserFunctions.prototype['pf_lcfirst'] = function ( target, argList, argDict ) {
 71+ if ( target ) {
 72+ return [{
 73+ type: 'TEXT',
 74+ value: target[0].toLowerCase() + target.substr(1)
 75+ }];
 76+ } else {
 77+ return [];
 78+ }
 79+};
 80+
 81+ParserFunctions.prototype['pf_#tag'] = function ( target, argList, argDict ) {
 82+ return [{type: 'TAG', name: target, attribs: argList}];
 83+};
 84+
 85+// FIXME
 86+ParserFunctions.prototype['pf_#ifexpr'] = function ( target, argList, argDict ) {
 87+ return [];
 88+};
 89+ParserFunctions.prototype['pf_#iferror'] = function ( target, argList, argDict ) {
 90+ return [];
 91+};
 92+ParserFunctions.prototype['pf_#expr'] = function ( target, argList, argDict ) {
 93+ return [];
 94+};
 95+ParserFunctions.prototype['pf_#ifexist'] = function ( target, argList, argDict ) {
 96+ return [];
 97+};
 98+
 99+// pure fake..
 100+ParserFunctions.prototype['pf_formatnum'] = function ( target, argList, argDict ) {
 101+ return [{type: 'TEXT', value: target}];
 102+};
 103+ParserFunctions.prototype['pf_currentpage'] = function ( target, argList, argDict ) {
 104+ return [{type: 'TEXT', value: target}];
 105+};
 106+ParserFunctions.prototype['pf_pagename'] = function ( target, argList, argDict ) {
 107+ return [{type: 'TEXT', value: target}];
 108+};
 109+ParserFunctions.prototype['pf_pagename'] = function ( target, argList, argDict ) {
 110+ return [{type: 'TEXT', value: target}];
 111+};
 112+ParserFunctions.prototype['pf_fullpagename'] = function ( target, argList, argDict ) {
 113+ return [{type: 'TEXT', value: target}];
 114+};
 115+ParserFunctions.prototype['pf_fullpagenamee'] = function ( target, argList, argDict ) {
 116+ return [{type: 'TEXT', value: target}];
 117+};
 118+ParserFunctions.prototype['pf_fullurl'] = function ( target, argList, argDict ) {
 119+ return [{type: 'TEXT', value: target}];
 120+};
 121+ParserFunctions.prototype['pf_urlencode'] = function ( target, argList, argDict ) {
 122+ return [{type: 'TEXT', value: target}];
 123+};
 124+ParserFunctions.prototype['pf_namespace'] = function ( target, argList, argDict ) {
 125+ return [{type: 'TEXT', value: 'Main'}];
 126+};
 127+
 128+// FIXME! This is just fake.
 129+ParserFunctions.prototype['pf_#time'] = function ( target, argList, argDict ) {
 130+ return [{type: 'TEXT', value: new Date().toString()}];
 131+};
 132+
 133+// #time
 134+// #ifexp
 135+// PAGENAME
 136+// #expr
 137+// NAMESPACE
 138+// #iferror
 139+//
 140+
 141+//ParserFunctions.prototype['pf_FORMATNUM'] = function ( target, argList, argDict ) {
 142+//};
 143+
 144+if (typeof module == "object") {
 145+ module.exports.ParserFunctions = ParserFunctions;
 146+}
Property changes on: trunk/extensions/VisualEditor/modules/parser/ext.core.ParserFunctions.js
___________________________________________________________________
Added: svn:eol-style
1147 + native
Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt
——@@ -782,7 +782,7 @@
783783 } / c:template_param_text {
784784 return [[], flatten( c ) ];
785785 }
786 - / & '|' { return [[], []]; }
 786+ / & [|}] { return [[], []]; }
787787
788788
789789 // FIXME: handle template args and templates in key! (or even parser functions?)
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
——@@ -14,6 +14,7 @@
1515 request = require('request'),
1616 events = require('events'),
1717 qs = require('querystring'),
 18+ ParserFunctions = require('./ext.core.ParserFunctions.js').ParserFunctions,
1819 AttributeTransformManager = require('./mediawiki.TokenTransformManager.js')
1920 .AttributeTransformManager;
2021
——@@ -21,6 +22,7 @@
2223 function TemplateHandler ( manager ) {
2324 this.reset();
2425 this.register( manager );
 26+ this.parserFunctions = new ParserFunctions( manager );
2527 }
2628
2729 TemplateHandler.prototype.reset = function ( token ) {
——@@ -160,6 +162,30 @@
161163 var target = this.manager.env.normalizeTitle(
162164 this.manager.env.tokensToString( tplExpandData.target )
163165 );
 166+ var args = this.manager.env.KVtoHash( tplExpandData.expandedArgs );
 167+
 168+ this.manager.env.dp( 'argHash: ', args );
 169+
 170+ var prefix = target.split(':', 1)[0].toLowerCase();
 171+ if ( prefix && 'pf_' + prefix in this.parserFunctions ) {
 172+ var funcArg = target.substr( prefix.length + 1 );
 173+ this.manager.env.dp( 'entering prefix', funcArg, args );
 174+ var res = this.parserFunctions[ 'pf_' + prefix ]( funcArg,
 175+ tplExpandData.expandDone, args );
 176+
 177+ // XXX: support async parser functions!
 178+ if ( tplExpandData.overallAsync ) {
 179+ this.manager.env.dp( 'TemplateHandler._expandTemplate: calling back ' +
 180+ 'after parser func ' + prefix + ' with res:' + JSON.stringify( res ) );
 181+ return tplExpandData.cb( res, false );
 182+ } else {
 183+ this.manager.env.dp( 'TemplateHandler._expandTemplate: sync return ' +
 184+ 'after parser func ' + prefix + ' with res:' + JSON.stringify( res ) );
 185+ return { tokens: res };
 186+ //data.reset();
 187+ }
 188+ }
 189+
164190 var checkRes = this.manager.loopAndDepthCheck.check( target );
165191 if( checkRes ) {
166192 // Loop detected, abort!
——@@ -191,10 +217,9 @@
192218 // 'text/wiki' input and asynchronous stage-2 transforms).
193219 var inputPipeline = this.manager.newChildPipeline(
194220 this.manager.inputType || 'text/wiki',
195 - this.manager.env.KVtoHash( tplExpandData.expandedArgs ),
 221+ args,
196222 tplExpandData.target
197223 );
198 - this.manager.env.dp( 'argHash:', this.manager.env.KVtoHash( tplExpandData.expandedArgs ) );
199224
200225 // Hook up the inputPipeline output events to our handlers
201226 inputPipeline.addListener( 'chunk', this._onChunk.bind ( this, tplExpandData ) );
——@@ -301,7 +326,7 @@
302327 // Unwind the stack
303328 process.nextTick(
304329 function () {
305 - callback( self.manager.env.pageCache[title], title )
 330+ callback( self.manager.env.pageCache[title], title );
306331 }
307332 );
308333 } else if ( ! this.manager.env.fetchTemplates ) {
——@@ -314,7 +339,7 @@
315340 this.manager.env.dp( 'trying to fetch ' + title );
316341
317342 // Start a new request if none is outstanding
318 - this.manager.env.dp( 'requestQueue: ', this.manager.env.requestQueue)
 343+ this.manager.env.dp( 'requestQueue: ', this.manager.env.requestQueue);
319344 if ( this.manager.env.requestQueue[title] === undefined ) {
320345 this.manager.env.requestQueue[title] = new TemplateRequest( this.manager, title );
321346 }
——@@ -365,9 +390,9 @@
366391 // ' vs. ' + JSON.stringify( this.manager.args ) );
367392 res = this.manager.args[argName];
368393 } else {
369 - console.log( 'templateArg not found: ' + argName +
 394+ this.manager.env.dp( 'templateArg not found: ' + argName +
370395 ' vs. ' + JSON.stringify( this.manager.args ) );
371 - if ( false && defaultValue.length ) {
 396+ if ( defaultValue.length ) {
372397 res = defaultValue;
373398 } else {
374399 res = [{ type: 'TEXT', value: '{{{' + argName + '}}}' }];
——@@ -414,10 +439,12 @@
415440 manager.env.dp(error);
416441 self.emit('src', 'Page/template fetch failure for title ' + title, title);
417442 } else if(response.statusCode == 200) {
418 - var src = '';
 443+ var src = '',
 444+ data,
 445+ normalizedTitle;
419446 try {
420447 //console.log( 'body: ' + body );
421 - var data = JSON.parse( body );
 448+ data = JSON.parse( body );
422449 } catch(e) {
423450 console.log( "Error: while parsing result. Error was: " );
424451 console.log( e );
——@@ -426,13 +453,13 @@
427454 console.log( "------------------------------------------" );
428455 }
429456 try {
430 - $.each(data.query.pages, function(i, page) {
 457+ $.each( data.query.pages, function(i, page) {
431458 if (page.revisions && page.revisions.length) {
432459 src = page.revisions[0]['*'];
433 - title = page.title;
 460+ normalizeTitle = page.title;
434461 }
435462 });
436 - } catch ( e ) {
 463+ } catch ( e2 ) {
437464 console.log( 'Did not find page revisions in the returned body:' + body );
438465 src = '';
439466 }
——@@ -445,13 +472,15 @@
446473 // XXX: handle other status codes
447474
448475 // Remove self from request queue
 476+ manager.env.dp( 'trying to remove ' + title + ' from requestQueue' );
449477 delete manager.env.requestQueue[title];
 478+ manager.env.dp( 'after deletion:', manager.env.requestQueue );
450479 });
451480 }
452481
453482 /*
454 - * XXX: The jQuery version does not quite work with node, but we keep
455 - * it around for now.
 483+ * XXX: The jQuery version does not quite work with node, but we keep
 484+ * it around for now.
456485 $.ajax({
457486 url: url,
458487 data: {
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
——@@ -116,7 +116,7 @@
117117 var token = tokens[i];
118118 if ( ! token ) {
119119 console.trace();
120 - console.log( 'MWParserEnvironment.tokensToString, invalid token: ' +
 120+ this.dp( 'MWParserEnvironment.tokensToString, invalid token: ' +
121121 JSON.stringify( token ) );
122122 continue;
123123 }
——@@ -126,7 +126,7 @@
127127 // strip comments and newlines
128128 } else {
129129 var tstring = JSON.stringify( token );
130 - console.log ( 'MWParserEnvironment.tokensToString, non-text token: ' +
 130+ this.dp ( 'MWParserEnvironment.tokensToString, non-text token: ' +
131131 tstring + JSON.stringify( tokens, null, 2 ) );
132132 out.push( tstring );
133133 }

Comments

#Comment by Yair rand (talk | contribs)   00:16, 23 January 2012

Not sure if this matters, but accessing individual characters in a String via "String[number]", as is used in lcfirst and ucfirst, does not work in IE7<.

#Comment by GWicke (talk | contribs)   02:08, 23 January 2012

Good to know. There are bigger things that make it incompatible with browsers currently (some of the node modules for example), but we should get to this eventually.

Status & tagging log