r97979 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r97978‎ | r97979 | r97980 >
Date:00:53, 24 September 2011
Author:brion
Status:deferred
Tags:
Comment:
Commit some more of this half-rewritten code before I lose it. :)

Expansion bits half-ported from the PHP, needs to be finished & the formats normalized a bit.
Attempting to model the template expansion in a node tree on PPFrame::expand from preprocessor, but outputting another node tree (this is broken right now) rather than string output.
It needs to be able to exit and re-enter later when fetching templates asynchronously, though it doesn't use a very efficient plan yet (just waits).
Modified paths:
  • /trunk/extensions/ParserPlayground/modules/ext.parserPlayground.pegParser.js (modified) (history)
  • /trunk/extensions/ParserPlayground/modules/mediawiki.parser.environment.js (modified) (history)
  • /trunk/extensions/ParserPlayground/modules/pegParser.pegjs.txt (modified) (history)
  • /trunk/extensions/ParserPlayground/tests/expansionTest.js (added) (history)
  • /trunk/extensions/ParserPlayground/tests/tests.html (added) (history)

Diff [purge]

Index: trunk/extensions/ParserPlayground/tests/expansionTest.js
——@@ -0,0 +1,66 @@
 2+var pageDatabase = {
 3+ 'Template:Parens': {
 4+ type: 'root',
 5+ contents: [
 6+ '(',
 7+ {
 8+ type: 'tplarg',
 9+ /*
 10+ contents: [
 11+ '1'
 12+ ]*/
 13+ name: '1'
 14+ },
 15+ ')'
 16+ ]
 17+ },
 18+ 'ParenCaller': {
 19+ type: 'root',
 20+ contents: [
 21+ {
 22+ type: 'template',
 23+ /*
 24+ contents: [
 25+ {
 26+ type: 'title',
 27+ contents: [
 28+ 'Parens'
 29+ ]
 30+ },
 31+ {
 32+ type: 'part',
 33+ contents: [
 34+ {
 35+ type: 'name',
 36+ index: 1
 37+ },
 38+ {
 39+ type: 'value',
 40+ contents: [
 41+ 'bizbax'
 42+ ]
 43+ }
 44+ ]
 45+ }
 46+ ]*/
 47+ name: 'Parens',
 48+ params: {
 49+ 1: 'bizbax'
 50+ }
 51+ }
 52+ ]
 53+ }
 54+};
 55+
 56+var env = new MWParserEnvironment({
 57+ 'domCache': pageDatabase
 58+});
 59+var frame = new PPFrame(env);
 60+frame.expand(pageDatabase['ParenCaller'], 0, function(node, err) {
 61+ if (err) {
 62+ console.log('error', err);
 63+ } else {
 64+ console.log(node);
 65+ }
 66+});
 67+
Property changes on: trunk/extensions/ParserPlayground/tests/expansionTest.js
___________________________________________________________________
Added: svn:eol-style
168 + native
Index: trunk/extensions/ParserPlayground/tests/tests.html
——@@ -0,0 +1,6 @@
 2+<!DOCTYPE html>
 3+
 4+<script src="../../../resources/jquery/jquery.js"></script>
 5+<script src="../modules/mediawiki.parser.environment.js"></script>
 6+
 7+<script src="expansionTest.js"></script>
Index: trunk/extensions/ParserPlayground/modules/ext.parserPlayground.pegParser.js
——@@ -9,8 +9,8 @@
1010 * to point at the MW page name containing the parser peg definition; default
1111 * is 'MediaWiki:Gadget-ParserPlayground-PegParser.pegjs'.
1212 */
13 -function PegParser(options) {
14 - this.options = options;
 13+function PegParser(env) {
 14+ this.env = env || {};
1515 }
1616
1717 PegParser.src = false;
——@@ -34,8 +34,58 @@
3535 * @param {function(tree, error)} callback
3636 */
3737 PegParser.prototype.expandTree = function(tree, callback) {
38 - // no-op!
39 - callback(tree, null);
 38+ var self = this;
 39+ var subParseArray = function(listOfTrees) {
 40+ var content = [];
 41+ $.each(listOfTrees, function(i, subtree) {
 42+ self.expandTree(subtree, function(substr, err) {
 43+ content.push(tree);
 44+ });
 45+ });
 46+ return content;
 47+ };
 48+ var src;
 49+ if (typeof tree === "string") {
 50+ callback(tree);
 51+ return;
 52+ }
 53+ if (tree.type == 'template') {
 54+ // expand a template node!
 55+
 56+ // Resolve a possibly relative link
 57+ var templateName = this.env.resolveTitle( tree.target, 'Template' );
 58+ this.env.fetchTemplate( tree.target, tree.params || {}, function( templateSrc, error ) {
 59+ // @fixme should pre-parse/cache these too?
 60+ self.parseToTree( templateSrc, function( templateTree, error ) {
 61+ if ( error ) {
 62+ callback({
 63+ type: 'placeholder',
 64+ orig: tree,
 65+ content: [
 66+ {
 67+ // @fixme broken link?
 68+ type: 'link',
 69+ target: templateName
 70+ }
 71+ ]
 72+ });
 73+ } else {
 74+ callback({
 75+ type: 'placeholder',
 76+ orig: tree,
 77+ content: self.env.expandTemplateArgs( templateTree, tree.params )
 78+ });
 79+ }
 80+ })
 81+ } );
 82+ // Wait for async...
 83+ return;
 84+ }
 85+ var out = $.extend( tree ); // @fixme prefer a deep copy?
 86+ if (tree.content) {
 87+ out.content = subParseArray(tree.content);
 88+ }
 89+ callback(out);
4090 };
4191
4292 PegParser.prototype.initSource = function(callback) {
Index: trunk/extensions/ParserPlayground/modules/mediawiki.parser.environment.js
——@@ -1,11 +1,15 @@
22 var MWParserEnvironment = function(opts) {
33 var options = {
44 tagHooks: {},
5 - parserFunctions: {}
 5+ parserFunctions: {},
 6+ pageCache: {}, // @fixme use something with managed space
 7+ domCache: {}
68 };
79 $.extend(options, opts);
810 this.tagHooks = options.tagHooks;
911 this.parserFunctions = options.parserFunctions;
 12+ this.pageCache = options.pageCache;
 13+ this.domCache = options.domCache;
1014 };
1115
1216 $.extend(MWParserEnvironment.prototype, {
——@@ -38,12 +42,346 @@
3943 } else {
4044 return null;
4145 }
 46+ },
 47+
 48+ /**
 49+ * @fixme do this for real eh
 50+ */
 51+ resolveTitle: function( name, namespace ) {
 52+ // hack!
 53+ if (name.indexOf(':') == 0 && typeof namespace ) {
 54+ // hack hack hack
 55+ name = namespace + ':' + name;
 56+ }
 57+ return name;
 58+ },
 59+
 60+ /**
 61+ * Async.
 62+ *
 63+ * @todo make some optimizations for fetching multiple at once
 64+ *
 65+ * @param string name
 66+ * @param function(text, err) callback
 67+ */
 68+ fetchTemplate: function( title, callback ) {
 69+ this.fetchTemplateAndTitle( title, function( text, title, err ) {
 70+ callback(title, err);
 71+ });
 72+ },
 73+
 74+ fetchTemplateAndTitle: function( title, callback ) {
 75+ // @fixme normalize name?
 76+ if (title in this.pageCache) {
 77+ // @fixme should this be forced to run on next event?
 78+ callback( this.pageCache[title] );
 79+ } else {
 80+ // whee fun hack!
 81+ $.ajax({
 82+ url: wgScriptPath + '/api' + wgScriptExtension,
 83+ data: {
 84+ format: 'json',
 85+ action: 'query',
 86+ prop: 'revisions',
 87+ rvprop: 'content',
 88+ titles: name
 89+ },
 90+ success: function(data, xhr) {
 91+ var src = null, title = null;
 92+ $.each(data.query.pages, function(i, page) {
 93+ if (page.revisions && page.revisions.length) {
 94+ src = page.revisions[0]['*'];
 95+ title = page.title;
 96+ }
 97+ });
 98+ if (typeof src !== 'string') {
 99+ callback(null, null, 'Page not found');
 100+ } else {
 101+ callback(src, title);
 102+ }
 103+ },
 104+ error: function(msg) {
 105+ callback(null, null, 'Page/template fetch failure');
 106+ },
 107+ dataType: 'json',
 108+ cache: false // @fixme caching, versions etc?
 109+ }, 'json');
 110+ }
 111+ },
 112+
 113+ getTemplateDom: function( title, callback ) {
 114+ var self = this;
 115+ if (title in this.domCache) {
 116+ callback(this.domCache[title], null);
 117+ }
 118+ this.fetchTemplateAndTitle( title, function( text, title, err ) {
 119+ if (err) {
 120+ callback(null, err);
 121+ return;
 122+ }
 123+ self.pageCache[title] = text;
 124+ self.parser.parseToTree( text, function( templateTree, err ) {
 125+ this.domCache[title] = templateTree;
 126+ callback(templateTree, err);
 127+ });
 128+ });
 129+ },
 130+
 131+ braceSubstitution: function( templateNode, frame, callback ) {
 132+ // stuff in Parser.braceSubstitution
 133+ // expand/flatten the 'title' piece (to get the template reference)
 134+ frame.flatten(templateNode.name, function(templateName, err) {
 135+ if (err) {
 136+ callback(null, err);
 137+ return;
 138+ }
 139+ var out = {
 140+ type: 'placeholder',
 141+ orig: templateNode,
 142+ contents: []
 143+ };
 144+
 145+ // check for 'subst:'
 146+ // check for variable magic names
 147+ // check for msg, msgnw, raw magics
 148+ // check for parser functions
 149+
 150+ // resolve template name
 151+ // load template w/ canonical name
 152+ // load template w/ variant names
 153+ // recursion depth check
 154+ // fetch from DB or interwiki
 155+ // infinte loop check
 156+ this.getTemplateDom(templateName, function(dom, err) {
 157+ // Expand in-place!
 158+ var templateFrame = frame.newChild(templateName.params || []);
 159+ templateFrame.expand(dom, 0, function(expandedTemplateNode) {
 160+ out.contents = expandedTemplateNode.contents;
 161+ callback(out);
 162+ return; // done
 163+ });
 164+ return; // wait for async
 165+ });
 166+ });
 167+ },
 168+
 169+ argSubstitution: function( argNode, frame, callback ) {
 170+ frame.flatten(argNode.name, function(argName, err) {
 171+ if (err) {
 172+ callback(null, err);
 173+ return;
 174+ }
 175+
 176+ var arg = frame.getArgument(argName);
 177+ if (arg === false && 'params' in argNode && argNode.params.length) {
 178+ // No match in frame, use the supplied default
 179+ arg = argNode.params[0].val;
 180+ }
 181+ var out = {
 182+ type: 'placeholder',
 183+ orig: argNode,
 184+ contents: [arg]
 185+ };
 186+ callback(out);
 187+ });
42188 }
43 -
 189+
 190+
44191 });
45192
 193+function PPFrame(env) {
 194+ this.env = env;
 195+ this.loopCheckHash = [];
 196+ this.depth = 0;
 197+}
46198
 199+// Flag constants
 200+$.extend(PPFrame, {
 201+ NO_ARGS: 1,
 202+ NO_TEMPLATES: 2,
 203+ STRIP_COMMENTS: 4,
 204+ NO_IGNORE: 8,
 205+ RECOVER_COMMENTS: 16
 206+});
 207+PPFrame.RECOVER_ORIG = PPFrame.NO_ARGS
 208+ | PPFrame.NO_TEMPLATES
 209+ | PPFrame.STRIP_COMMENTS
 210+ | PPFrame.NO_IGNORE
 211+ | PPFrame.RECOVER_COMMENTS;
47212
 213+$.extend(PPFrame.prototype, {
 214+ newChild: function(args, title) {
 215+ //
 216+ },
 217+
 218+ /**
 219+ * Using simple recursion for now -- PHP version is a little fancier.
 220+ *
 221+ * The iterator loop is set off in a closure so we can continue it after
 222+ * waiting for an asynchronous template fetch.
 223+ *
 224+ * Note that this is inefficient, as we have to wait for the entire round
 225+ * trip before continuing -- in browser-based work this may be particularly
 226+ * slow. This can be mitigated by prefetching templates based on previous
 227+ * knowledge or an initial tree-walk.
 228+ *
 229+ * @param {object} tree
 230+ * @param {number} flags
 231+ * @param {function(tree, error)} callback
 232+ */
 233+ expand: function(root, flags, callback) {
 234+ /**
 235+ * Clone a node, but give the clone an empty contents
 236+ */
 237+ var cloneNode = function(node) {
 238+ var out = $.extend({}, node);
 239+ out.contents = [];
 240+ return out;
 241+ }
 242+ // stub node to write into
 243+ var rootOut = cloneNode(root);
 244+
 245+ var self = this,
 246+ expansionDepth = 0,
 247+ outStack = [{contents: []}, rootOut],
 248+ iteratorStack = [false, root],
 249+ indexStack = [0, 0],
 250+ contextNode = false,
 251+ newIterator = false,
 252+ continuing = false;
 253+
 254+ var iteration = function() {
 255+ // This while loop is a tail call recursion optimization simulator :)
 256+ while (iteratorStack.length > 1) {
 257+ var level = outStack.length - 1,
 258+ iteratorNode = iteratorStack[level],
 259+ out = outStack[level],
 260+ index = indexStack[level]; // ????
 261+
 262+ if (continuing) {
 263+ // If we're re-entering from an asynchronous data fetch,
 264+ // skip over this part, we've done it before.
 265+ continuing = false;
 266+ } else {
 267+ if ($.isArray(iteratorNode)) {
 268+ if (index >= iteratorNode.length) {
 269+ // All done with this iterator.
 270+ iteratorStack[level] = false;
 271+ contextNode = false;
 272+ } else {
 273+ contextNode = iteratorNode[index];
 274+ indexStack[level]++;
 275+ }
 276+ } else {
 277+ // Copy to contextNode and then delete from iterator stack,
 278+ // because this is not an iterator but we do have to execute it once
 279+ contextNode = iteratorStack[level];
 280+ iteratorStack[level] = false;
 281+ }
 282+ }
 283+
 284+ if (contextNode === false) {
 285+ // nothing to do
 286+ } else if (typeof contextNode === 'string') {
 287+ out.contents.push(contextNode);
 288+ } else if (contextNode.type === 'template') {
 289+ // Double-brace expansion
 290+ continuing = true;
 291+ self.env.braceSubstitution(contextNode, self, function(replacementNode, err) {
 292+ out.contents.push(replacementNode);
 293+ // ... and continue on the next node!
 294+ iteration();
 295+ });
 296+ return; // pause for async work...
 297+ } else if (contextNode.type == 'tplarg') {
 298+ // Triple-brace expansion
 299+ continuing = true;
 300+ self.env.argSubstitution(contextNode, self, function(replacementNode, err) {
 301+ out.contents.push(replacementNode);
 302+ // ... and continue on the next node!
 303+ iteration();
 304+ });
 305+ return; // pause for async work...
 306+ } else {
 307+ if ('content' in contextNode && contextNode.content.length) {
 308+ // Generic recursive expansion
 309+ newIterator = contextNode;
 310+ } else {
 311+ // No children; push as-is.
 312+ out.contents.push(contextNode);
 313+ }
 314+ }
 315+
 316+ if (newIterator !== false) {
 317+ outStack.push(cloneNode(newIterator));
 318+ iteratorStack.push(newIterator);
 319+ indexStack.push(0);
 320+ } else if ( iteratorStack[level] === false) {
 321+ // Return accumulated value to parent
 322+ // With tail recursion
 323+ while (iteratorStack[level] === false && level > 0) {
 324+ outStack[level - 1].contents.push(out);
 325+ outStack.pop();
 326+ iteratorStack.pop();
 327+ indexStack.pop();
 328+ level--;
 329+ }
 330+ }
 331+ }
 332+ // We've reached the end of the loop!
 333+ --expansionDepth;
 334+ callback(outStack.pop(), null);
 335+ };
 336+ iteration();
 337+ },
 338+
 339+ implodeWithFlags: function(sep, flags) {
 340+
 341+ },
 342+
 343+ implode: function(sep) {
 344+
 345+ },
 346+
 347+ virtualImport: function(sep) {
 348+
 349+ },
 350+
 351+ virtualBracketedImplode: function(start, sep, end /*, ... */ ) {
 352+
 353+ },
 354+
 355+ isEmpty: function() {
 356+
 357+ },
 358+
 359+ getArguments: function() {
 360+
 361+ },
 362+
 363+ getNumberedArguments: function() {
 364+
 365+ },
 366+
 367+ getNamedArguments: function() {
 368+
 369+ },
 370+
 371+ getArgument: function( name ) {
 372+
 373+ },
 374+
 375+ loopCheck: function(title) {
 376+ },
 377+
 378+ isTemplate: function() {
 379+
 380+ }
 381+
 382+});
 383+
 384+
 385+
48386 /**
49387 * @parm MWParserEnvironment env
50388 * @constructor
——@@ -67,7 +405,7 @@
68406
69407 MWParserFunction = function( env) {
70408 if (!env) {
71 - throw new Error( 'Parser funciton requires a parser environment.');
 409+ throw new Error( 'Parser function requires a parser environment.');
72410 }
73411 this.env = env;
74412 };
Index: trunk/extensions/ParserPlayground/modules/pegParser.pegjs.txt
——@@ -174,6 +174,17 @@
175175 };
176176 }
177177
 178+tplarg = "{{{" name:link_target params:("|" p:template_param { return p })* "}}}" {
 179+ var obj = {
 180+ type: 'tplarg',
 181+ name: name
 182+ };
 183+ if (params && params.length) {
 184+ obj.params = params;
 185+ }
 186+ return obj;
 187+}
 188+
178189 template_param_name
179190 = h:( !"}}" x:([^=|]) { return x } )* { return h.join(''); }
180191

Status & tagging log