| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js |
| — | — | @@ -417,7 +417,7 @@ |
| 418 | 418 | if ( ! this.tailAccumulator ) { |
| 419 | 419 | this.emit( 'chunk', res.tokens ); |
| 420 | 420 | } else { |
| 421 | | - this.tailAccumulator.push( res.tokens ); |
| | 421 | + this.tailAccumulator.append( res.tokens ); |
| 422 | 422 | } |
| 423 | 423 | |
| 424 | 424 | if ( res.async ) { |
| — | — | @@ -932,7 +932,17 @@ |
| 933 | 933 | return this.accum.push(token); |
| 934 | 934 | }; |
| 935 | 935 | |
| | 936 | +/** |
| | 937 | + * Append tokens to an accumulator |
| | 938 | + * |
| | 939 | + * @method |
| | 940 | + * @param {Object} token |
| | 941 | + */ |
| | 942 | +TokenAccumulator.prototype.append = function ( token ) { |
| | 943 | + this.accum = this.accum.concat( token ); |
| | 944 | +}; |
| 936 | 945 | |
| | 946 | + |
| 937 | 947 | /** |
| 938 | 948 | * Loop check helper class for AsyncTokenTransformManager. |
| 939 | 949 | * |
| Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt |
| — | — | @@ -263,7 +263,8 @@ |
| 264 | 264 | // be part of a stream, only when tokenizing complete |
| 265 | 265 | // texts) |
| 266 | 266 | //console.log( pp( flatten ( e ) ) ); |
| 267 | | - return flatten(e); |
| | 267 | + __parseArgs[2]( [ { type: 'END' } ] ); |
| | 268 | + return []; //flatten(e); |
| 268 | 269 | } |
| 269 | 270 | |
| 270 | 271 | |
| — | — | @@ -397,19 +398,21 @@ |
| 398 | 399 | = & { blockStart = pos; return true; } b:block { |
| 399 | 400 | b = flatten(b); |
| 400 | 401 | var bs = b[0]; |
| 401 | | - //dp('toplevelblock:' + pp(b)); |
| 402 | | - if (bs.attribs === undefined) { |
| 403 | | - bs.attribs = []; |
| | 402 | + if ( b.length ) { |
| | 403 | + //dp('toplevelblock:' + pp(b)); |
| | 404 | + if (bs.attribs === undefined) { |
| | 405 | + bs.attribs = []; |
| | 406 | + } |
| | 407 | + bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]); |
| 404 | 408 | } |
| 405 | | - bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]); |
| 406 | 409 | // XXX: only run this for lines that actually need it! |
| 407 | 410 | //b.push({type: 'NEWLINE'}); |
| 408 | 411 | // Move this to a token stream transform! |
| 409 | 412 | //console.log('about to emit' + pp(self)); |
| 410 | | - //console.log( pp( result._tokenizer )); |
| 411 | | - //console.log('emitted chunk' + pp(b)); |
| | 413 | + //console.log( '__parseArgs call: ' + pp( b )); |
| | 414 | + __parseArgs[2]( flatten( b ) ); |
| 412 | 415 | //return []; |
| 413 | | - return b; |
| | 416 | + return true; |
| 414 | 417 | } |
| 415 | 418 | |
| 416 | 419 | block |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.tokenizer.peg.js |
| — | — | @@ -27,7 +27,11 @@ |
| 28 | 28 | var out, err; |
| 29 | 29 | if ( !this.parser ) { |
| 30 | 30 | // Only create a single parser, as parse() is a static method. |
| 31 | | - PegTokenizer.prototype.parser = PEG.buildParser(this.src); |
| | 31 | + var parserSource = PEG.buildParser(this.src).toSource(); |
| | 32 | + parserSource = parserSource.replace( 'parse: function(input, startRule) {', |
| | 33 | + 'parse: function(input, startRule) { var __parseArgs = arguments;' ); |
| | 34 | + //console.log( parserSource ); |
| | 35 | + PegTokenizer.prototype.parser = eval( parserSource ); |
| 32 | 36 | // add reference to this for event emission |
| 33 | 37 | // XXX: pass a cb into parse() instead, but need to modify pegjs a bit |
| 34 | 38 | // for that. |
| — | — | @@ -46,10 +50,10 @@ |
| 47 | 51 | // reasonable traces. Calling a trace on the extension does not really cut |
| 48 | 52 | // it. |
| 49 | 53 | //try { |
| 50 | | - out = this.parser.parse(text); |
| | 54 | + this.parser.parse(text, 'start', this.emit.bind( this, 'chunk' )); |
| 51 | 55 | // emit tokens here until we get that to work per toplevelblock in the |
| 52 | 56 | // actual tokenizer |
| 53 | | - this.emit('chunk', out.concat( [{ type: 'END' }] ) ); |
| | 57 | + //this.emit('chunk', out.concat( [{ type: 'END' }] ) ); |
| 54 | 58 | this.emit('end'); |
| 55 | 59 | //} catch (e) { |
| 56 | 60 | //err = e; |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js |
| — | — | @@ -417,7 +417,7 @@ |
| 418 | 418 | |
| 419 | 419 | function TemplateRequest ( manager, title ) { |
| 420 | 420 | // Increase the number of maximum listeners a bit.. |
| 421 | | - this.setMaxListeners( 1000 ); |
| | 421 | + this.setMaxListeners( 10000 ); |
| 422 | 422 | var self = this, |
| 423 | 423 | url = manager.env.wgScriptPath + '/api' + |
| 424 | 424 | manager.env.wgScriptExtension + |