| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformDispatcher.js |
| — | — | @@ -12,7 +12,8 @@ |
| 13 | 13 | var events = require('events'); |
| 14 | 14 | |
| 15 | 15 | /** |
| 16 | | - * Central dispatcher for potentially asynchronous token transformations. |
| | 16 | + * Central dispatcher and manager for potentially asynchronous token |
| | 17 | + * transformations. |
| 17 | 18 | * |
| 18 | 19 | * @class |
| 19 | 20 | * @constructor |
| — | — | @@ -44,7 +45,7 @@ |
| 45 | 46 | any: [] // all tokens, before more specific handlers are run |
| 46 | 47 | } |
| 47 | 48 | }; |
| 48 | | - this.reset(); |
| | 49 | + this._reset(); |
| 49 | 50 | } |
| 50 | 51 | |
| 51 | 52 | // Inherit from EventEmitter |
| — | — | @@ -69,9 +70,9 @@ |
| 70 | 71 | * |
| 71 | 72 | * @method |
| 72 | 73 | */ |
| 73 | | -TokenTransformDispatcher.prototype.reset = function ( env ) { |
| | 74 | +TokenTransformDispatcher.prototype._reset = function ( env ) { |
| 74 | 75 | this.tailAccumulator = undefined; |
| 75 | | - this.phase2TailCB = this.returnTokens01.bind( this ); |
| | 76 | + this.phase2TailCB = this._returnTokens01.bind( this ); |
| 76 | 77 | this.accum = new TokenAccumulator(null); |
| 77 | 78 | this.firstaccum = this.accum; |
| 78 | 79 | this.prevToken = undefined; |
| — | — | @@ -82,7 +83,7 @@ |
| 83 | 84 | // Should be as static as possible re this and frame |
| 84 | 85 | // This is circular, but that should not really matter for non-broken GCs |
| 85 | 86 | // that handle pure JS ref loops. |
| 86 | | - this.frame.transformPhase = this.transformPhase01.bind( this, this.frame ); |
| | 87 | + this.frame.transformPhase = this._transformPhase01.bind( this, this.frame ); |
| 87 | 88 | }; |
| 88 | 89 | |
| 89 | 90 | TokenTransformDispatcher.prototype._rankToPhase = function ( rank ) { |
| — | — | @@ -101,6 +102,8 @@ |
| 102 | 103 | * |
| 103 | 104 | * @method |
| 104 | 105 | * @param {Function} transform. |
| | 106 | + * @param {Number} rank, [0,3) with [0,1) in-order on input token stream, |
| | 107 | + * [1,2) out-of-order and [2,3) in-order on output token stream |
| 105 | 108 | * @param {String} type, one of 'tag', 'text', 'newline', 'comment', 'end', |
| 106 | 109 | * 'martian' (unknown token), 'any' (any token, matched before other matches). |
| 107 | 110 | * @param {String} tag name for tags, omitted for non-tags |
| — | — | @@ -130,7 +133,9 @@ |
| 131 | 134 | * Remove a transform registration |
| 132 | 135 | * |
| 133 | 136 | * @method |
| 134 | | - * @param {Number} rank, the numeric rank of the handler. |
| | 137 | + * @param {Function} transform. |
| | 138 | + * @param {Number} rank, [0,3) with [0,1) in-order on input token stream, |
| | 139 | + * [1,2) out-of-order and [2,3) in-order on output token stream |
| 135 | 140 | * @param {String} type, one of 'tag', 'text', 'newline', 'comment', 'end', |
| 136 | 141 | * 'martian' (unknown token), 'any' (any token, matched before other matches). |
| 137 | 142 | * @param {String} tag name for tags, omitted for non-tags |
| — | — | @@ -255,10 +260,10 @@ |
| 256 | 261 | // skip transformation, was already applied. |
| 257 | 262 | continue; |
| 258 | 263 | } |
| 259 | | - // Transform token with side effects |
| 260 | | - // XXX: it should be a better idea to move the token.rank out of |
| 261 | | - // token and into a wrapper object to ensure that transformations |
| 262 | | - // don't mess with it! |
| | 264 | + // Transform the token. |
| | 265 | + // XXX: consider moving the rank out of the token itself to avoid |
| | 266 | + // transformations messing with it in broken ways. Not sure if |
| | 267 | + // some transformations need to manipulate it though. gwicke |
| 263 | 268 | res = transformer.transform( res.token, cb, frame, this.prevToken ); |
| 264 | 269 | if ( !res.token || |
| 265 | 270 | res.token.type !== token.type ) { |
| — | — | @@ -284,7 +289,7 @@ |
| 285 | 290 | */ |
| 286 | 291 | TokenTransformDispatcher.prototype.transformTokens = function ( tokens ) { |
| 287 | 292 | //console.log('TokenTransformDispatcher transformTokens'); |
| 288 | | - var res = this.transformPhase01 ( this.frame, tokens, this.phase2TailCB ); |
| | 293 | + var res = this._transformPhase01 ( this.frame, tokens, this.phase2TailCB ); |
| 289 | 294 | this.phase2TailCB( tokens, true ); |
| 290 | 295 | if ( res.async ) { |
| 291 | 296 | this.tailAccumulator = res.async; |
| — | — | @@ -293,9 +298,10 @@ |
| 294 | 299 | }; |
| 295 | 300 | |
| 296 | 301 | /** |
| 297 | | - * Callback for the event emitted from the tokenizer. |
| 298 | | - * |
| 299 | | - * This simply decrements the outstanding counter on the top-level |
| | 302 | + * Callback for the end event emitted from the tokenizer. |
| | 303 | + * Either signals the end of input to the tail of an ongoing asynchronous |
| | 304 | + * processing pipeline, or directly emits 'end' if the processing was fully |
| | 305 | + * synchronous. |
| 300 | 306 | */ |
| 301 | 307 | TokenTransformDispatcher.prototype.onEndEvent = function () { |
| 302 | 308 | if ( this.tailAccumulator ) { |
| — | — | @@ -303,23 +309,23 @@ |
| 304 | 310 | } else { |
| 305 | 311 | // nothing was asynchronous, so we'll have to emit end here. |
| 306 | 312 | this.emit('end'); |
| | 313 | + this._reset(); |
| 307 | 314 | } |
| 308 | 315 | }; |
| 309 | 316 | |
| | 317 | + |
| 310 | 318 | /** |
| 311 | | - * add parent, parentref args |
| 312 | | - * return |
| 313 | | - * {tokens: [tokens], async: true}: async expansion -> outstanding++ in parent |
| 314 | | - * {tokens: [tokens], async: false}: fully expanded |
| 315 | | - * {token: {token}}: single-token return |
| 316 | | - * child after first expand (example: template expanded) |
| 317 | | - * return some finished tokens, reuse parent accumulator |
| 318 | | - * if new accumulator: set parent, ref |
| | 319 | + * Run transformations from phases 0 and 1. This includes starting and |
| | 320 | + * managing asynchronous transformations. |
| | 321 | + * |
| | 322 | + * return protocol for transform*Token: |
| | 323 | + * { tokens: [tokens], async: true }: async expansion -> outstanding++ in parent |
| | 324 | + * { tokens: [tokens] }: fully expanded, tokens will be reprocessed |
| | 325 | + * { token: token }: single-token return |
| 319 | 326 | */ |
| | 327 | +TokenTransformDispatcher.prototype._transformPhase01 = function ( frame, tokens, parentCB ) { |
| 320 | 328 | |
| 321 | | -TokenTransformDispatcher.prototype.transformPhase01 = function ( frame, tokens, parentCB ) { |
| 322 | | - |
| 323 | | - //console.log('transformPhase01: ' + JSON.stringify(tokens) ); |
| | 329 | + //console.log('_transformPhase01: ' + JSON.stringify(tokens) ); |
| 324 | 330 | |
| 325 | 331 | var res, |
| 326 | 332 | phaseEndRank = 2, |
| — | — | @@ -401,30 +407,30 @@ |
| 402 | 408 | * Callback from tokens fully processed for phase 0 and 1, which are now ready |
| 403 | 409 | * for synchronous and globally in-order phase 2 processing. |
| 404 | 410 | */ |
| 405 | | -TokenTransformDispatcher.prototype.returnTokens01 = function ( tokens, notYetDone ) { |
| | 411 | +TokenTransformDispatcher.prototype._returnTokens01 = function ( tokens, notYetDone ) { |
| 406 | 412 | // FIXME: store frame in object? |
| 407 | | - tokens = this.transformPhase2( this.frame, tokens, this.parentCB ); |
| 408 | | - //console.log('returnTokens01, after transformPhase2.'); |
| | 413 | + tokens = this._transformPhase2( this.frame, tokens, this.parentCB ); |
| | 414 | + //console.log('_returnTokens01, after _transformPhase2.'); |
| 409 | 415 | |
| 410 | 416 | this.emit( 'chunk', tokens ); |
| 411 | 417 | |
| 412 | 418 | if ( ! notYetDone ) { |
| 413 | | - console.log('returnTokens01 done.'); |
| | 419 | + console.log('_returnTokens01 done.'); |
| 414 | 420 | // signal our done-ness to consumers. |
| 415 | 421 | this.emit( 'end' ); |
| 416 | 422 | // and reset internal state. |
| 417 | | - this.reset(); |
| | 423 | + this._reset(); |
| 418 | 424 | } |
| 419 | 425 | }; |
| 420 | 426 | |
| 421 | 427 | |
| 422 | 428 | /** |
| 423 | | - * Phase 2 |
| | 429 | + * Phase 3 (rank [2,3)) |
| 424 | 430 | * |
| 425 | 431 | * Global in-order traversal on expanded token stream (after async phase 1). |
| 426 | | - * Very similar to transformPhase01, but without async handling. |
| | 432 | + * Very similar to _transformPhase01, but without async handling. |
| 427 | 433 | */ |
| 428 | | -TokenTransformDispatcher.prototype.transformPhase2 = function ( frame, tokens, cb ) { |
| | 434 | +TokenTransformDispatcher.prototype._transformPhase2 = function ( frame, tokens, cb ) { |
| 429 | 435 | var res, |
| 430 | 436 | phaseEndRank = 3, |
| 431 | 437 | localAccum = [], |
| — | — | @@ -513,7 +519,7 @@ |
| 514 | 520 | * @returns {Function} |
| 515 | 521 | */ |
| 516 | 522 | TokenAccumulator.prototype.getParentCB = function ( reference ) { |
| 517 | | - return this.returnTokens01.bind( this, reference ); |
| | 523 | + return this._returnTokens01.bind( this, reference ); |
| 518 | 524 | }; |
| 519 | 525 | |
| 520 | 526 | /** |
| — | — | @@ -522,7 +528,7 @@ |
| 523 | 529 | * @method |
| 524 | 530 | * @param {Object} token |
| 525 | 531 | */ |
| 526 | | -TokenAccumulator.prototype.returnTokens01 = function ( reference, tokens, notYetDone ) { |
| | 532 | +TokenAccumulator.prototype._returnTokens01 = function ( reference, tokens, notYetDone ) { |
| 527 | 533 | var res, |
| 528 | 534 | cb, |
| 529 | 535 | returnTokens = []; |
| — | — | @@ -534,7 +540,7 @@ |
| 535 | 541 | if ( reference === 'child' ) { |
| 536 | 542 | // XXX: Use some marker to avoid re-transforming token chunks several |
| 537 | 543 | // times? |
| 538 | | - res = this.transformPhase01( this.frame, tokens, this.parentCB ); |
| | 544 | + res = this._transformPhase01( this.frame, tokens, this.parentCB ); |
| 539 | 545 | |
| 540 | 546 | if ( res.async ) { |
| 541 | 547 | // new asynchronous expansion started, chain of accumulators |
| — | — | @@ -582,7 +588,7 @@ |
| 583 | 589 | * Mark the sibling as done (normally at the tail of a chain). |
| 584 | 590 | */ |
| 585 | 591 | TokenAccumulator.prototype.siblingDone = function () { |
| 586 | | - this.returnTokens01 ( 'sibling', [], false ); |
| | 592 | + this._returnTokens01 ( 'sibling', [], false ); |
| 587 | 593 | }; |
| 588 | 594 | |
| 589 | 595 | |
| — | — | @@ -598,24 +604,6 @@ |
| 599 | 605 | |
| 600 | 606 | |
| 601 | 607 | |
| 602 | | -/* TODO list |
| 603 | | - * |
| 604 | | - * transformPhase01 called first for phase 0-1 (in-order per source file) |
| 605 | | - * then only phase 2 (order independent, if 2 <= token phase < 3, 3 ~ done) |
| 606 | | - * -> don't execute order-dependent transforms in this phase! |
| 607 | | - * * enforce phase on tokens, but not priority within phase |
| 608 | | - * -> cycles possible in async phase |
| 609 | | - * final transform (phase 2) globally in-order and synchronous in root returnTokens01 |
| 610 | | - * |
| 611 | | - * |
| 612 | | - * Transformation phases |
| 613 | | - * [0,2) |
| 614 | | - * [2,3] (and 1..2 in templates etc, but clamp phase on *returned* tokens to 2) |
| 615 | | - * 3 |
| 616 | | - * |
| 617 | | - */ |
| 618 | | - |
| 619 | | - |
| 620 | 608 | if (typeof module == "object") { |
| 621 | 609 | module.exports.TokenTransformDispatcher = TokenTransformDispatcher; |
| 622 | 610 | } |