| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js |
| — | — | @@ -110,28 +110,25 @@ |
| 111 | 111 | name: token.name, |
| 112 | 112 | data: this._att(token.attribs)}); |
| 113 | 113 | break; |
| 114 | | - default: |
| 115 | | - switch (token.type) { |
| 116 | | - case "COMMENT": |
| 117 | | - this.emit('token', {type: 'Comment', |
| 118 | | - data: token.value}); |
| 119 | | - break; |
| 120 | | - case "END": |
| 121 | | - this.emit('end'); |
| 122 | | - this.emit('token', { type: 'EOF' } ); |
| 123 | | - this.document = this.parser.document; |
| 124 | | - if ( ! this.document.body ) { |
| 125 | | - // HACK: This should not be needed really. |
| 126 | | - this.document.body = this.parser.document.getElementsByTagName('body')[0]; |
| 127 | | - } |
| 128 | | - // Emit the document to consumers |
| 129 | | - //this.emit('document', this.document); |
| 130 | | - break; |
| 131 | | - default: |
| 132 | | - console.warn("Unhandled token: " + JSON.stringify(token)); |
| 133 | | - break; |
| | 114 | + case CommentTk: |
| | 115 | + this.emit('token', {type: 'Comment', |
| | 116 | + data: token.value}); |
| | 117 | + break; |
| | 118 | + case EOFTk: |
| | 119 | + this.emit('end'); |
| | 120 | + this.emit('token', { type: 'EOF' } ); |
| | 121 | + this.document = this.parser.document; |
| | 122 | + if ( ! this.document.body ) { |
| | 123 | + // HACK: This should not be needed really. |
| | 124 | + this.document.body = this.parser.document.getElementsByTagName('body')[0]; |
| 134 | 125 | } |
| | 126 | + // Emit the document to consumers |
| | 127 | + //this.emit('document', this.document); |
| 135 | 128 | break; |
| | 129 | + default: |
| | 130 | + console.warn("Unhandled token: " + JSON.stringify(token)); |
| | 131 | + break; |
| | 132 | + break; |
| 136 | 133 | } |
| 137 | 134 | }; |
| 138 | 135 | |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.NoIncludeOnly.js |
| — | — | @@ -36,7 +36,7 @@ |
| 37 | 37 | |
| 38 | 38 | OnlyInclude.prototype.onAnyInclude = function ( token, manager ) { |
| 39 | 39 | //this.manager.env.dp( 'onAnyInclude', token, this ); |
| 40 | | - if ( token.type === 'END' ) { |
| | 40 | + if ( token.constructor === EOFTk ) { |
| 41 | 41 | this.inOnlyInclude = false; |
| 42 | 42 | if ( this.accum.length && ! this.foundOnlyInclude ) { |
| 43 | 43 | var res = this.accum; |
| — | — | @@ -92,7 +92,7 @@ |
| 93 | 93 | } else { |
| 94 | 94 | tokens.shift(); |
| 95 | 95 | if ( tokens.length && |
| 96 | | - tokens[tokens.length - 1].type !== 'END' ) { |
| | 96 | + tokens[tokens.length - 1].constructor !== EOFTk ) { |
| 97 | 97 | tokens.pop(); |
| 98 | 98 | } |
| 99 | 99 | return { tokens: tokens }; |
| — | — | @@ -114,7 +114,7 @@ |
| 115 | 115 | if ( isInclude ) { |
| 116 | 116 | tokens.shift(); |
| 117 | 117 | if ( tokens.length && |
| 118 | | - tokens[tokens.length - 1].type !== 'END' ) { |
| | 118 | + tokens[tokens.length - 1].constructor !== EOFTk ) { |
| 119 | 119 | tokens.pop(); |
| 120 | 120 | } |
| 121 | 121 | return { tokens: tokens }; |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformManager.js |
| — | — | @@ -37,7 +37,7 @@ |
| 38 | 38 | |
| 39 | 39 | TokenTransformManager.prototype._construct = function () { |
| 40 | 40 | this.transformers = { |
| 41 | | - tag: {}, // for TAG, ENDTAG, SELFCLOSINGTAG, keyed on name |
| | 41 | + tag: {}, // for TagTk, EndTagTk, SelfclosingTagTk, keyed on name |
| 42 | 42 | text: [], |
| 43 | 43 | newline: [], |
| 44 | 44 | comment: [], |
| — | — | @@ -452,19 +452,16 @@ |
| 453 | 453 | case SelfclosingTagTk: |
| 454 | 454 | res = this._transformTagToken( token, cb ); |
| 455 | 455 | break; |
| | 456 | + case CommentTk: |
| | 457 | + res = this._transformToken( token, ts.comment, cb ); |
| | 458 | + break; |
| | 459 | + case EOFTk: |
| | 460 | + res = this._transformToken( token, ts.end, cb ); |
| | 461 | + break; |
| 456 | 462 | default: |
| 457 | | - switch( token.type ) { |
| 458 | | - case 'COMMENT': |
| 459 | | - res = this._transformToken( token, ts.comment, cb ); |
| 460 | | - break; |
| 461 | | - case 'END': |
| 462 | | - res = this._transformToken( token, ts.end, cb ); |
| 463 | | - break; |
| 464 | | - default: |
| 465 | | - res = this._transformToken( token, ts.martian, cb ); |
| 466 | | - break; |
| 467 | | - } |
| | 463 | + res = this._transformToken( token, ts.martian, cb ); |
| 468 | 464 | break; |
| | 465 | + break; |
| 469 | 466 | } |
| 470 | 467 | |
| 471 | 468 | if( res.tokens ) { |
| — | — | @@ -526,15 +523,9 @@ |
| 527 | 524 | function ( tokens, notYetDone, allTokensProcessed ) { |
| 528 | 525 | //tokens = this._transformPhase2( this.frame, tokens, this.parentCB ); |
| 529 | 526 | |
| 530 | | - //if ( tokens.length && tokens[tokens.length - 1].type === 'END' ) { |
| 531 | | - // this.env.dp( 'AsyncTokenTransformManager, stripping end ' ); |
| 532 | | - // tokens.pop(); |
| 533 | | - //} |
| 534 | | - |
| 535 | 527 | this.env.dp( 'AsyncTokenTransformManager._returnTokens, emitting chunk: ', |
| 536 | 528 | tokens ); |
| 537 | 529 | |
| 538 | | - |
| 539 | 530 | if( !allTokensProcessed ) { |
| 540 | 531 | var res = this.transformTokens( tokens, this._returnTokens.bind(this) ); |
| 541 | 532 | this.emit( 'chunk', res.tokens ); |
| — | — | @@ -559,13 +550,7 @@ |
| 560 | 551 | this.emit( 'chunk', tokens ); |
| 561 | 552 | |
| 562 | 553 | if ( ! notYetDone ) { |
| 563 | | - //console.warn('AsyncTokenTransformManager._returnTokens done. tokens:' + |
| 564 | | - // JSON.stringify( tokens, null, 2 ) + ', listeners: ' + |
| 565 | | - // JSON.stringify( this.listeners( 'chunk' ), null, 2 ) ); |
| 566 | 554 | // signal our done-ness to consumers. |
| 567 | | - //if ( this.atTopLevel ) { |
| 568 | | - // this.emit( 'chunk', [{type: 'END'}]); |
| 569 | | - //} |
| 570 | 555 | this.emit( 'end' ); |
| 571 | 556 | // and reset internal state. |
| 572 | 557 | this._reset(); |
| — | — | @@ -663,18 +648,15 @@ |
| 664 | 649 | case SelfclosingTagTk: |
| 665 | 650 | res = this._transformTagToken( token, this.prevToken ); |
| 666 | 651 | break; |
| | 652 | + case CommentTk: |
| | 653 | + res = this._transformToken( token, ts.comment, this.prevToken ); |
| | 654 | + break; |
| | 655 | + case EOFTk: |
| | 656 | + res = this._transformToken( token, ts.end, this.prevToken ); |
| | 657 | + break; |
| 667 | 658 | default: |
| 668 | | - switch( token.type ) { |
| 669 | | - case 'COMMENT': |
| 670 | | - res = this._transformToken( token, ts.comment, this.prevToken ); |
| 671 | | - break; |
| 672 | | - case 'END': |
| 673 | | - res = this._transformToken( token, ts.end, this.prevToken ); |
| 674 | | - break; |
| 675 | | - default: |
| 676 | | - res = this._transformToken( token, ts.martian, this.prevToken ); |
| 677 | | - break; |
| 678 | | - } |
| | 659 | + res = this._transformToken( token, ts.martian, this.prevToken ); |
| | 660 | + break; |
| 679 | 661 | } |
| 680 | 662 | |
| 681 | 663 | if( res.tokens ) { |
| — | — | @@ -766,7 +748,7 @@ |
| 767 | 749 | pipe.on( 'end', |
| 768 | 750 | this.onEnd.bind( this, this._returnAttributeKey.bind( this, i ) ) |
| 769 | 751 | ); |
| 770 | | - pipe.process( attributes[i].k.concat([{type:'END'}]) ); |
| | 752 | + pipe.process( attributes[i].k.concat([ new EOFTk() ]) ); |
| 771 | 753 | } else { |
| 772 | 754 | kv.key = cur.k; |
| 773 | 755 | } |
| — | — | @@ -785,7 +767,7 @@ |
| 786 | 768 | this.onEnd.bind( this, this._returnAttributeValue.bind( this, i ) ) |
| 787 | 769 | ); |
| 788 | 770 | //console.warn('starting attribute transform of ' + JSON.stringify( attributes[i].v ) ); |
| 789 | | - pipe.process( cur.v.concat([{type:'END'}]) ); |
| | 771 | + pipe.process( cur.v.concat([ new EOFTk() ]) ); |
| 790 | 772 | } else { |
| 791 | 773 | kv.value = cur.v; |
| 792 | 774 | } |
| — | — | @@ -820,7 +802,7 @@ |
| 821 | 803 | * Collect chunks returned from the pipeline |
| 822 | 804 | */ |
| 823 | 805 | AttributeTransformManager.prototype.onChunk = function ( cb, chunk ) { |
| 824 | | - if ( chunk.length && chunk[chunk.length - 1].type === 'END' ) { |
| | 806 | + if ( chunk.length && chunk[chunk.length - 1].constructor === EOFTk ) { |
| 825 | 807 | chunk.pop(); |
| 826 | 808 | } |
| 827 | 809 | cb( chunk, true ); |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.defines.js |
| — | — | @@ -18,7 +18,6 @@ |
| 19 | 19 | TagTk.prototype.toString = toString; |
| 20 | 20 | |
| 21 | 21 | function EndTagTk( name, attribs ) { |
| 22 | | - //this.type = 'ENDTAG'; |
| 23 | 22 | this.name = name; |
| 24 | 23 | this.attribs = attribs || []; |
| 25 | 24 | } |
| — | — | @@ -52,7 +51,6 @@ |
| 53 | 52 | NlTk.prototype.toString = toString; |
| 54 | 53 | |
| 55 | 54 | function CommentTk( value ) { |
| 56 | | - this.type = 'COMMENT'; |
| 57 | 55 | this.value = value; |
| 58 | 56 | } |
| 59 | 57 | CommentTk.prototype = new Object(); |
| — | — | @@ -62,9 +60,7 @@ |
| 63 | 61 | CommentTk.prototype.constructor = CommentTk; |
| 64 | 62 | CommentTk.prototype.toString = toString; |
| 65 | 63 | |
| 66 | | -function EOFTk( ) { |
| 67 | | - this.type = 'END'; |
| 68 | | -} |
| | 64 | +function EOFTk( ) { } |
| 69 | 65 | EOFTk.prototype = new Object(); |
| 70 | 66 | EOFTk.prototype.toJSON = function () { |
| 71 | 67 | return $.extend( { type: 'EOFTk' }, this ); |
| Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt |
| — | — | @@ -354,16 +354,6 @@ |
| 355 | 355 | start |
| 356 | 356 | = e:toplevelblock* newline* { |
| 357 | 357 | // end is passed inline as a token, as well as a separate event for now. |
| 358 | | - |
| 359 | | - // this does not work yet. |
| 360 | | - //console.warn('about to emit' + pp(self)); |
| 361 | | - //self._tokenizer.emit('chunk', [ { type: 'END' } ] ); |
| 362 | | - //self._tokenizer.emit('end'); |
| 363 | | - // Append the end (for obvious reasons this should not |
| 364 | | - // be part of a stream, only when tokenizing complete |
| 365 | | - // texts) |
| 366 | | - //console.warn( pp( flatten ( e ) ) ); |
| 367 | | - cache = {}; |
| 368 | 358 | __parseArgs[2]( [ new EOFTk( ) ] ); |
| 369 | 359 | return []; //flatten(e); |
| 370 | 360 | } |
| — | — | @@ -550,7 +540,7 @@ |
| 551 | 541 | comment |
| 552 | 542 | = '<!--' c:comment_chars* ('-->' / eof) |
| 553 | 543 | cs:(space* newline space* cn:comment { return cn })* { |
| 554 | | - return [{ type: 'COMMENT', value: c.join('') }].concat(cs); |
| | 544 | + return [new CommentTk( c.join('') )].concat(cs); |
| 555 | 545 | } |
| 556 | 546 | |
| 557 | 547 | comment_chars |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.LinkHandler.js |
| — | — | @@ -170,9 +170,9 @@ |
| 171 | 171 | var token = tokens[i]; |
| 172 | 172 | if ( token.constructor === String ) { |
| 173 | 173 | s += token; |
| 174 | | - } else if ( token.type === 'NEWLINE' ) { |
| | 174 | + } else if ( token.constructor === NlTk ) { |
| 175 | 175 | s += '\n'; // XXX: preserve original newline |
| 176 | | - } else if ( token.type === 'COMMENT' ) { |
| | 176 | + } else if ( token.constructor === CommentTk ) { |
| 177 | 177 | // strip it |
| 178 | 178 | } else { |
| 179 | 179 | var res = this.imageParser.processImageOptions( s, 'img_options' ), |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.Cite.js |
| — | — | @@ -125,7 +125,7 @@ |
| 126 | 126 | return tokenCTX; |
| 127 | 127 | } else if ( this.isActive && |
| 128 | 128 | // Also accept really broken ref close tags.. |
| 129 | | - ['TAG', 'ENDTAG', 'SELFCLOSINGTAG'].indexOf(token.type) >= 0 && |
| | 129 | + [TagTk, EndTagTk, SelfclosingTagTk].indexOf(token.constructor) >= 0 && |
| 130 | 130 | token.name.toLowerCase() === 'ref' |
| 131 | 131 | ) |
| 132 | 132 | { |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js |
| — | — | @@ -280,7 +280,7 @@ |
| 281 | 281 | // Strip 'end' tokens and trailing newlines |
| 282 | 282 | var l = res[res.length - 1]; |
| 283 | 283 | while ( res.length && |
| 284 | | - ( l.type === 'END' || l.constructor === NlTk ) |
| | 284 | + ( l.constructor === EOFTk || l.constructor === NlTk ) |
| 285 | 285 | ) |
| 286 | 286 | { |
| 287 | 287 | this.manager.env.dp( 'TemplateHandler, stripping end or whitespace tokens' ); |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.PostExpandParagraphHandler.js |
| — | — | @@ -73,7 +73,7 @@ |
| 74 | 74 | PostExpandParagraphHandler.prototype.onAny = function ( token, frame, cb ) { |
| 75 | 75 | //console.warn( 'PostExpandParagraphHandler.onAny' ); |
| 76 | 76 | this.tokens.push( token ); |
| 77 | | - if ( token.type === 'COMMENT' || |
| | 77 | + if ( token.constructor === CommentTk || |
| 78 | 78 | ( token.constructor === String && token.match( /^[\t ]+$/ ) ) |
| 79 | 79 | ) |
| 80 | 80 | { |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js |
| — | — | @@ -194,7 +194,7 @@ |
| 195 | 195 | } |
| 196 | 196 | if ( token.constructor === String ) { |
| 197 | 197 | out.push( token ); |
| 198 | | - } else if ( token.type === 'COMMENT' || token.type === 'NEWLINE' ) { |
| | 198 | + } else if ( token.constructor === CommentTk || token.constructor === NlTk ) { |
| 199 | 199 | // strip comments and newlines |
| 200 | 200 | } else { |
| 201 | 201 | if ( strict ) { |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.util.TokenCollector.js |
| — | — | @@ -61,7 +61,7 @@ |
| 62 | 62 | this.tokens.push ( token ); |
| 63 | 63 | this.isActive = false; |
| 64 | 64 | this.manager.removeTransform( this.rank + this._anyDelta, 'any' ); |
| 65 | | - if ( token.type !== 'END' || this.toEnd ) { |
| | 65 | + if ( token.constructor !== EOFTk || this.toEnd ) { |
| 66 | 66 | // end token |
| 67 | 67 | res = this.transformation ( this.tokens, this.cb, this.manager ); |
| 68 | 68 | this.tokens = []; |
| — | — | @@ -75,7 +75,7 @@ |
| 76 | 76 | this.tokens = []; |
| 77 | 77 | return { tokens: res }; |
| 78 | 78 | } |
| 79 | | - } else if ( token.type !== 'END' ) { |
| | 79 | + } else if ( token.constructor !== EOFTk ) { |
| 80 | 80 | this.manager.env.dp( 'starting collection on ', token ); |
| 81 | 81 | // start collection |
| 82 | 82 | this.tokens.push ( token ); |