| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformDispatcher.js |
| — | — | @@ -53,12 +53,12 @@ |
| 54 | 54 | |
| 55 | 55 | /** |
| 56 | 56 | * Register to a token source, normally the tokenizer. |
| 57 | | - * The event emitter emits an 'tokens' event which contains a chunk of tokens, |
| | 57 | + * The event emitter emits a 'chunk' event with a chunk of tokens, |
| 58 | 58 | * and signals the end of tokens by triggering the 'end' event. |
| 59 | 59 | * |
| 60 | 60 | * @param {Object} EventEmitter token even emitter. |
| 61 | 61 | */ |
| 62 | | -TokenTransformDispatcher.prototype.subscribeToTokenEmitter = function ( tokenEmitter ) { |
| | 62 | +TokenTransformDispatcher.prototype.listenForTokensFrom = function ( tokenEmitter ) { |
| 63 | 63 | tokenEmitter.addListener('chunk', this.transformTokens.bind( this ) ); |
| 64 | 64 | tokenEmitter.addListener('end', this.onEndEvent.bind( this ) ); |
| 65 | 65 | }; |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.HTML5TreeBuilder.node.js |
| — | — | @@ -22,7 +22,11 @@ |
| 23 | 23 | |
| 24 | 24 | FauxHTML5.TreeBuilder.prototype = new events.EventEmitter(); |
| 25 | 25 | |
| 26 | | -FauxHTML5.TreeBuilder.prototype.subscribeToTokenEmitter = function ( emitter ) { |
| | 26 | +/** |
| | 27 | + * Register for (token) 'chunk' and 'end' events from a token emitter, |
| | 28 | + * normally the TokenTransformDispatcher. |
| | 29 | + */ |
| | 30 | +FauxHTML5.TreeBuilder.prototype.listenForTokensFrom = function ( emitter ) { |
| 27 | 31 | emitter.addListener('chunk', this.onChunk.bind( this ) ); |
| 28 | 32 | emitter.addListener('end', this.onEnd.bind( this ) ); |
| 29 | 33 | }; |
| — | — | @@ -41,6 +45,8 @@ |
| 42 | 46 | this.document.body = this.parser |
| 43 | 47 | .document.getElementsByTagName('body')[0]; |
| 44 | 48 | |
| | 49 | + this.emit( 'document', this.document ); |
| | 50 | + |
| 45 | 51 | // XXX: more clean up to allow reuse. |
| 46 | 52 | this.parser.setup(); |
| 47 | 53 | this.processToken({type: 'TAG', name: 'body'}); |
| — | — | @@ -97,6 +103,8 @@ |
| 98 | 104 | // HACK: This should not be needed really. |
| 99 | 105 | this.document.body = this.parser.document.getElementsByTagName('body')[0]; |
| 100 | 106 | } |
| | 107 | + // Emit the document to consumers |
| | 108 | + this.emit('document', this.document); |
| 101 | 109 | break; |
| 102 | 110 | case "NEWLINE": |
| 103 | 111 | //this.emit('end'); |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMPostProcessor.js |
| — | — | @@ -1,5 +1,7 @@ |
| 2 | 2 | /* Perform post-processing steps on an already-built HTML DOM. */ |
| 3 | 3 | |
| | 4 | +var events = require('events'); |
| | 5 | + |
| 4 | 6 | var isBlock = function isBlock (name) { |
| 5 | 7 | switch (name.toLowerCase()) { |
| 6 | 8 | case 'div': |
| — | — | @@ -79,13 +81,25 @@ |
| 80 | 82 | this.processors = [process_inlines_in_p]; |
| 81 | 83 | } |
| 82 | 84 | |
| | 85 | +// Inherit from EventEmitter |
| | 86 | +DOMPostProcessor.prototype = new events.EventEmitter(); |
| | 87 | + |
| 83 | 88 | DOMPostProcessor.prototype.doPostProcess = function ( document ) { |
| 84 | 89 | for(var i = 0; i < this.processors.length; i++) { |
| 85 | 90 | this.processors[i](document); |
| 86 | 91 | } |
| | 92 | + this.emit( 'document', document ); |
| 87 | 93 | }; |
| 88 | 94 | |
| 89 | 95 | |
| | 96 | +/** |
| | 97 | + * Register for the 'document' event, normally emitted form the HTML5 tree |
| | 98 | + * builder. |
| | 99 | + */ |
| | 100 | +DOMPostProcessor.prototype.listenForDocumentFrom = function ( emitter ) { |
| | 101 | + emitter.addListener( 'document', this.doPostProcess.bind( this ) ); |
| | 102 | +} |
| | 103 | + |
| 90 | 104 | if (typeof module == "object") { |
| 91 | 105 | module.exports.DOMPostProcessor = DOMPostProcessor; |
| 92 | 106 | } |
| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.js |
| — | — | @@ -29,52 +29,71 @@ |
| 30 | 30 | |
| 31 | 31 | this.wikiTokenizer = new PegTokenizer(); |
| 32 | 32 | |
| 33 | | - this.tokenDispatcher = new TokenTransformDispatcher (); |
| | 33 | + /** |
| | 34 | + * Token stream transformations. |
| | 35 | + * This is where all the wiki functionality is implemented. |
| | 36 | + * See https://www.mediawiki.org/wiki/Future/Parser_development/Token_stream_transformations |
| | 37 | + */ |
| | 38 | + this.tokenTransformer = new TokenTransformDispatcher (); |
| 34 | 39 | |
| 35 | 40 | // Add token transformations.. |
| 36 | 41 | var qt = new QuoteTransformer(); |
| 37 | | - qt.register(this.tokenDispatcher); |
| | 42 | + qt.register(this.tokenTransformer); |
| 38 | 43 | |
| 39 | 44 | //var citeExtension = new Cite(); |
| 40 | 45 | //citeExtension.register(this.tokenDispatcher); |
| 41 | 46 | |
| 42 | | - this.tokenDispatcher.subscribeToTokenEmitter( this.wikiTokenizer ); |
| | 47 | + this.tokenTransformer.listenForTokensFrom( this.wikiTokenizer ); |
| 43 | 48 | |
| 44 | | - // Create a new tree builder, which also creates a new document. |
| 45 | | - // XXX: implicitly clean up old state after processing end token, so |
| 46 | | - // that we can reuse the tree builder. |
| 47 | | - // XXX: convert to event listener listening for token chunks from the |
| 48 | | - // token transformer and and emitting an additional 'done' event after |
| 49 | | - // processing the 'end' token. |
| | 49 | + /** |
| | 50 | + * The tree builder creates a DOM tree from the token soup emitted from |
| | 51 | + * the TokenTransformDispatcher. |
| | 52 | + */ |
| 50 | 53 | this.treeBuilder = new FauxHTML5.TreeBuilder(); |
| 51 | | - this.treeBuilder.subscribeToTokenEmitter( this.tokenDispatcher ); |
| | 54 | + this.treeBuilder.listenForTokensFrom( this.tokenTransformer ); |
| 52 | 55 | |
| 53 | | - // Prepare these two, but only call them from parse and getWikiDom for |
| 54 | | - // now. These will be called in a callback later, when the full pipeline |
| 55 | | - // is used asynchronously. |
| | 56 | + /** |
| | 57 | + * Final processing on the HTML DOM. |
| | 58 | + */ |
| | 59 | + |
| | 60 | + // Generic DOM transformer. |
| | 61 | + // This currently performs minor tree-dependent clean up like wrapping |
| | 62 | + // plain text in paragraphs. For HTML output, it would also be configured |
| | 63 | + // to perform more aggressive nesting cleanup. |
| 56 | 64 | this.postProcessor = new DOMPostProcessor(); |
| | 65 | + this.postProcessor.listenForDocumentFrom( this.treeBuilder ); |
| 57 | 66 | |
| | 67 | + |
| | 68 | + /** |
| | 69 | + * Conversion from HTML DOM to WikiDOM. This is not needed if plain HTML |
| | 70 | + * DOM output is needed, so it should only be registered to the |
| | 71 | + * DOMPostProcessor 'document' event if WikiDom output is requested. We |
| | 72 | + * could emit events for 'dom', 'wikidom', 'html' and so on, but only |
| | 73 | + * actually set up the needed pipeline stages if a listener is registered. |
| | 74 | + * Overriding the addListener method should make this possible. |
| | 75 | + */ |
| 58 | 76 | this.DOMConverter = new DOMConverter(); |
| | 77 | + |
| | 78 | + |
| | 79 | + // Lame hack for now, see above for an idea for the external async |
| | 80 | + // interface and pipeline setup |
| | 81 | + this.postProcessor.addListener( 'document', this.setDocumentProperty.bind( this ) ); |
| 59 | 82 | } |
| 60 | 83 | |
| 61 | 84 | ParserPipeline.prototype.parse = function ( text ) { |
| 62 | 85 | // Set the pipeline in motion by feeding the tokenizer |
| 63 | 86 | this.wikiTokenizer.tokenize( text ); |
| | 87 | +}; |
| 64 | 88 | |
| 65 | | - // XXX: Convert parse to an async pipeline as well! |
| 66 | | - // The remaining processing below will have to happen in a callback, |
| 67 | | - // triggered on the treeBuilder 'end' event, followed by an event emission |
| 68 | | - // or callback calling instead of returning. |
| 69 | | - this.document = this.treeBuilder.document; |
| | 89 | +// XXX: Lame hack: set document property. Instead, emit events |
| | 90 | +// and convert parser tests etc to listen on it! See comments above for ideas. |
| | 91 | +ParserPipeline.prototype.setDocumentProperty = function ( document ) { |
| | 92 | + this.document = document; |
| | 93 | +} |
| 70 | 94 | |
| 71 | | - //console.log(this.document.body.innerHTML); |
| 72 | 95 | |
| 73 | | - // Perform synchronous post-processing on DOM. |
| 74 | | - // XXX: convert to event listener (listening on treeBuilder 'end' |
| 75 | | - // event) |
| 76 | | - this.postProcessor.doPostProcess( this.document ); |
| 77 | | -}; |
| 78 | | - |
| | 96 | +// XXX: remove JSON serialization here, that should only be performed when |
| | 97 | +// needed. |
| 79 | 98 | ParserPipeline.prototype.getWikiDom = function () { |
| 80 | 99 | return JSON.stringify( |
| 81 | 100 | this.DOMConverter.HTMLtoWiki( this.document.body ), |