| Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformer.js |
| — | — | @@ -1,6 +1,21 @@ |
| 2 | 2 | /* Generic token transformation dispatcher with support for asynchronous token |
| 3 | 3 | * expansion. Individual transformations register for the token types they are |
| 4 | | - * interested in and are called on each matching token. */ |
| | 4 | + * interested in and are called on each matching token. |
| | 5 | + * |
| | 6 | + * A transformer might return null, a single token, or an array of tokens. |
| | 7 | + * - Null removes the token and stops further processing for this token. |
| | 8 | + * - A single token is further processed using the remaining transformations |
| | 9 | + * registered for this token, and finally placed in the output token list. |
| | 10 | + * - A list of tokens stops the processing for this token. Instead, processing |
| | 11 | + * restarts with the first returned token. |
| | 12 | + * |
| | 13 | + * Additionally, transformers performing asynchronous actions on a token can |
| | 14 | + * create a new TokenAccumulator using .newAccumulator(). This creates a new |
| | 15 | + * accumulator for each asynchronous result, with the asynchronously processed |
| | 16 | + * token last in its internal accumulator. This setup avoids the need to apply |
| | 17 | + * operational-transform-like index transformations when parallel expansions |
| | 18 | + * insert tokens in front of other ongoing expansion tasks. |
| | 19 | + * */ |
| 5 | 20 | |
| 6 | 21 | function TokenTransformer( callback ) { |
| 7 | 22 | this.cb = callback; // Called with transformed token list when done |
| — | — | @@ -60,62 +75,121 @@ |
| 61 | 76 | } |
| 62 | 77 | }; |
| 63 | 78 | |
| 64 | | -TokenTransformer.prototype._transformTagToken = function ( token, lastToken ) { |
| | 79 | +/* Constructor for information context relevant to token transformers |
| | 80 | + * |
| | 81 | + * @param token The token to precess |
| | 82 | + * @param accum {TokenAccumulator} The active TokenAccumulator. |
| | 83 | + * @param processor {TokenTransformer} The TokenTransformer object. |
| | 84 | + * @param lastToken Last returned token or {undefined}. |
| | 85 | + * @returns {TokenContext}. |
| | 86 | + */ |
| | 87 | +function TokenContext ( token, accum, transformer, lastToken ) { |
| | 88 | + this.token = token; |
| | 89 | + this.accum = accum; |
| | 90 | + this.transformer = transformer; |
| | 91 | + this.lastToken = lastToken; |
| | 92 | + return this; |
| | 93 | +} |
| | 94 | + |
| | 95 | +/* Call all transformers on a tag. |
| | 96 | + * |
| | 97 | + * @param {TokenContext} The current token and its context. |
| | 98 | + * @returns {TokenContext} Context with updated token and/or accum. |
| | 99 | + */ |
| | 100 | +TokenTransformer.prototype._transformTagToken = function ( tokenCTX ) { |
| 65 | 101 | var ts = this.transformers.tag[token.name]; |
| 66 | 102 | if ( ts ) { |
| 67 | 103 | for (var i = 0, l = ts.length; i < l; i++ ) { |
| 68 | | - token = ts[i]( token, lastToken, this.accum, this ); |
| 69 | | - if ( token === null ) { |
| | 104 | + // Transform token with side effects |
| | 105 | + tokenCTX = ts[i]( tokenCTX ); |
| | 106 | + if ( tokenCTX.token === null || $.isArray(tokenCTX.token) ) { |
| 70 | 107 | break; |
| 71 | 108 | } |
| | 109 | + |
| 72 | 110 | } |
| 73 | 111 | } |
| 74 | | - return token; |
| | 112 | + return tokenCTX; |
| 75 | 113 | }; |
| 76 | 114 | |
| 77 | | -TokenTransformer.prototype._transformToken = function ( ts, token, lastToken ) { |
| | 115 | +/* Call all transformers on other token types. |
| | 116 | + * |
| | 117 | + * @param tokenCTX {TokenContext} The current token and its context. |
| | 118 | + * @param ts List of token transformers for this token type. |
| | 119 | + * @returns {TokenContext} Context with updated token and/or accum. |
| | 120 | + */ |
| | 121 | +TokenTransformer.prototype._transformToken = function ( tokenCTX, ts ) { |
| 78 | 122 | if ( ts ) { |
| 79 | 123 | for (var i = 0, l = ts.length; i < l; i++ ) { |
| 80 | | - token = ts[i]( token, lastToken, this.accum, this ); |
| 81 | | - if ( token === null ) { |
| | 124 | + tokenCTX = ts[i]( tokenCTX ); |
| | 125 | + if ( tokenCTX.token === null || $.isArray(tokenCTX.token) ) { |
| 82 | 126 | break; |
| 83 | 127 | } |
| 84 | 128 | } |
| 85 | 129 | } |
| 86 | | - return token; |
| | 130 | + return tokenCTX; |
| 87 | 131 | }; |
| 88 | 132 | |
| 89 | | -TokenTransformer.prototype.transformTokens = function ( tokens ) { |
| 90 | | - var currentOutout = []; |
| 91 | | - var lastToken; |
| | 133 | +/* Transform and expand tokens. |
| | 134 | + * |
| | 135 | + * Normally called with undefined accum. Asynchronous expansions will call |
| | 136 | + * this with their known accum, which allows expanded tokens to be spliced in |
| | 137 | + * at the appropriate location in the token list, which is always at the tail |
| | 138 | + * end of the current accumulator. |
| | 139 | + * |
| | 140 | + * @param tokens {List of tokens} Tokens to process. |
| | 141 | + * @param accum {TokenAccumulator} object. Undefined for first call, set to |
| | 142 | + * accumulator with expanded token at tail for asynchronous |
| | 143 | + * expansions. |
| | 144 | + * @returns nothing: Calls back registered callback if there are no more |
| | 145 | + * outstanding asynchronous expansions. |
| | 146 | + * */ |
| | 147 | +TokenTransformer.prototype.transformTokens = function ( tokens, accum ) { |
| | 148 | + if ( accum === undefined ) { |
| | 149 | + accum = this.accum; |
| | 150 | + } else { |
| | 151 | + // Prepare to replace the last token in the current accumulator. |
| | 152 | + accum.pop(); |
| | 153 | + } |
| | 154 | + var tokenCTX = TokenContext(undefined, accum, this, undefined); |
| 92 | 155 | for ( var i = 0, l = tokens.length; i < l; i++ ) { |
| 93 | | - var token = tokens[i]; |
| | 156 | + tokenCTX.lastToken = tokenCTX.token; |
| | 157 | + tokenCTX.token = tokens[i]; |
| 94 | 158 | var ts; |
| 95 | | - switch(token.type) { |
| | 159 | + switch(tokenCTX.token.type) { |
| 96 | 160 | case 'TAG': |
| 97 | 161 | case 'ENDTAG': |
| 98 | 162 | case 'SELFCLOSINGTAG': |
| 99 | | - lastToken = this._transformTagToken( token, lastToken ); |
| | 163 | + tokenCTX = this._transformTagToken( tokenCTX ); |
| 100 | 164 | break; |
| 101 | 165 | case 'TEXT': |
| 102 | | - lastToken = this._transformToken(this.transformers.text, token, lastToken); |
| | 166 | + tokenCTX = this._transformToken( tokenCTX, this.transformers.text ); |
| 103 | 167 | break; |
| 104 | 168 | case 'COMMENT': |
| 105 | | - lastToken = this._transformToken(this.transformers.comment, token, lastToken); |
| | 169 | + tokenCTX = this._transformToken( tokenCTX, this.transformers.comment); |
| 106 | 170 | break; |
| 107 | 171 | case 'NEWLINE': |
| 108 | | - lastToken = this._transformToken(this.transformers.newline, token, lastToken); |
| | 172 | + tokenCTX = this._transformToken( tokenCTX, this.transformers.newline ); |
| 109 | 173 | break; |
| 110 | 174 | case 'END': |
| 111 | | - lastToken = this._transformToken(this.transformers.end, token, lastToken); |
| | 175 | + tokenCTX = this._transformToken( tokenCTX, this.transformers.end ); |
| 112 | 176 | break; |
| 113 | 177 | default: |
| 114 | | - lastToken = this._transformToken(this.transformers.martian, token, lastToken); |
| | 178 | + tokenCTX = this._transformToken( tokenCTX, this.transformers.martian ); |
| 115 | 179 | break; |
| 116 | 180 | } |
| 117 | | - if(lastToken) { |
| 118 | | - this.accum.push(lastToken); |
| | 181 | + if( $.isArray(tokenCTX.token) ) { |
| | 182 | + // Splice in the returned tokens (while replacing the original |
| | 183 | + // token), and process them next. |
| | 184 | + [].splice.apply(tokens, [i, 1].concat(tokenCTX.token)); |
| | 185 | + l += res.token.length - 1; |
| | 186 | + i--; // continue at first inserted token |
| | 187 | + } else if (tokenCTX.token) { |
| | 188 | + // push to accumulator (not necessarily the last one) |
| | 189 | + accum.push(tokenCTX.token); |
| 119 | 190 | } |
| | 191 | + // Update current accum, in case a new one was spliced in by a |
| | 192 | + // transformation starting asynch work. |
| | 193 | + accum = tokenCTX.accum; |
| 120 | 194 | } |
| 121 | 195 | this.finish(); |
| 122 | 196 | }; |
| — | — | @@ -123,23 +197,22 @@ |
| 124 | 198 | TokenTransformer.prototype.finish = function ( ) { |
| 125 | 199 | this.outstanding--; |
| 126 | 200 | if ( this.outstanding === 0 ) { |
| 127 | | - // Join back the token accumulators into a single token list |
| | 201 | + // Join the token accumulators back into a single token list |
| 128 | 202 | var a = this.firstaccum; |
| 129 | 203 | var accums = [a.accum]; |
| 130 | 204 | while ( a.next !== undefined ) { |
| 131 | 205 | a = a.next; |
| 132 | | - accums.push(a.accum); |
| | 206 | + accums.concat(a.accum); |
| 133 | 207 | } |
| 134 | | - // Call our callback with the token list |
| 135 | | - this.cb(accums.join()); |
| | 208 | + // Call our callback with the flattened token list |
| | 209 | + this.cb(accums); |
| 136 | 210 | } |
| 137 | 211 | }; |
| 138 | 212 | |
| 139 | | - |
| 140 | | - |
| 141 | | -// Start a new accumulator with the given tokens. |
| 142 | | -TokenTransformer.prototype.newAccumulator = function ( tokens ) { |
| 143 | | - this.accum = this.accum.spliceAccumulator( tokens ); |
| | 213 | +/* Start a new accumulator for asynchronous work. */ |
| | 214 | +TokenTransformer.prototype.newAccumulator = function ( ) { |
| | 215 | + this.outstanding++; |
| | 216 | + return this.accum.insertAccumulator( ); |
| 144 | 217 | }; |
| 145 | 218 | |
| 146 | 219 | // Token accumulators in a linked list. Using a linked list simplifies async |
| — | — | @@ -156,7 +229,11 @@ |
| 157 | 230 | this.accum.push(token); |
| 158 | 231 | }; |
| 159 | 232 | |
| 160 | | -TokenAccumulator.prototype.spliceAccumulator = function ( tokens ) { |
| | 233 | +TokenAccumulator.prototype.pop = function ( ) { |
| | 234 | + return this.accum.pop(); |
| | 235 | +}; |
| | 236 | + |
| | 237 | +TokenAccumulator.prototype.insertAccumulator = function ( ) { |
| 161 | 238 | this.next = new TokenAccumulator(this.next, tokens); |
| 162 | 239 | return this.next; |
| 163 | 240 | }; |