| Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt |
| — | — | @@ -338,61 +338,9 @@ |
| 339 | 339 | / ' ' & ':' { return "\u00a0"; } |
| 340 | 340 | / t:text_char )+ |
| 341 | 341 | |
| 342 | | -directive |
| 343 | | - = comment |
| 344 | | - / tplarg_or_template |
| 345 | | - / htmlentity |
| 346 | 342 | |
| 347 | | -spaceless_preprocessor_text |
| 348 | | - = r:( t:[^'<~[{\n\r|!\]}\t &=]+ { return t.join(''); } |
| 349 | | - / directive |
| 350 | | - / !inline_breaks !' ' text_char )+ { |
| 351 | | - return flatten_string ( r ); |
| 352 | | - } |
| 353 | 343 | |
| 354 | | -link_preprocessor_text |
| 355 | | - = r:( t:[^'<~[{\n\r|!\]}\t &="']+ { return t.join(''); } |
| 356 | | - / directive |
| 357 | | - / urlencoded_char |
| 358 | | - / !inline_breaks no_punctuation_char |
| 359 | | - / s:[.:,] !(space / eolf) { return s } |
| 360 | | - / [&%] )+ { |
| 361 | | - return flatten_string ( r ); |
| 362 | | - } |
| 363 | 344 | |
| 364 | | -attribute_preprocessor_text |
| 365 | | - = r:( ts:(!inline_breaks t:[^=<>{\n\r&'"\t ] {return t})+ { return ts.join(''); } |
| 366 | | - / directive |
| 367 | | - / !inline_breaks [&%] )* { |
| 368 | | - //console.log('prep'); |
| 369 | | - return flatten_string ( r ); |
| 370 | | - } |
| 371 | | -attribute_preprocessor_text_single |
| 372 | | - = r:( t:[^>{\n\r&']+ { return t.join(''); } |
| 373 | | - / directive |
| 374 | | - / !inline_breaks [&%] )* { |
| 375 | | - return flatten_string ( r ); |
| 376 | | - } |
| 377 | | -attribute_preprocessor_text_double |
| 378 | | - = r:( t:[^>{\n\r&"]+ { return t.join(''); } |
| 379 | | - / directive |
| 380 | | - / !inline_breaks [&%] )* { |
| 381 | | - //console.log( 'double:' + pp(r) ); |
| 382 | | - return flatten_string ( r ); |
| 383 | | - } |
| 384 | | - |
| 385 | | -// Plain text, but can contain templates, template arguments, comments etc- |
| 386 | | -// all stuff that is normally handled by the preprocessor |
| 387 | | -// Returns either a list of tokens, or a plain string (if nothing is to be |
| 388 | | -// processed). |
| 389 | | -preprocessor_text |
| 390 | | - = r:( t:[^<~[{\n\r\t|!\]} &=]+ { return t.join(''); } |
| 391 | | - / directive |
| 392 | | - / !inline_breaks text_char )+ { |
| 393 | | - return flatten ( r ); |
| 394 | | - } |
| 395 | | - |
| 396 | | - |
| 397 | 345 | /* |
| 398 | 346 | '//', // for protocol-relative URLs, but not in text! |
| 399 | 347 | 'ftp://', |
| — | — | @@ -480,6 +428,67 @@ |
| 481 | 429 | |
| 482 | 430 | eolf = newline / eof |
| 483 | 431 | |
| | 432 | + |
| | 433 | +// 'Preprocessor' directive- higher-level things that can occur in otherwise |
| | 434 | +// plain-text content. |
| | 435 | +directive |
| | 436 | + = comment |
| | 437 | + / tplarg_or_template |
| | 438 | + / htmlentity |
| | 439 | + |
| | 440 | +// Plain text, but can contain templates, template arguments, comments etc- |
| | 441 | +// all stuff that is normally handled by the preprocessor |
| | 442 | +// Returns either a list of tokens, or a plain string (if nothing is to be |
| | 443 | +// processed). |
| | 444 | +preprocessor_text |
| | 445 | + = r:( t:[^<~[{\n\r\t|!\]} &=]+ { return t.join(''); } |
| | 446 | + / directive |
| | 447 | + / !inline_breaks text_char )+ { |
| | 448 | + return flatten ( r ); |
| | 449 | + } |
| | 450 | + |
| | 451 | +spaceless_preprocessor_text |
| | 452 | + = r:( t:[^'<~[{\n\r|!\]}\t &=]+ { return t.join(''); } |
| | 453 | + / directive |
| | 454 | + / !inline_breaks !' ' text_char )+ { |
| | 455 | + return flatten_string ( r ); |
| | 456 | + } |
| | 457 | + |
| | 458 | +link_preprocessor_text |
| | 459 | + = r:( t:[^'<~[{\n\r|!\]}\t &="']+ { return t.join(''); } |
| | 460 | + / directive |
| | 461 | + / urlencoded_char |
| | 462 | + / !inline_breaks no_punctuation_char |
| | 463 | + / s:[.:,] !(space / eolf) { return s } |
| | 464 | + / [&%] )+ { |
| | 465 | + return flatten_string ( r ); |
| | 466 | + } |
| | 467 | + |
| | 468 | +// Attribute values with preprocessor support |
| | 469 | +attribute_preprocessor_text |
| | 470 | + = r:( ts:(!inline_breaks t:[^=<>{\n\r&'"\t ] {return t})+ { return ts.join(''); } |
| | 471 | + / directive |
| | 472 | + / !inline_breaks [&%] )* { |
| | 473 | + //console.log('prep'); |
| | 474 | + return flatten_string ( r ); |
| | 475 | + } |
| | 476 | +attribute_preprocessor_text_single |
| | 477 | + = r:( t:[^{&']+ { return t.join(''); } |
| | 478 | + / directive |
| | 479 | + / !inline_breaks [{&] )* { |
| | 480 | + return flatten_string ( r ); |
| | 481 | + } |
| | 482 | +attribute_preprocessor_text_double |
| | 483 | + = r:( t:[^{&"]+ { return t.join(''); } |
| | 484 | + / directive |
| | 485 | + / !inline_breaks [{&] )* { |
| | 486 | + //console.log( 'double:' + pp(r) ); |
| | 487 | + return flatten_string ( r ); |
| | 488 | + } |
| | 489 | + |
| | 490 | + |
| | 491 | +// A document (start production) is a sequence of toplevelblocks. Tokens are |
| | 492 | +// emitted in chunks per toplevelblock to avoid buffering the full document. |
| 484 | 493 | toplevelblock |
| 485 | 494 | = & { blockStart = pos; return true; } b:block { |
| 486 | 495 | b = flatten(b); |
| — | — | @@ -496,14 +505,13 @@ |
| 497 | 506 | bs.attribs.push(new KV('data-sourcePos', blockStart + ':' + pos)); |
| 498 | 507 | //console.log( 'toplevelblock: ' + pp( bs )); |
| 499 | 508 | } |
| 500 | | - // XXX: only run this for lines that actually need it! |
| 501 | | - //b.push({type: 'NEWLINE'}); |
| 502 | | - // Move this to a token stream transform! |
| 503 | | - //console.log('about to emit' + pp(self)); |
| 504 | | - //console.log( '__parseArgs call: ' + pp( b )); |
| | 509 | + |
| | 510 | + // Emit tokens for this toplevelblock. This feeds a chunk to the parser |
| | 511 | + // pipeline. |
| 505 | 512 | __parseArgs[2]( flatten( b ) ); |
| 506 | 513 | |
| 507 | | - //return []; |
| | 514 | + // We don't return any tokens to the start production to save memory. We |
| | 515 | + // just emitted them already to our consumers. |
| 508 | 516 | return true; |
| 509 | 517 | } |
| 510 | 518 | |
| — | — | @@ -549,6 +557,9 @@ |
| 550 | 558 | / pre |
| 551 | 559 | |
| 552 | 560 | |
| | 561 | +// A paragraph. We don't emit 'p' tokens to avoid issues with template |
| | 562 | +// transclusions, <p> tags in the source and the like. Instead, we perform |
| | 563 | +// some paragraph wrapping on the DOM. |
| 553 | 564 | para |
| 554 | 565 | = s1:sol s2:sol c:inlineline { |
| 555 | 566 | return s1.concat(s2, /* [new TagTk('p')],*/ c); |
| — | — | @@ -896,13 +907,13 @@ |
| 897 | 908 | } |
| 898 | 909 | } else { |
| 899 | 910 | if (trail) { |
| 900 | | - textTokens = target.concat( [ trail.join('') ] ); |
| | 911 | + textTokens = $.extend(true, [], target).concat( [ trail.join('') ] ); |
| 901 | 912 | } else { |
| 902 | 913 | // copy list |
| 903 | | - textTokens = target.concat([]); |
| | 914 | + textTokens = $.extend(true, [], target); |
| 904 | 915 | } |
| 905 | 916 | } |
| 906 | | - //console.log( "XXX:" + pp(obj) ); |
| | 917 | + //console.log( "XXX:" + pp([obj].concat(textTokens, [new EndTagTk( 'a' )])) ); |
| 907 | 918 | return [obj].concat(textTokens, [new EndTagTk( 'a' )]); |
| 908 | 919 | } |
| 909 | 920 | |
| Index: trunk/extensions/VisualEditor/modules/parser/ext.core.AttributeExpander.js |
| — | — | @@ -1,8 +1,7 @@ |
| 2 | 2 | /** |
| 3 | | - * Geniric attribute expansion handler. |
| | 3 | + * Generic attribute expansion handler. |
| 4 | 4 | * |
| 5 | 5 | * @author Gabriel Wicke <gwicke@wikimedia.org> |
| 6 | | - * @author Brion Vibber <brion@wikimedia.org> |
| 7 | 6 | */ |
| 8 | 7 | var $ = require('jquery'), |
| 9 | 8 | request = require('request'), |
| — | — | @@ -77,7 +76,8 @@ |
| 78 | 77 | AttributeExpander.prototype._returnAttributes = function ( expandData, |
| 79 | 78 | attributes ) |
| 80 | 79 | { |
| 81 | | - this.manager.env.dp( 'AttributeExpander._returnAttributes: ' + JSON.stringify(attributes) ); |
| | 80 | + this.manager.env.dp( 'AttributeExpander._returnAttributes: ' + |
| | 81 | + JSON.stringify(attributes) ); |
| 82 | 82 | // Remove the target from the attributes |
| 83 | 83 | expandData.token.attribs = attributes; |
| 84 | 84 | if ( expandData.async ) { |