| Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt |
| — | — | @@ -1,5 +1,12 @@ |
| 2 | | -/* Combined Wiki (MediaWiki) and HTML tokenizer. Produces a token stream |
| 3 | | -** (actually a list of tokens for now) suitable for a HTML5TreeBuilder. */ |
| | 2 | +/** |
| | 3 | + * Combined Wiki (MediaWiki) and HTML tokenizer based on pegjs. Emits several |
| | 4 | + * chunks of tokens (one chunk per top-level block matched) and eventually an |
| | 5 | + * end event. Tokens map to HTML tags as far as possible, with custom tokens |
| | 6 | + * used where further processing on the token stream is needed. |
| | 7 | + * |
| | 8 | + * @author Gabriel Wicke <gwicke@wikimedia.org> |
| | 9 | + * @author Brion Vibber <brion@wikimedia.org> |
| | 10 | + */ |
| 4 | 11 | { |
| 5 | 12 | /* Fixme: use static functions to separate module! Unfortunately, this |
| 6 | 13 | * does not work: |
| — | — | @@ -586,9 +593,12 @@ |
| 587 | 594 | / & { return syntaxFlags['extlink']; } "]" { return true; } |
| 588 | 595 | / & { return syntaxFlags['linkdesc']; } link_end { return true; } |
| 589 | 596 | / & { return syntaxFlags['h']; } '='+ space* newline { return true; } |
| 590 | | - / & { return syntaxFlags['template']; } ('|' / '}}' ) { return true; } |
| | 597 | + / & { return syntaxFlags['template']; } ('|' / '}}' ) { |
| | 598 | + //console.log( 'template break @' + pos + input.substr(pos-1, 4) ); |
| | 599 | + return true; |
| | 600 | + } |
| 591 | 601 | / & { return syntaxFlags['equal']; } '=' { |
| 592 | | - //console.log( 'equal stop!' ); |
| | 602 | + //console.log( 'equal stop @' + pos + input.substr(pos-1, 4) ); |
| 593 | 603 | return true; |
| 594 | 604 | } |
| 595 | 605 | |
| — | — | @@ -614,7 +624,7 @@ |
| 615 | 625 | if (text.length) { |
| 616 | 626 | out.push( text.join('') ); |
| 617 | 627 | } |
| 618 | | - //dp('inline out:' + pp(out)); |
| | 628 | + //console.log('inline out:' + pp(out)); |
| 619 | 629 | return out; |
| 620 | 630 | } |
| 621 | 631 | |
| — | — | @@ -648,7 +658,7 @@ |
| 649 | 659 | inline_element |
| 650 | 660 | = //& { dp('inline_element enter' + input.substr(pos, 10)); return true; } |
| 651 | 661 | & '<' ( comment / xmlish_tag ) |
| 652 | | - / & '{' ( & '{{{{{' template / tplarg / template ) |
| | 662 | + /// & '{' ( & '{{{{{' template / tplarg / template ) |
| 653 | 663 | / & '{' tplarg_or_template |
| 654 | 664 | /// & '{' ( tplarg / template ) |
| 655 | 665 | // Eat three opening brackets as text. |
| — | — | @@ -849,16 +859,18 @@ |
| 850 | 860 | } |
| 851 | 861 | |
| 852 | 862 | template_param |
| 853 | | - = name:template_param_name space* "=" space* c:template_param_text? { |
| 854 | | - //console.log( 'named template_param matched' + pp([name, flatten( c )]) ); |
| 855 | | - if ( c !== '' ) { |
| 856 | | - return new KV(name, flatten( c )); |
| | 863 | + = name:template_param_name space* |
| | 864 | + value:( s0:"=" s1:space* |
| | 865 | + //& { console.log( 'entering value' ); return true } |
| | 866 | + s2:template_param_text { return [s0, s1, s2] } )? { |
| | 867 | + //console.log( 'named template_param matched' + pp([name, value ]) ); |
| | 868 | + if ( value !== '' ) { |
| | 869 | + return new KV(name, flatten( value[2] ) || []); |
| 857 | 870 | } else { |
| 858 | | - return new KV(name, []); |
| | 871 | + return new KV([], flatten(name)); |
| 859 | 872 | } |
| 860 | | - } / c:template_param_text { |
| 861 | | - return new KV([], flatten( c ) ); |
| 862 | | - } |
| | 873 | + } |
| | 874 | + // empty parameter |
| 863 | 875 | / & [|}] { return new KV([], []); } |
| 864 | 876 | |
| 865 | 877 | |
| — | — | @@ -866,8 +878,8 @@ |
| 867 | 879 | template_param_name |
| 868 | 880 | = & { return setFlag( 'equal' ) } |
| 869 | 881 | tpt:template_param_text |
| 870 | | - & { clearFlag( 'equal' ); return true; } |
| 871 | 882 | { |
| | 883 | + clearFlag( 'equal' ); |
| 872 | 884 | //console.log( 'template param name matched: ' + pp( tpt ) ); |
| 873 | 885 | return tpt; |
| 874 | 886 | } |
| — | — | @@ -879,6 +891,7 @@ |
| 880 | 892 | = & { return setFlag('template') } |
| 881 | 893 | il:inline { |
| 882 | 894 | clearFlag('template'); |
| | 895 | + //console.log( 'tpt match: ' + pp (il)); |
| 883 | 896 | return il; |
| 884 | 897 | } |
| 885 | 898 | / & { return clearFlag('template'); } |
| — | — | @@ -930,12 +943,20 @@ |
| 931 | 944 | |
| 932 | 945 | link_text |
| 933 | 946 | = & { return setFlag('linkdesc'); } |
| 934 | | - h:inlineline |
| | 947 | + h:inline |
| | 948 | + // 'equal' syntaxFlag is set for links in template parameters. Consume the |
| | 949 | + // '=' here. |
| | 950 | + hs:( '=' inline)? |
| 935 | 951 | { |
| | 952 | + //console.log('link_text' + pp(h) + pp(hs)); |
| 936 | 953 | clearFlag('linkdesc'); |
| 937 | | - return h; |
| | 954 | + if( hs !== '' ) { |
| | 955 | + return h.concat(hs); |
| | 956 | + } else { |
| | 957 | + return h; |
| | 958 | + } |
| 938 | 959 | } |
| 939 | | - / & { clearFlag('linkdesc'); return false } |
| | 960 | + / & { return clearFlag('linkdesc'); } |
| 940 | 961 | |
| 941 | 962 | link_end = "]]" |
| 942 | 963 | |