| Index: trunk/extensions/VisualEditor/modules/parser/pegTokenizer.pegjs.txt |
| — | — | @@ -401,7 +401,8 @@ |
| 402 | 402 | |
| 403 | 403 | start |
| 404 | 404 | = e:toplevelblock* newline* { |
| 405 | | - return flatten(e); |
| | 405 | + // XXX: move doQuotes out to general token stream transformer |
| | 406 | + return doQuotes(flatten(e)); |
| 406 | 407 | } |
| 407 | 408 | |
| 408 | 409 | |
| — | — | @@ -505,6 +506,8 @@ |
| 506 | 507 | newline |
| 507 | 508 | = '\n' / '\r\n' |
| 508 | 509 | |
| | 510 | +newlineToken = newline { return [{type: 'NEWLINE'}] } |
| | 511 | + |
| 509 | 512 | eolf = newline / eof |
| 510 | 513 | |
| 511 | 514 | toplevelblock |
| — | — | @@ -517,8 +520,8 @@ |
| 518 | 521 | } |
| 519 | 522 | bs.attribs.push(['data-sourcePos', blockStart + ':' + pos]); |
| 520 | 523 | // XXX: only run this for lines that actually need it! |
| 521 | | - b.push({type: 'NEWLINE'}); |
| 522 | | - b = doQuotes(b); |
| | 524 | + //b.push({type: 'NEWLINE'}); |
| | 525 | + // Move this to a token stream transform! |
| 523 | 526 | return b; |
| 524 | 527 | } |
| 525 | 528 | |
| — | — | @@ -1084,7 +1087,7 @@ |
| 1085 | 1088 | |
| 1086 | 1089 | /* Tables */ |
| 1087 | 1090 | table |
| 1088 | | - = tas:table_start space* c:table_caption? b:table_body? table_end { |
| | 1091 | + = tas:table_start space* c:table_caption? b:table_body? te:table_end { |
| 1089 | 1092 | var res = {type: 'TAG', name: 'table'} |
| 1090 | 1093 | var body = b !== '' ? b : []; |
| 1091 | 1094 | dp("body: " + pp(body)); |
| — | — | @@ -1096,7 +1099,7 @@ |
| 1097 | 1100 | |
| 1098 | 1101 | if (c != '') { |
| 1099 | 1102 | var caption = [{type: 'TAG', name: 'caption'}] |
| 1100 | | - .concat(c, [{type: 'ENDTAG', name: 'caption'}]); |
| | 1103 | + .concat(c, [{type: 'ENDTAG', name: 'caption'}], te); |
| 1101 | 1104 | } else { |
| 1102 | 1105 | var caption = []; |
| 1103 | 1106 | } |
| — | — | @@ -1122,9 +1125,9 @@ |
| 1123 | 1126 | = text / ! inline_breaks !newline ![|] c:. { return c } |
| 1124 | 1127 | |
| 1125 | 1128 | table_caption |
| 1126 | | - = newline |
| | 1129 | + = n:newlineToken |
| 1127 | 1130 | "|+" c:inline* { |
| 1128 | | - return c; |
| | 1131 | + return n.concat(c); |
| 1129 | 1132 | } |
| 1130 | 1133 | |
| 1131 | 1134 | table_body |
| — | — | @@ -1148,15 +1151,15 @@ |
| 1149 | 1152 | |
| 1150 | 1153 | table_row |
| 1151 | 1154 | = //& { dp("table row enter"); return true; } |
| 1152 | | - newline |
| | 1155 | + n:newlineToken |
| 1153 | 1156 | "|-" thtd_attribs? space* td:(table_data / table_header)* { |
| 1154 | | - return [{type: 'TAG', name: 'tr'}] |
| 1155 | | - .concat(td, [{type: 'ENDTAG', name: 'tr'}]); |
| | 1157 | + return n.concat([{type: 'TAG', name: 'tr'}] |
| | 1158 | + , td, [{type: 'ENDTAG', name: 'tr'}]); |
| 1156 | 1159 | } |
| 1157 | 1160 | |
| 1158 | 1161 | table_data |
| 1159 | 1162 | = //& { dp("table_data enter, pos=" + pos + input.substr(pos,10)); return true; } |
| 1160 | | - ("||" / newline "|") |
| | 1163 | + n:("||" { return [] } / nt:newlineToken "|" { return nt }) |
| 1161 | 1164 | ! [}+-] |
| 1162 | 1165 | //& { dp('before attrib, pos=' + pos); return true; } |
| 1163 | 1166 | a:(as:generic_attribute+ space* "|" !"|" { return as } )? |
| — | — | @@ -1169,19 +1172,19 @@ |
| 1170 | 1173 | a = []; |
| 1171 | 1174 | } |
| 1172 | 1175 | //dp("table data result: " + pp(td) + ", attribts: " + pp(a)); |
| 1173 | | - return [{ type: 'TAG', name: 'td', attribs: a}] |
| 1174 | | - .concat(td, [{type: 'ENDTAG', name: 'td'}]); |
| | 1176 | + return n.concat( [{ type: 'TAG', name: 'td', attribs: a}] |
| | 1177 | + , td, [{type: 'ENDTAG', name: 'td'}] ); |
| 1175 | 1178 | } |
| 1176 | 1179 | |
| 1177 | 1180 | table_header |
| 1178 | | - = ("!!" / newline "!") |
| | 1181 | + = n:("!!" { return [] } / nl:newlineToken "!" { return nl }) |
| 1179 | 1182 | a:(as:generic_attribute+ "!" !"!" { return as } )? |
| 1180 | 1183 | c:inline { |
| 1181 | 1184 | if ( a == '' ) { |
| 1182 | 1185 | a = []; |
| 1183 | 1186 | } |
| 1184 | | - return [{type: 'TAG', name: 'th', attribs: a}] |
| 1185 | | - .concat(c, [{type: 'ENDTAG', name: 'th'}]); |
| | 1187 | + return n.concat( [{type: 'TAG', name: 'th', attribs: a}] |
| | 1188 | + , c, [{type: 'ENDTAG', name: 'th'}]); |
| 1186 | 1189 | } |
| 1187 | 1190 | |
| 1188 | 1191 | thtd_attribs |
| — | — | @@ -1192,8 +1195,13 @@ |
| 1193 | 1196 | |
| 1194 | 1197 | |
| 1195 | 1198 | table_end |
| 1196 | | - = newline? "|}" { clearFlag('table'); } |
| 1197 | | - / newline? eof |
| | 1199 | + = nt:newlineToken? ( "|}" / eof ) { |
| | 1200 | + clearFlag('table'); |
| | 1201 | + if(nt) |
| | 1202 | + return nt; |
| | 1203 | + else |
| | 1204 | + return []; |
| | 1205 | + } |
| 1198 | 1206 | |
| 1199 | 1207 | |
| 1200 | 1208 | /* Tabs do not mix well with the hybrid production syntax */ |