r105876 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r105875‎ \| r105876 \| r105877 >
Date:	14:03, 12 December 2011
Author:	gwicke
Status:	deferred
Tags:
Comment:	Refactor parserTests somewhat into a class-like structure, and wire up the TokenTransformer.
Modified paths:	/trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformer.js (modified) (history) /trunk/extensions/VisualEditor/modules/parser/mediawiki.tokenizer.peg.js (modified) (history) /trunk/extensions/VisualEditor/tests/parser/parserTests.js (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/tests/parser/parserTests.js
—	—	@@ -19,195 +19,228 @@
20	20	jsDiff = require('diff'),
21	21	colors = require('colors'),
22	22	util = require( 'util' ),
23		~~- HTML5 = require('html5').HTML5;~~
	23	+ HTML5 = require('html5').HTML5,
	24	+ PEG = require('pegjs'),
	25	+ // Handle options/arguments with optimist module
	26	+ optimist = require('optimist');
24	27
25		~~-// Name of file used to cache the parser tests cases~~
26		~~-var cache_file = "parserTests.cache";~~
	28	+// @fixme wrap more or this setup in a common module
27	29
28		~~-// XXX: avoid a global here!~~
29		~~-global.PEG = require('pegjs');~~
	30	+// track files imported / required
	31	+var fileDependencies = [];
30	32
31		~~-// Handle options/arguments with optimist module~~
32		~~-var optimist = require('optimist');~~
	33	+// Fetch up some of our wacky parser bits...
33	34
34		~~-var argv = optimist.usage( 'Usage: $0', {~~
	35	+var basePath = path.join(path.dirname(path.dirname(process.cwd())), 'modules');
	36	+function _require(filename) {
	37	+ var fullpath = path.join( basePath, filename );
	38	+ fileDependencies.push( fullpath );
	39	+ return require( fullpath );
	40	+}
	41	+
	42	+function _import(filename, symbols) {
	43	+ var module = _require(filename);
	44	+ symbols.forEach(function(symbol) {
	45	+ global[symbol] = module[symbol];
	46	+ });
	47	+}
	48	+
	49	+
	50	+// For now most modules only need this for $.extend and $.each :)
	51	+global.$ = require('jquery');
	52	+
	53	+var pj = path.join;
	54	+
	55	+// Our code...
	56	+
	57	+var testWhiteList = require('./parserTests-whitelist.js').testWhiteList;
	58	+
	59	+_import(pj('parser', 'mediawiki.tokenizer.peg.js'), ['PegTokenizer']);
	60	+_import(pj('parser', 'mediawiki.parser.environment.js'), ['MWParserEnvironment']);
	61	+_import(pj('parser', 'mediawiki.TokenTransformer.js'), ['TokenTransformer']);
	62	+_import(pj('parser', 'ext.cite.taghook.ref.js'), ['MWRefTagHook']);
	63	+
	64	+_import(pj('parser', 'mediawiki.HTML5TreeBuilder.node.js'), ['FauxHTML5']);
	65	+_import(pj('parser', 'mediawiki.DOMPostProcessor.js'), ['DOMPostProcessor']);
	66	+
	67	+// WikiDom and serializers
	68	+_require(pj('es', 'es.js'));
	69	+_require(pj('es', 'es.Html.js'));
	70	+_require(pj('es', 'serializers', 'es.AnnotationSerializer.js'));
	71	+_require(pj('es', 'serializers', 'es.HtmlSerializer.js'));
	72	+_require(pj('es', 'serializers', 'es.WikitextSerializer.js'));
	73	+_require(pj('es', 'serializers', 'es.JsonSerializer.js'));
	74	+
	75	+
	76	+function ParserTests () {
	77	+
	78	+ this.argv = optimist.usage( 'Usage: $0', {
35	79	'quick': {
36	80	description: 'Suppress diff output of failed tests',
37	81	boolean: true,
38		~~- default: false,~~
	82	+ default: false
39	83	},
40	84	'quiet': {
41	85	description: 'Suppress notification of passed tests (shows only failed tests)',
42	86	boolean: true,
43		~~- default: false,~~
	87	+ default: false
44	88	},
45	89	'color': {
46	90	description: 'Enable color output Ex: --no-color',
47	91	boolean: true,
48		~~- default: true,~~
	92	+ default: true
49	93	},
50	94	'cache': {
51		~~- description: 'Get tests cases from cache file ' + cache_file,~~
	95	+ description: 'Get tests cases from cache file ' + this.cache_file,
52	96	boolean: true,
53		~~- default: false,~~
	97	+ default: false
54	98	},
55	99	'filter': {
56	100	description: 'Only run tests whose descriptions which match given regex',
57		~~- alias: 'regex',~~
	101	+ alias: 'regex'
58	102	},
59	103	'whitelist': {
60	104	description: 'Alternatively compare against manually verified parser output from whitelist',
61	105	default: true,
62		~~- boolean: true,~~
	106	+ boolean: true
63	107	},
64	108	'help': {
65	109	description: 'Show this help message',
66		~~- alias: 'h',~~
	110	+ alias: 'h'
67	111	},
68	112	'disabled': {
69	113	description: 'Run disabled tests (option not implemented)',
70	114	default: false,
71		~~- boolean: true,~~
	115	+ boolean: true
72	116	},
73	117	'printwhitelist': {
74	118	description: 'Print out a whitelist entry for failing tests. Default false.',
75	119	default: false,
76		~~- boolean: true,~~
77		~~- },~~
	120	+ boolean: true
	121	+ }
78	122	}
79	123	).check( function(argv) {
80	124	if( argv.filter === true ) {
81	125	throw "--filter need an argument";
82	126	}
83	127	}
84		~~- ).argv // keep that~~
85		~~- ;~~
	128	+ ).argv; // keep that
86	129
87	130
88		~~-if( argv.help ) {~~
89		~~- optimist.showHelp();~~
90		~~- process.exit( 0 );~~
91		-}
92		~~-var test_filter = null;~~
93		~~-if( argv.filter ) { // null is the default by definition~~
94		~~- try {~~
95		~~- test_filter = new RegExp( argv.filter );~~
96		~~- } catch(e) {~~
97		~~- console.error( "\nERROR> --filter was given an invalid regular expression.");~~
98		~~- console.error( "ERROR> See below for JS engine error:\n" + e + "\n" );~~
99		~~- process.exit( 1 );~~
	131	+ if( this.argv.help ) {
	132	+ optimist.showHelp();
	133	+ process.exit( 0 );
100	134	}
101		~~- console.log( "Filtering title test using Regexp " + test_filter );~~
102		-}
103		~~-if( !argv.color ) {~~
104		~~- colors.mode = 'none';~~
105		-}
	135	+ this.test_filter = null;
	136	+ if( this.argv.filter ) { // null is the default by definition
	137	+ try {
	138	+ this.test_filter = new RegExp( this.argv.filter );
	139	+ } catch(e) {
	140	+ console.error( "\nERROR> --filter was given an invalid regular expression.");
	141	+ console.error( "ERROR> See below for JS engine error:\n" + e + "\n" );
	142	+ process.exit( 1 );
	143	+ }
	144	+ console.log( "Filtering title test using Regexp " + this.test_filter );
	145	+ }
	146	+ if( !this.argv.color ) {
	147	+ colors.mode = 'none';
	148	+ }
106	149
107		~~-// @fixme wrap more or this setup in a common module~~
	150	+ // Name of file used to cache the parser tests cases
	151	+ this.cache_file = "parserTests.cache";
108	152
109		~~-// track files imported / required~~
110		~~-var fileDependencies = [];~~
	153	+ // Preload the grammar file...
	154	+ PegTokenizer.src = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');
111	155
112		~~-// Fetch up some of our wacky parser bits...~~
	156	+ this.wikiTokenizer = new PegTokenizer();
113	157
114		~~-var basePath = path.join(path.dirname(path.dirname(process.cwd())), 'modules');~~
115		~~-function _require(filename) {~~
116		~~- var fullpath = path.join( basePath, filename );~~
117		~~- fileDependencies.push( fullpath );~~
118		~~- return require( fullpath );~~
119		-}
	158	+ this.testFileName = '../../../../phase3/tests/parser/parserTests.txt'; // default
	159	+ this.testFileName2 = '../../../../tests/parser/parserTests.txt'; // Fallback. Not everyone fetch at phase3 level
120	160
121		~~-function _import(filename, symbols) {~~
122		~~- var module = _require(filename);~~
123		~~- symbols.forEach(function(symbol) {~~
124		~~- global[symbol] = module[symbol];~~
125		~~- });~~
126		-}
	161	+ if (this.argv._[0]) {
	162	+ // hack :D
	163	+ this.testFileName = this.argv._[0] ;
	164	+ this.testFileName2 = null;
	165	+ }
127	166
	167	+ try {
	168	+ this.testParser = PEG.buildParser(fs.readFileSync('parserTests.pegjs', 'utf8'));
	169	+ } catch (e) {
	170	+ console.log(e);
	171	+ }
128	172
129		~~-// needed for html5 parser adapter~~
130		~~-//var events = require('events');~~
	173	+ this.cases = this.getTests();
131	174
132		~~-// For now most modules only need this for $.extend and $.each :)~~
133		~~-global.$ = require('jquery');~~
	175	+ this.articles = {};
134	176
135		~~-// hack for renderer~~
136		~~-global.document = $('<div>')[0].ownerDocument;~~
	177	+ this.htmlparser = new HTML5.Parser();
137	178
138		~~-var pj = path.join;~~
	179	+ this.postProcessor = new DOMPostProcessor();
139	180
	181	+ var pt = this;
	182	+ this.tokenTransformer = new TokenTransformer ( function ( tokens ) {
	183	+ //console.log("TOKENS: " + JSON.stringify(tokens, null, 2));
	184	+ // Create a new tree builder, which also creates a new document.
	185	+ var treeBuilder = new FauxHTML5.TreeBuilder();
140	186
	187	+ // Build a DOM tree from tokens using the HTML tree
	188	+ // builder/parser.
	189	+ pt.buildTree( tokens, treeBuilder );
141	190
142		~~-// Our code...~~
	191	+ // Perform post-processing on DOM.
	192	+ pt.postProcessor.doPostProcess(treeBuilder.parser.document);
143	193
144		~~-var testWhiteList = require('./parserTests-whitelist.js').testWhiteList;~~
	194	+ // And serialize the result.
	195	+ var out = treeBuilder.body().innerHTML;
145	196
146		~~-_import(pj('parser', 'mediawiki.tokenizer.peg.js'), ['PegTokenizer']);~~
147		~~-_import(pj('parser', 'mediawiki.parser.environment.js'), ['MWParserEnvironment']);~~
148		~~-_import(pj('parser', 'ext.cite.taghook.ref.js'), ['MWRefTagHook']);~~
	197	+ pt.checkResult( pt.currentItem, out );
	198	+ });
149	199
150		~~-_import(pj('parser', 'mediawiki.HTML5TreeBuilder.node.js'), ['FauxHTML5']);~~
151		~~-_import(pj('parser', 'mediawiki.DOMPostProcessor.js'), ['DOMPostProcessor']);~~
	200	+ // Test statistics
	201	+ this.passedTests = 0;
	202	+ this.passedTestsManual = 0;
	203	+ this.failParseTests = 0;
	204	+ this.failTreeTests = 0;
	205	+ this.failOutputTests = 0;
152	206
153		~~-// WikiDom and serializers~~
154		~~-_require(pj('es', 'es.js'));~~
155		~~-_require(pj('es', 'es.Html.js'));~~
156		~~-_require(pj('es', 'serializers', 'es.AnnotationSerializer.js'));~~
157		~~-_require(pj('es', 'serializers', 'es.HtmlSerializer.js'));~~
158		~~-_require(pj('es', 'serializers', 'es.WikitextSerializer.js'));~~
159		~~-_require(pj('es', 'serializers', 'es.JsonSerializer.js'));~~
	207	+ this.currentItem = undefined;
160	208
161		~~-// Preload the grammar file...~~
162		~~-PegTokenizer.src = fs.readFileSync(path.join(basePath, 'parser', 'pegTokenizer.pegjs.txt'), 'utf8');~~
	209	+ return this;
	210	+}
163	211
164		~~-var wikiTokenizer = new PegTokenizer();~~
165	212
166		~~-var testFileName = '../../../../phase3/tests/parser/parserTests.txt'; // default~~
167		~~-var testFileName2 = '../../../../tests/parser/parserTests.txt'; // Fallback. Not everyone fetch at phase3 level~~
168	213
169		~~-if (argv._[0]) {~~
170		~~- // hack :D~~
171		~~- testFileName = argv._[0] ;~~
172		~~- testFileName2 = null;~~
173		-}
174	214
175		~~-try {~~
176		~~- var testParser = PEG.buildParser(fs.readFileSync('parserTests.pegjs', 'utf8'));~~
177		~~-} catch (e) {~~
178		~~- console.log(e);~~
179		-}
180		-
181		-
182	215	/**
183	216	* Get an object holding our tests cases. Eventually from a cache file
184	217	*/
185		~~-function getTests() {~~
	218	+ParserTests.prototype.getTests = function () {
186	219
187	220	// Startup by loading .txt test file
188	221	var testFile;
189	222	try {
190		~~- testFile = fs.readFileSync(testFileName, 'utf8');~~
191		~~- fileDependencies.push( testFileName );~~
	223	+ testFile = fs.readFileSync(this.testFileName, 'utf8');
	224	+ fileDependencies.push( this.testFileName );
192	225	} catch (e) {
193	226	// Try opening fallback file
194		~~- if( testFileName2 !== '' ) {~~
	227	+ if( this.testFileName2 !== '' ) {
195	228	try {
196		~~- testFile = fs.readFileSync( testFileName2, 'utf8' );~~
197		~~- fileDependencies.push( testFileName2 );~~
	229	+ testFile = fs.readFileSync( this.testFileName2, 'utf8' );
	230	+ fileDependencies.push( this.testFileName2 );
198	231	}
199	232	catch(e) { console.log(e); }
200	233	}
201	234	}
202		~~- if( !argv.cache ) {~~
	235	+ if( !this.argv.cache ) {
203	236	// Cache not wanted, parse file and return object
204		~~- return parseTestCase( testFile );~~
	237	+ return this.parseTestCase( testFile );
205	238	}
206	239
207	240	// Find out modification time of all files depencies and then hashes those
208	241	// as a unique value using sha1.
209	242	var mtimes = '';
210	243	fileDependencies.sort().forEach( function (file) {
211		~~- mtimes += fs.statSync( file )['mtime'];~~
	244	+ mtimes += fs.statSync( file ).mtime;
212	245	});
213	246	var sha1 = require('crypto').createHash('sha1')
214	247	.update( mtimes ).digest( 'hex' );
—	—	@@ -216,8 +249,8 @@
217	250	var cache_content;
218	251	var cache_file_digest;
219	252	try {
220		~~- console.log( "Looking for cache file " + cache_file );~~
221		~~- cache_content = fs.readFileSync( cache_file, 'utf8' );~~
	253	+ console.log( "Looking for cache file " + this.cache_file );
	254	+ cache_content = fs.readFileSync( this.cache_file, 'utf8' );
222	255	// Fetch previous digest
223	256	cache_file_digest = cache_content.match( /^CACHE: (\w+)\n/ )[1];
224	257	} catch(e) {
—	—	@@ -232,9 +265,9 @@
233	266	} else {
234	267	// Write new file cache, content preprended with current digest
235	268	console.log( "Cache file either inexistant or outdated" );
236		~~- var parse = parseTestCase( testFile )~~
237		~~- console.log( "Writing parse result to " +cache_file );~~
238		~~- fs.writeFileSync( cache_file,~~
	269	+ var parse = this.parseTestCase( testFile );
	270	+ console.log( "Writing parse result to " + this.cache_file );
	271	+ fs.writeFileSync( this.cache_file,
239	272	"CACHE: " + sha1 + "\n" + JSON.stringify( parse ),
240	273	'utf8'
241	274	);
—	—	@@ -242,26 +275,23 @@
243	276	// We can now return the parsed object
244	277	return parse;
245	278	}
246		-}
	279	+};
247	280
248	281	/**
249	282	* Parse given tests cases given as plaintext
250	283	*/
251		~~-function parseTestCase( content ) {~~
	284	+ParserTests.prototype.parseTestCase = function ( content ) {
252	285	console.log( "Parsing tests case from file, this takes a few seconds ..." );
253	286	try {
254		~~- return testParser.parse(content);~~
255	287	console.log( "Done parsing." );
	288	+ return this.testParser.parse(content);
256	289	} catch (e) {
257	290	console.log(e);
258	291	}
259		-}
	292	+};
260	293
261		~~-var cases = getTests();~~
262	294
263		~~-var articles = {};~~
264		-
265		~~-function normalizeTitle(name) {~~
	295	+ParserTests.prototype.normalizeTitle = function(name) {
266	296	if (typeof name !== 'string') {
267	297	throw new Error('nooooooooo not a string');
268	298	}
—	—	@@ -271,36 +301,32 @@
272	302	throw new Error('Invalid/empty title');
273	303	}
274	304	return name;
275		-}
	305	+};
276	306
277		~~-function fetchArticle(name) {~~
	307	+ParserTests.prototype.fetchArticle = function(name) {
	308	+ // very simple for now..
278	309	var norm = normalizeTitle(name);
279		~~- if (norm in articles) {~~
280		~~- return articles[norm];~~
	310	+ if (norm in this.articles) {
	311	+ return this.articles[norm];
281	312	}
282		-}
	313	+};
283	314
284		~~-function processArticle(item) {~~
285		~~- var norm = normalizeTitle(item.title);~~
286		~~- articles[norm] = item.text;~~
287		-}
	315	+ParserTests.prototype.processArticle = function(item) {
	316	+ var norm = this.normalizeTitle(item.title);
	317	+ this.articles[norm] = item.text;
	318	+};
288	319
289		~~-function nodeToHtml(node) {~~
290		~~- return $('<div>').append(node).html();~~
291		-}
292	320
293		~~-var htmlparser = new HTML5.Parser();~~
294		-
295	321	/* Normalize the expected parser output by parsing it using a HTML5 parser and
296	322	* re-serializing it to HTML. Ideally, the parser would normalize inter-tag
297	323	* whitespace for us. For now, we fake that by simply stripping all newlines.
298	324	*/
299		~~-function normalizeHTML(source) {~~
	325	+ParserTests.prototype.normalizeHTML = function (source) {
300	326	// TODO: Do not strip newlines in pre and nowiki blocks!
301	327	source = source.replace(/\n/g, '');
302	328	try {
303		~~- htmlparser.parse('<body>' + source + '</body>');~~
304		~~- return htmlparser.document~~
	329	+ this.htmlparser.parse('<body>' + source + '</body>');
	330	+ return this.htmlparser.document
305	331	.getElementsByTagName('body')[0]
306	332	.innerHTML
307	333	// a few things we ignore for now..
—	—	@@ -323,34 +349,44 @@
324	350	return source;
325	351	}
326	352
327		-}
	353	+};
328	354
329	355	// Specialized normalization of the wiki parser output, mostly to ignore a few
330	356	// known-ok differences.
331		~~-function normalizeOut ( out ) {~~
	357	+ParserTests.prototype.normalizeOut = function ( out ) {
332	358	// TODO: Do not strip newlines in pre and nowiki blocks!
333	359	return out.replace(/\n\| data-[a-zA-Z]+="[^">]*"/g, '')
334	360	.replace(/<!--.*?-->\n?/gm, '');
335		-}
	361	+};
336	362
337		~~-function formatHTML ( source ) {~~
	363	+ParserTests.prototype.formatHTML = function ( source ) {
338	364	// Quick hack to insert newlines before some block level start tags
339	365	return source.replace(
340	366	/(?!^)<((div\|dd\|dt\|li\|p\|table\|tr\|td\|tbody\|dl\|ol\|ul\|h1\|h2\|h3\|h4\|h5\|h6)[^>]*)>/g,
341	367	'\n<$1>');
342		-}
	368	+};
343	369
344		~~-var passedTests = 0,~~
345		~~- passedTestsManual = 0,~~
346		~~- failParseTests = 0,~~
347		~~- failTreeTests = 0,~~
348		~~- failOutputTests = 0;~~
349	370
350		~~-var postProcessor = new DOMPostProcessor();~~
351	371
352		~~-function processTest(item) {~~
353		~~- // Create a new tree builder, which also creates a new document.~~
354		~~- var treeBuilder = new FauxHTML5.TreeBuilder();~~
	372	+ParserTests.prototype.printTitle = function( item, failure_only ) {
	373	+ if( failure_only ) {
	374	+ console.log('FAILED'.red + ': ' + item.title.yellow);
	375	+ return;
	376	+ }
	377	+ console.log('=====================================================');
	378	+ console.log('FAILED'.red + ': ' + item.title.yellow);
	379	+ console.log(item.comments.join('\n'));
	380	+ if (item.options) {
	381	+ console.log("OPTIONS".cyan + ":");
	382	+ console.log(item.options + '\n');
	383	+ }
	384	+ console.log("INPUT".cyan + ":");
	385	+ console.log(item.input + "\n");
	386	+};
	387	+
	388	+
	389	+
	390	+ParserTests.prototype.processTest = function (item) {
355	391	if (!('title' in item)) {
356	392	console.log(item);
357	393	throw new Error('Missing title from test case.');
—	—	@@ -364,130 +400,108 @@
365	401	throw new Error('Missing input from test case ' + item.title);
366	402	}
367	403
368		~~- function printTitle( failure_only ) {~~
369		~~- if( failure_only ) {~~
370		~~- console.log('FAILED'.red + ': ' + item.title.yellow);~~
371		~~- return;~~
372		~~- }~~
373		~~- console.log('=====================================================');~~
374		~~- console.log('FAILED'.red + ': ' + item.title.yellow);~~
375		~~- console.log(item.comments.join('\n'));~~
376		~~- if (item.options) {~~
377		~~- console.log("OPTIONS".cyan + ":");~~
378		~~- console.log(item.options + '\n');~~
379		~~- }~~
380		~~- console.log("INPUT".cyan + ":");~~
381		~~- console.log(item.input + "\n");~~
382		~~- }~~
	404	+ this.currentItem = item;
383	405
384		~~- wikiTokenizer.tokenize(item.input + "\n", function(tokens, err) {~~
385		~~- if (err) {~~
386		~~- printTitle();~~
387		~~- failParseTests++;~~
388		~~- console.log('PARSE FAIL', err);~~
389		~~- } else {~~
390		~~- //var environment = new MWParserEnvironment({~~
391		~~- // tagHooks: {~~
392		~~- // 'ref': MWRefTagHook,~~
393		~~- // 'references': MWReferencesTagHook~~
394		~~- // }~~
395		~~- //});~~
396		~~- //var res = es.HtmlSerializer.stringify(tokens,environment);~~
	406	+ // Tokenize the input
	407	+ var res = this.wikiTokenizer.tokenize(item.input + "\n");
397	408
398		~~- //console.log(JSON.stringify(tokens));~~
399		~~- //Slightly better token output debugging:~~
400		~~- //console.log( util.inspect( tokens, false, null ).yellow);~~
	409	+ // Check for errors
	410	+ if (res.err) {
	411	+ this.printTitle(item);
	412	+ this.failParseTests++;
	413	+ console.log('PARSE FAIL', res.err);
	414	+ } else {
	415	+ //var environment = new MWParserEnvironment({
	416	+ // tagHooks: {
	417	+ // 'ref': MWRefTagHook,
	418	+ // 'references': MWReferencesTagHook
	419	+ // }
	420	+ //});
	421	+ //var res = es.HtmlSerializer.stringify(tokens,environment);
401	422
402		~~- try {~~
403		~~- // Build a DOM tree from tokens using the HTML tree~~
404		~~- // builder/parser.~~
405		~~- processTokens(tokens, treeBuilder);~~
	423	+ //console.log(JSON.stringify(tokens));
	424	+ //Slightly better token output debugging:
	425	+ //console.log( util.inspect( tokens, false, null ).yellow);
406	426
407		~~- // Perform post-processing on DOM.~~
408		~~- postProcessor.doPostProcess(treeBuilder.parser.document);~~
	427	+ // Transform tokens using the TokenTransformer. When done, the
	428	+ // TokenTransformer calls buildTree() and checkResult() with the
	429	+ // transformed tokens.
	430	+ this.tokenTransformer.transformTokens( res.tokens );
	431	+ }
	432	+};
409	433
410		~~- // And serialize the result.~~
411		~~- var out = treeBuilder.body()~~
412		~~- .innerHTML;~~
413		~~- } catch ( e ) {~~
414		~~- printTitle();~~
415		~~- failTreeTests++;~~
416		~~- console.log('RENDER FAIL', e);~~
417		~~- return;~~
418		~~- }~~
419		-
420		~~- var normalizedOut = normalizeOut(out);~~
421		~~- var normalizedExpected = normalizeHTML(item.result);~~
422		~~- if ( normalizedOut !== normalizedExpected ) {~~
423		~~- if (argv.whitelist &&~~
424		~~- item.title in testWhiteList &&~~
425		~~- normalizeOut(testWhiteList[item.title]) === normalizedOut) {~~
426		~~- if( !argv.quiet ) {~~
427		~~- console.log( 'PASSED (whiteList)'.green + ': ' + item.title.yellow );~~
428		~~- }~~
429		~~- passedTestsManual++;~~
430		~~- return;~~
	434	+ParserTests.prototype.checkResult = function ( item, out ) {
	435	+ var normalizedOut = this.normalizeOut(out);
	436	+ var normalizedExpected = this.normalizeHTML(item.result);
	437	+ if ( normalizedOut !== normalizedExpected ) {
	438	+ if (this.argv.whitelist &&
	439	+ item.title in testWhiteList &&
	440	+ this.normalizeOut(testWhiteList[item.title]) === normalizedOut) {
	441	+ if( !this.argv.quiet ) {
	442	+ console.log( 'PASSED (whiteList)'.green + ': ' + item.title.yellow );
	443	+ }
	444	+ this.passedTestsManual++;
	445	+ return;
431	446	}
432		~~- printTitle( argv.quick );~~
433		~~- failOutputTests++;~~
	447	+ this.printTitle( item, this.argv.quick );
	448	+ this.failOutputTests++;
434	449
435		~~- if( !argv.quick ) {~~
436		~~- console.log('RAW EXPECTED'.cyan + ':');~~
437		~~- console.log(item.result + "\n");~~
	450	+ if( !this.argv.quick ) {
	451	+ console.log('RAW EXPECTED'.cyan + ':');
	452	+ console.log(item.result + "\n");
438	453
439		~~- console.log('RAW RENDERED'.cyan + ':');~~
440		~~- console.log(formatHTML(out) + "\n");~~
	454	+ console.log('RAW RENDERED'.cyan + ':');
	455	+ console.log(this.formatHTML(out) + "\n");
441	456
442		~~- var a = formatHTML(normalizedExpected);~~
	457	+ var a = this.formatHTML(normalizedExpected);
443	458
444		~~- console.log('NORMALIZED EXPECTED'.magenta + ':');~~
445		~~- console.log(a + "\n");~~
	459	+ console.log('NORMALIZED EXPECTED'.magenta + ':');
	460	+ console.log(a + "\n");
446	461
447		~~- var b = formatHTML(normalizedOut);~~
	462	+ var b = this.formatHTML(normalizedOut);
448	463
449		~~- console.log('NORMALIZED RENDERED'.magenta + ':')~~
450		~~- console.log(formatHTML(normalizeOut(out)) + "\n");~~
451		~~- var patch = jsDiff.createPatch('wikitext.txt', a, b, 'before', 'after');~~
	464	+ console.log('NORMALIZED RENDERED'.magenta + ':');
	465	+ console.log(this.formatHTML(this.normalizeOut(out)) + "\n");
	466	+ var patch = jsDiff.createPatch('wikitext.txt', a, b, 'before', 'after');
452	467
453		~~- console.log('DIFF'.cyan +': ');~~
	468	+ console.log('DIFF'.cyan +': ');
454	469
455		~~- // Strip the header from the patch, we know how diffs work..~~
456		~~- patch = patch.replace(/^[^\n]\n[^\n]\n[^\n]\n[^\n]\n/, '');~~
	470	+ // Strip the header from the patch, we know how diffs work..
	471	+ patch = patch.replace(/^[^\n]\n[^\n]\n[^\n]\n[^\n]\n/, '');
457	472
458		~~- var colored_diff = patch.split( '\n' ).map( function(line) {~~
459		~~- // Add some colors to diff output~~
460		~~- switch( line.charAt(0) ) {~~
461		~~- case '-':~~
462		~~- return line.red;~~
463		~~- case '+':~~
464		~~- return line.blue;~~
465		~~- default:~~
466		~~- return line;~~
467		~~- }~~
468		~~- }).join( "\n" );~~
469		-
	473	+ var colored_diff = patch.split( '\n' ).map( function(line) {
	474	+ // Add some colors to diff output
	475	+ switch( line.charAt(0) ) {
	476	+ case '-':
	477	+ return line.red;
	478	+ case '+':
	479	+ return line.blue;
	480	+ default:
	481	+ return line;
	482	+ }
	483	+ }).join( "\n" );
470	484
471		~~- console.log( colored_diff );~~
472		-
473		~~- if(argv.printwhitelist) {~~
474		~~- console.log("WHITELIST ENTRY:".cyan);~~
475		~~- console.log("testWhiteList[" +~~
476		~~- JSON.stringify(item.title) + "] = " +~~
477		~~- JSON.stringify(out) +~~
478		~~- ";\n");~~
479		~~- }~~
480		~~- }~~
481		~~- } else {~~
482		~~- passedTests++;~~
483		~~- if( !argv.quiet ) {~~
484		~~- console.log( 'PASSED'.green + ': ' + item.title.yellow );~~
485		~~- }~~
	485	+
	486	+ console.log( colored_diff );
	487	+
	488	+ if(this.argv.printwhitelist) {
	489	+ console.log("WHITELIST ENTRY:".cyan);
	490	+ console.log("testWhiteList[" +
	491	+ JSON.stringify(item.title) + "] = " +
	492	+ JSON.stringify(out) +
	493	+ ";\n");
486	494	}
487	495	}
488		~~- });~~
489		-}
	496	+ } else {
	497	+ this.passedTests++;
	498	+ if( !this.argv.quiet ) {
	499	+ console.log( 'PASSED'.green + ': ' + item.title.yellow );
	500	+ }
	501	+ }
	502	+};
490	503
491		~~-function processTokens ( tokens, treeBuilder ) {~~
	504	+
	505	+ParserTests.prototype.buildTree = function ( tokens, treeBuilder ) {
492	506	// push a body element, just to be sure to have one
493	507	treeBuilder.processToken({type: 'TAG', name: 'body'});
494	508	// Process all tokens
—	—	@@ -496,45 +510,15 @@
497	511	}
498	512	// And signal the end
499	513	treeBuilder.processToken({type: 'END'});
500		-}
	514	+};
501	515
502		~~-var comments = [];~~
503		-
504		~~-console.log( "Initialisation complete. Now launching tests." );~~
505		~~-cases.forEach(function(item) {~~
506		~~- if (typeof item == 'object') {~~
507		~~- switch(item.type) {~~
508		~~- case 'article':~~
509		~~- //processArticle(item);~~
510		~~- comments = [];~~
511		~~- break;~~
512		~~- case 'test':~~
513		~~- if( test_filter && -1 === item.title.search( test_filter ) ) {~~
514		~~- // Skip test whose title does not match --filter~~
515		~~- break;~~
516		~~- }~~
517		~~- // Add comments to following test.~~
518		~~- item.comments = comments;~~
519		~~- comments = [];~~
520		~~- processTest(item);~~
521		~~- break;~~
522		~~- case 'comment':~~
523		~~- comments.push(item.comment);~~
524		~~- break;~~
525		~~- default:~~
526		~~- comments = [];~~
527		~~- break;~~
528		~~- }~~
529		~~- }~~
530		~~-});~~
531		-
532	516	/**
533	517	* Colorize given number if <> 0
534	518	*
535	519	* @param count Integer: a number to colorize
536	520	* @param color String: 'green' or 'red'
537	521	*/
538		~~-function ColorizeCount( count, color ) {~~
	522	+ParserTests.prototype.ColorizeCount = function ( count, color ) {
539	523	if( count === 0 ) {
540	524	return count;
541	525	}
—	—	@@ -548,32 +532,87 @@
549	533
550	534	default: return count;
551	535	}
552		-}
	536	+};
553	537
554		~~-var failTotalTests = (failParseTests + failTreeTests + failOutputTests);~~
	538	+ParserTests.prototype.reportSummary = function () {
555	539
556		~~-console.log( "==========================================================");~~
557		~~-console.log( "SUMMARY: ");~~
	540	+ var failTotalTests = (this.failParseTests + this.failTreeTests +
	541	+ this.failOutputTests);
558	542
559		~~-if( failTotalTests !== 0 ) {~~
560		~~-console.log( ColorizeCount( passedTests , 'green' ) + " passed");~~
561		~~-console.log( ColorizeCount( passedTestsManual , 'green' ) + " passed from whitelist");~~
562		~~-console.log( ColorizeCount( failParseTests , 'red' ) + " parse failures");~~
563		~~-console.log( ColorizeCount( failTreeTests , 'red' ) + " tree build failures");~~
564		~~-console.log( ColorizeCount( failOutputTests, 'red' ) + " output differences");~~
565		~~-console.log( "\n" );~~
566		~~-console.log( ColorizeCount( passedTests + passedTestsManual , 'green' ) +~~
567		~~- ' total passed tests, ' +~~
568		~~- ColorizeCount( failTotalTests , 'red' ) + " total failures");~~
	543	+ console.log( "==========================================================");
	544	+ console.log( "SUMMARY: ");
569	545
570		~~-} else {~~
571		~~- if( test_filter !== null ) {~~
572		~~- console.log( "Passed " + passedTests + passedTestsManual + " of " + passedTests + " tests matching " + test_filter + "... " + "ALL TESTS PASSED!".green );~~
	546	+ if( failTotalTests !== 0 ) {
	547	+ console.log( this.ColorizeCount( this.passedTests , 'green' ) +
	548	+ " passed");
	549	+ console.log( this.ColorizeCount( this.passedTestsManual , 'green' ) +
	550	+ " passed from whitelist");
	551	+ console.log( this.ColorizeCount( this.failParseTests , 'red' ) +
	552	+ " parse failures");
	553	+ console.log( this.ColorizeCount( this.failTreeTests , 'red' ) +
	554	+ " tree build failures");
	555	+ console.log( this.ColorizeCount( this.failOutputTests, 'red' ) +
	556	+ " output differences");
	557	+ console.log( "\n" );
	558	+ console.log( this.ColorizeCount( this.passedTests + this.passedTestsManual , 'green' ) +
	559	+ ' total passed tests, ' +
	560	+ this.ColorizeCount( failTotalTests , 'red' ) + " total failures");
	561	+
573	562	} else {
574		~~- // Should not happen if it does: Champagne!~~
575		~~- console.log( "Passed " + passedTests + " of " + passedTests + " tests... " + "ALL TESTS PASSED!".green );~~
	563	+ if( this.test_filter !== null ) {
	564	+ console.log( "Passed " + this.passedTests + pthis.assedTestsManual +
	565	+ " of " + passedTests + " tests matching " + this.test_filter +
	566	+ "... " + "ALL TESTS PASSED!".green );
	567	+ } else {
	568	+ // Should not happen if it does: Champagne!
	569	+ console.log( "Passed " + this.passedTests + " of " + this.passedTests +
	570	+ " tests... " + "ALL TESTS PASSED!".green );
	571	+ }
576	572	}
577		-}
578		~~-console.log( "==========================================================");~~
	573	+ console.log( "==========================================================");
579	574
	575	+};
	576	+
	577	+ParserTests.prototype.main = function () {
	578	+ console.log( "Initialisation complete. Now launching tests." );
	579	+
	580	+ var comments = [],
	581	+ pt = this;
	582	+ this.cases.forEach(function(item) {
	583	+ if (typeof item == 'object') {
	584	+ switch(item.type) {
	585	+ case 'article':
	586	+ pt.processArticle(item);
	587	+ comments = [];
	588	+ break;
	589	+ case 'test':
	590	+ if( pt.test_filter &&
	591	+ -1 === item.title.search( pt.test_filter ) ) {
	592	+ // Skip test whose title does not match --filter
	593	+ break;
	594	+ }
	595	+ // Add comments to following test.
	596	+ item.comments = comments;
	597	+ comments = [];
	598	+ pt.processTest(item);
	599	+ break;
	600	+ case 'comment':
	601	+ comments.push(item.comment);
	602	+ break;
	603	+ default:
	604	+ comments = [];
	605	+ break;
	606	+ }
	607	+ }
	608	+ });
	609	+
	610	+ // print out the summary
	611	+ this.reportSummary();
	612	+};
	613	+
	614	+var pt = new ParserTests();
	615	+console.log(pt.processArticle);
	616	+pt.main();
	617	+
	618	+
580	619	})();
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.TokenTransformer.js
—	—	@@ -19,8 +19,6 @@
20	20
21	21	function TokenTransformer( callback ) {
22	22	this.cb = callback; // Called with transformed token list when done
23		~~- this.accum = new TokenAccumulator();~~
24		~~- this.firstaccum = this.accum;~~
25	23	this.transformers = {
26	24	tag: {}, // for TAG, ENDTAG, SELFCLOSINGTAG, keyed on name
27	25	text: [],
—	—	@@ -29,11 +27,17 @@
30	28	end: [], // eof
31	29	martian: [] // none of the above
32	30	};
33		~~- this.outstanding = 1; // Number of outstanding processing steps~~
34		~~- // (e.g., async template fetches/expansions)~~
	31	+ this.reset();
35	32	return this;
36	33	}
37	34
	35	+TokenTransformer.prototype.reset = function () {
	36	+ this.accum = new TokenAccumulator();
	37	+ this.firstaccum = this.accum;
	38	+ this.outstanding = 1; // Number of outstanding processing steps
	39	+ // (e.g., async template fetches/expansions)
	40	+};
	41	+
38	42	TokenTransformer.prototype.appendListener = function ( listener, type, name ) {
39	43	if ( type === 'tag' ) {
40	44	if ( $.isArray(this.transformers.tag.name) ) {
—	—	@@ -97,7 +101,7 @@
98	102	* @returns {TokenContext} Context with updated token and/or accum.
99	103	*/
100	104	TokenTransformer.prototype._transformTagToken = function ( tokenCTX ) {
101		~~- var ts = this.transformers.tag[token.name];~~
	105	+ var ts = this.transformers.tag[tokenCTX.token.name];
102	106	if ( ts ) {
103	107	for (var i = 0, l = ts.length; i < l; i++ ) {
104	108	// Transform token with side effects
—	—	@@ -145,6 +149,7 @@
146	150	* */
147	151	TokenTransformer.prototype.transformTokens = function ( tokens, accum ) {
148	152	if ( accum === undefined ) {
	153	+ this.reset();
149	154	accum = this.accum;
150	155	} else {
151	156	// Prepare to replace the last token in the current accumulator.
—	—	@@ -199,13 +204,13 @@
200	205	if ( this.outstanding === 0 ) {
201	206	// Join the token accumulators back into a single token list
202	207	var a = this.firstaccum;
203		~~- var accums = [a.accum];~~
	208	+ var tokens = a.accum;
204	209	while ( a.next !== undefined ) {
205	210	a = a.next;
206		~~- accums.concat(a.accum);~~
	211	+ tokens.concat(a.accum);
207	212	}
208	213	// Call our callback with the flattened token list
209		~~- this.cb(accums);~~
	214	+ this.cb(tokens);
210	215	}
211	216	};
212	217
—	—	@@ -237,3 +242,7 @@
238	243	this.next = new TokenAccumulator(this.next, tokens);
239	244	return this.next;
240	245	};
	246	+
	247	+if (typeof module == "object") {
	248	+ module.exports.TokenTransformer = TokenTransformer;
	249	+}
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.tokenizer.peg.js
—	—	@@ -5,27 +5,28 @@
6	6	* Use along with a HTML5TreeBuilder and the DOMPostProcessor(s) for HTML
7	7	* output.
8	8	*/
	9	+
	10	+var PEG = require('pegjs');
	11	+
9	12	function PegTokenizer(env) {
10	13	this.env = env \|\| {};
11	14	}
12	15
13	16	PegTokenizer.src = false;
14	17
15		~~-PegTokenizer.prototype.tokenize = function(text, callback) {~~
16		~~- this.initSource(function() {~~
17		~~- var out, err;~~
18		~~- try {~~
19		~~- if ( !this.parser ) {~~
20		~~- this.parser = PEG.buildParser(PegTokenizer.src);~~
21		~~- }~~
22		~~- out = this.parser.parse(text);~~
23		~~- } catch (e) {~~
24		~~- err = e;~~
25		~~- console.trace();~~
26		~~- } finally {~~
27		~~- callback(out, err);~~
28		~~- }~~
29		~~- });~~
	18	+PegTokenizer.prototype.tokenize = function( text ) {
	19	+ var out, err;
	20	+ if ( !this.parser ) {
	21	+ this.parser = PEG.buildParser(PegTokenizer.src);
	22	+ }
	23	+ try {
	24	+ out = this.parser.parse(text);
	25	+ } catch (e) {
	26	+ err = e;
	27	+ console.trace();
	28	+ } finally {
	29	+ return {tokens: out, err: err};
	30	+ }
30	31	}
31	32
32	33	/**

Status & tagging log

20:55, 12 December 2011 GWicke (talk | contribs) changed the status of r105876 [removed: new added: deferred]