r109604 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r109603‎ | r109604 | r109605 >
Date:02:36, 20 January 2012
Author:gwicke
Status:deferred
Tags:
Comment:
Collapse all requests per template into a single outstanding request using an
event-emitting TemplateRequest object and a request queue.
Modified paths:
  • /trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js (modified) (history)
  • /trunk/extensions/VisualEditor/modules/parser/parse.js (modified) (history)

Diff [purge]

Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.DOMConverter.js
——@@ -1,5 +1,7 @@
22 /**
33 * Conversions between HTML DOM and WikiDom
 4+ *
 5+ * @author Gabriel Wicke <gwicke@wikimedia.org>
46 *
57 * @class
68 * @constructor
Index: trunk/extensions/VisualEditor/modules/parser/parse.js
——@@ -1,6 +1,9 @@
22 /**
33 * Command line wikidom parse utility.
44 * Read from STDIN, write to STDOUT.
 5+ *
 6+ * @author Neil Kandalgaonkar <neilk@wikimedia.org>
 7+ * @author Gabriel Wicke <gwicke@wikimedia.org>
58 */
69
710
——@@ -12,7 +15,11 @@
1316 optimist = require('optimist');
1417
1518 var env = new ParserEnv( {
 19+ // fetch templates from enwiki by default..
 20+ wgScriptPath: "http://en.wikipedia.org/w",
 21+ wgScriptExtension: ".php",
1622 fetchTemplates: true,
 23+ // enable/disable debug output using this switch
1724 debug: false
1825 } ),
1926 parser = new ParserPipeline( env );
Index: trunk/extensions/VisualEditor/modules/parser/ext.core.TemplateHandler.js
——@@ -12,6 +12,7 @@
1313 */
1414 var $ = require('jquery'),
1515 request = require('request'),
 16+ events = require('events'),
1617 qs = require('querystring'),
1718 AttributeTransformManager = require('./mediawiki.TokenTransformManager.js')
1819 .AttributeTransformManager;
——@@ -290,7 +291,7 @@
291292 TemplateHandler.prototype._fetchTemplateAndTitle = function ( title, callback, tplExpandData ) {
292293 // @fixme normalize name?
293294 var self = this;
294 - if (title in this.manager.env.pageCache) {
 295+ if (this.manager.env.pageCache[title]) {
295296 // Unroll the stack here
296297 process.nextTick(
297298 function () {
——@@ -300,115 +301,20 @@
301302 } else if ( ! this.manager.env.fetchTemplates ) {
302303 callback('Page/template fetching disabled, and no cache for ' + title);
303304 } else {
304 - // Not found in the pageCache, so go fetch the source using the
305 - // MediaWiki API.
306 -
 305+
307306 // We are about to start an async request for a template, so mark this
308307 // template expansion as such.
309308 tplExpandData.overallAsync = true;
310309 this.manager.env.dp( 'trying to fetch ' + title );
311 - //console.log(this.manager.env.pageCache);
312 - var url = this.manager.env.wgScriptPath + '/api' +
313 - this.manager.env.wgScriptExtension +
314 - '?' +
315 - qs.stringify( {
316 - format: 'json',
317 - action: 'query',
318 - prop: 'revisions',
319 - rvprop: 'content',
320 - titles: title
321 - } );
322 - //'?format=json&action=query&prop=revisions&rvprop=content&titles=' + title;
323310
324 - request({
325 - method: 'GET',
326 - followRedirect: true,
327 - url: url,
328 - headers: {
329 - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0.1) ' +
330 - 'Gecko/20100101 Firefox/9.0.1 Iceweasel/9.0.1'
331 - }
332 - },
333 - function (error, response, body) {
334 - //console.log( 'response for ' + title + ' :' + body + ':' );
335 - if(error) {
336 - self.manager.env.dp(error);
337 - callback('Page/template fetch failure for title ' + title, title);
338 - return ;
339 - }
 311+ // Start a new request if none is outstanding
 312+ this.manager.env.dp( 'requestQueue: ', this.manager.env.requestQueue)
 313+ if ( this.manager.env.requestQueue[title] === undefined ) {
 314+ this.manager.env.requestQueue[title] = new TemplateRequest( this.manager, title );
 315+ }
 316+ // Append a listener to the request
 317+ this.manager.env.requestQueue[title].addListener( 'src', callback );
340318
341 - if(response.statusCode == 200) {
342 - var src = '';
343 - try {
344 - //console.log( 'body: ' + body );
345 - var data = JSON.parse( body );
346 - } catch(e) {
347 - console.log( "Error: while parsing result. Error was: " );
348 - console.log( e );
349 - console.log( "Response that didn't parse was:");
350 - console.log( "------------------------------------------\n" + body );
351 - console.log( "------------------------------------------" );
352 - }
353 - try {
354 - $.each(data.query.pages, function(i, page) {
355 - if (page.revisions && page.revisions.length) {
356 - src = page.revisions[0]['*'];
357 - title = page.title;
358 - }
359 - });
360 - } catch ( e ) {
361 - console.log( 'Did not find page revisions in the returned body:' + body );
362 - src = '';
363 - }
364 - //console.log( 'Page ' + title + ': got ' + src );
365 - self.manager.env.dp( 'Success for ' + title + ' :' + body + ':' );
366 - self.manager.env.pageCache[title] = src;
367 - callback(src, title);
368 - self.manager.env.dp(data);
369 - }
370 - });
371 -
372 - /*
373 - * XXX: The jQuery version does not quite work with node, but we keep
374 - * it around for now.
375 - $.ajax({
376 - url: url,
377 - data: {
378 - format: 'json',
379 - action: 'query',
380 - prop: 'revisions',
381 - rvprop: 'content',
382 - titles: title
383 - },
384 - success: function(data, statusString, xhr) {
385 - console.log( 'Page ' + title + ' success ' + JSON.stringify( data ) );
386 - var src = null, title = null;
387 - $.each(data.query.pages, function(i, page) {
388 - if (page.revisions && page.revisions.length) {
389 - src = page.revisions[0]['*'];
390 - title = page.title;
391 - }
392 - });
393 - if (typeof src !== 'string') {
394 - console.log( 'Page ' + title + 'not found! Got ' + src );
395 - callback( 'Page ' + title + ' not found' );
396 - } else {
397 - // Add to cache
398 - console.log( 'Page ' + title + ': got ' + src );
399 - this.manager.env.pageCache[title] = src;
400 - callback(src, title);
401 - }
402 - },
403 - error: function(xhr, msg, err) {
404 - console.log( 'Page/template fetch failure for title ' +
405 - title + ', url=' + url + JSON.stringify(xhr) + ', err=' + err );
406 - callback('Page/template fetch failure for title ' + title);
407 - },
408 - dataType: 'json',
409 - cache: false, // @fixme caching, versions etc?
410 - crossDomain: true
411 - });
412 - */
413319 }
414320 };
415321
——@@ -468,6 +374,123 @@
469375 }
470376 };
471377
 378+
 379+/***************** Template fetch request helper class ********/
 380+
 381+function TemplateRequest ( manager, title ) {
 382+ // Increase the number of maximum listeners a bit..
 383+ this.setMaxListeners( 1000 );
 384+ var self = this,
 385+ url = manager.env.wgScriptPath + '/api' +
 386+ manager.env.wgScriptExtension +
 387+ '?' +
 388+ qs.stringify( {
 389+ format: 'json',
 390+ action: 'query',
 391+ prop: 'revisions',
 392+ rvprop: 'content',
 393+ titles: title
 394+ } );
 395+ //'?format=json&action=query&prop=revisions&rvprop=content&titles=' + title;
 396+
 397+ request({
 398+ method: 'GET',
 399+ followRedirect: true,
 400+ url: url,
 401+ headers: {
 402+ 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:9.0.1) ' +
 403+ 'Gecko/20100101 Firefox/9.0.1 Iceweasel/9.0.1'
 404+ }
 405+ },
 406+ function (error, response, body) {
 407+ //console.log( 'response for ' + title + ' :' + body + ':' );
 408+ if(error) {
 409+ manager.env.dp(error);
 410+ callback('Page/template fetch failure for title ' + title, title);
 411+ return ;
 412+ }
 413+
 414+ if(response.statusCode == 200) {
 415+ var src = '';
 416+ try {
 417+ //console.log( 'body: ' + body );
 418+ var data = JSON.parse( body );
 419+ } catch(e) {
 420+ console.log( "Error: while parsing result. Error was: " );
 421+ console.log( e );
 422+ console.log( "Response that didn't parse was:");
 423+ console.log( "------------------------------------------\n" + body );
 424+ console.log( "------------------------------------------" );
 425+ }
 426+ try {
 427+ $.each(data.query.pages, function(i, page) {
 428+ if (page.revisions && page.revisions.length) {
 429+ src = page.revisions[0]['*'];
 430+ title = page.title;
 431+ }
 432+ });
 433+ } catch ( e ) {
 434+ console.log( 'Did not find page revisions in the returned body:' + body );
 435+ src = '';
 436+ }
 437+ //console.log( 'Page ' + title + ': got ' + src );
 438+ manager.env.dp( 'Success for ' + title + ' :' + body + ':' );
 439+ manager.env.pageCache[title] = src;
 440+ manager.env.dp(data);
 441+ self.emit( 'src', src, title );
 442+ // Remove self from request queue
 443+ delete manager.env.requestQueue[title];
 444+ }
 445+ });
 446+}
 447+
 448+ /*
 449+ * XXX: The jQuery version does not quite work with node, but we keep
 450+ * it around for now.
 451+ $.ajax({
 452+ url: url,
 453+ data: {
 454+ format: 'json',
 455+ action: 'query',
 456+ prop: 'revisions',
 457+ rvprop: 'content',
 458+ titles: title
 459+ },
 460+ success: function(data, statusString, xhr) {
 461+ console.log( 'Page ' + title + ' success ' + JSON.stringify( data ) );
 462+ var src = null, title = null;
 463+ $.each(data.query.pages, function(i, page) {
 464+ if (page.revisions && page.revisions.length) {
 465+ src = page.revisions[0]['*'];
 466+ title = page.title;
 467+ }
 468+ });
 469+ if (typeof src !== 'string') {
 470+ console.log( 'Page ' + title + 'not found! Got ' + src );
 471+ callback( 'Page ' + title + ' not found' );
 472+ } else {
 473+ // Add to cache
 474+ console.log( 'Page ' + title + ': got ' + src );
 475+ this.manager.env.pageCache[title] = src;
 476+ callback(src, title);
 477+ }
 478+ },
 479+ error: function(xhr, msg, err) {
 480+ console.log( 'Page/template fetch failure for title ' +
 481+ title + ', url=' + url + JSON.stringify(xhr) + ', err=' + err );
 482+ callback('Page/template fetch failure for title ' + title);
 483+ },
 484+ dataType: 'json',
 485+ cache: false, // @fixme caching, versions etc?
 486+ crossDomain: true
 487+ });
 488+ */
 489+
 490+// Inherit from EventEmitter
 491+TemplateRequest.prototype = new events.EventEmitter();
 492+TemplateHandler.prototype.constructor = TemplateRequest;
 493+
 494+
472495 if (typeof module == "object") {
473496 module.exports.TemplateHandler = TemplateHandler;
474497 }
Index: trunk/extensions/VisualEditor/modules/parser/mediawiki.parser.environment.js
——@@ -12,6 +12,10 @@
1313 $.extend(this, options);
1414 };
1515
 16+// Outstanding page requests (for templates etc)
 17+// Class-static
 18+MWParserEnvironment.prototype.requestQueue = {};
 19+
1620 MWParserEnvironment.prototype.lookupKV = function ( kvs, key ) {
1721 if ( ! kvs ) {
1822 return null;

Status & tagging log