r86540 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r86539‎ | r86540 | r86541 >
Date:20:03, 20 April 2011
Author:diederik
Status:deferred
Tags:
Comment:
Use multiprocessing for the transform phase as well, updated patch
Modified paths:
  • /trunk/tools/editor_trends/etl/transformer.py (modified) (history)

Diff [purge]

Index: trunk/tools/editor_trends/etl/transformer.py
@@ -48,7 +48,7 @@
4949
5050
5151 class Editor:
52 - def __init__(self, editor_id, db_raw, db_dataset, **kwargs):
 52+ def __init__(self, db_raw, db_dataset, editor_id, **kwargs):
5353 self.editor_id = editor_id
5454 self.db_raw = db_raw
5555 self.db_dataset = db_dataset
@@ -293,11 +293,11 @@
294294
295295 def transform_editors_multi_launcher(rts):
296296 tasks = multiprocessing.JoinableQueue()
297 - input_db, output_db, editors = setup_database(rts)
 297+ db_raw, db_dataset, editors = setup_database(rts)
298298 transformers = [EditorConsumer(rts, tasks) for i in xrange(rts.number_of_processes)]
299299
300300 for editor in editors:
301 - tasks.put(Editor(rts.dbname, rts.editors_raw, editor))
 301+ tasks.put(Editor(db_raw, db_dataset, editor))
302302
303303 for x in xrange(rts.number_of_processes):
304304 tasks.put(None)