r86541 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r86540‎ | r86541 | r86542 >
Date:20:08, 20 April 2011
Author:diederik
Status:deferred
Tags:
Comment:
Use multiprocessing for the transform phase as well, updated patch
Modified paths:
  • /trunk/tools/editor_trends/etl/transformer.py (modified) (history)
  • /trunk/tools/editor_trends/manage.py (modified) (history)

Diff [purge]

Index: trunk/tools/editor_trends/manage.py
@@ -24,7 +24,6 @@
2525 import datetime
2626 import ConfigParser
2727 from argparse import ArgumentParser, RawTextHelpFormatter
28 -import cProfile
2928
3029 from classes import languages
3130 from classes import projects
@@ -420,5 +419,5 @@
421420
422421
423422 if __name__ == '__main__':
424 - #main()
425 - cProfile.run('main()')
 423+ main()
 424+ #cProfile.run('main()')
Index: trunk/tools/editor_trends/etl/transformer.py
@@ -48,15 +48,12 @@
4949
5050
5151 class Editor:
52 - def __init__(self, db_raw, db_dataset, editor_id, **kwargs):
53 - self.editor_id = editor_id
54 - self.db_raw = db_raw
55 - self.db_dataset = db_dataset
56 - for kw in kwargs:
57 - setattr(self, kw, kwargs[kw])
 52+ def __init__(self, rts, editor_id):
 53+ self.db_raw = storage.init_database(rts.storage, rts.dbname, rts.editors_raw)
 54+ self.db_dataset = storage.init_database(rts.storage, rts.dbname, rts.editors_dataset)
5855
5956 def __str__(self):
60 - return '%s' % (self.id)
 57+ return '%s' % (self.editor_id)
6158
6259 def __call__(self):
6360 cutoff = 9