| Index: trunk/tools/editor_trends/analyses/plugins/taxonomy_list_makers.py |
| — | — | @@ -66,7 +66,7 @@ |
| 67 | 67 | |
| 68 | 68 | def preload(rts): |
| 69 | 69 | collection = '%s%s_articles_raw' % (rts.language.code, rts.project.name) |
| 70 | | - db = storage.Database(rts.storage, rts.dbname, collection) |
| | 70 | + db = storage.init_database(rts.storage, rts.dbname, collection) |
| 71 | 71 | data = {} |
| 72 | 72 | cursor = db.find('category', 'List') |
| 73 | 73 | for c in cursor: |
| Index: trunk/tools/editor_trends/analyses/plugins/ppi_editor_productivity.py |
| — | — | @@ -26,7 +26,6 @@ |
| 27 | 27 | |
| 28 | 28 | edits = editor['character_count'] |
| 29 | 29 | username = editor['username'] |
| 30 | | - |
| 31 | 30 | try: |
| 32 | 31 | added = edits['2010']['11']['0']['added'] |
| 33 | 32 | except KeyError: |
| Index: trunk/tools/editor_trends/analyses/json_encoders.py |
| — | — | @@ -13,7 +13,7 @@ |
| 14 | 14 | ''' |
| 15 | 15 | |
| 16 | 16 | __author__ = '''\n'''.join(['Diederik van Liere (dvanliere@gmail.com)', ]) |
| 17 | | -__author__email = 'dvanliere at gmail dot com' |
| | 17 | +__email__ = 'dvanliere at gmail dot com' |
| 18 | 18 | __date__ = '2011-01-27' |
| 19 | 19 | __version__ = '0.1' |
| 20 | 20 | |
| Index: trunk/tools/editor_trends/analyses/inventory.py |
| — | — | @@ -13,14 +13,13 @@ |
| 14 | 14 | ''' |
| 15 | 15 | |
| 16 | 16 | __author__ = '''\n'''.join(['Diederik van Liere (dvanliere@gmail.com)', ]) |
| 17 | | -__author__email = 'dvanliere at gmail dot com' |
| | 17 | +__email__ = 'dvanliere at gmail dot com' |
| 18 | 18 | __date__ = '2011-02-11' |
| 19 | 19 | __version__ = '0.1' |
| 20 | 20 | |
| 21 | 21 | |
| 22 | 22 | import os |
| 23 | 23 | import sys |
| 24 | | -import types |
| 25 | 24 | |
| 26 | 25 | def available_analyses(caller='manage'): |
| 27 | 26 | ''' |
| — | — | @@ -30,24 +29,17 @@ |
| 31 | 30 | ignore: a list of functions that should never be called from manage.py, |
| 32 | 31 | they are not valid entry points. |
| 33 | 32 | ''' |
| 34 | | - assert caller == 'django' or caller == 'manage' |
| 35 | | - ignore = ['__init__'] |
| 36 | | - charts = {} |
| | 33 | + assert caller == 'webpy' or caller == 'manage' |
| 37 | 34 | |
| 38 | 35 | fn = os.path.realpath(__file__) |
| 39 | 36 | pos = fn.rfind(os.sep) |
| 40 | 37 | loc = fn[:pos] |
| 41 | | - path = os.path.join(loc , 'plugins') |
| | 38 | + path = os.path.join(loc, 'plugins') |
| 42 | 39 | modules = import_libs(path) |
| 43 | 40 | |
| 44 | | -# for module_name, module in modules.iteritems(): |
| 45 | | -# func = getattr(module, module_name) |
| 46 | | -# plugin = module() |
| 47 | | -# if isinstance(plugin, types.FunctionType) and plugin.func_name not in ignore: |
| 48 | | -# charts[plugin.func_name] = plugin |
| 49 | 41 | if caller == 'manage': |
| 50 | 42 | return modules |
| 51 | | - elif caller == 'django': |
| | 43 | + elif caller == 'webpy': |
| 52 | 44 | django_functions = [] |
| 53 | 45 | for module in modules: |
| 54 | 46 | fancy_name = module.replace('_', ' ').title() |
| — | — | @@ -62,9 +54,9 @@ |
| 63 | 55 | ''' |
| 64 | 56 | plugins = {} |
| 65 | 57 | sys.path.append(path) |
| 66 | | - for f in os.listdir(os.path.abspath(path)): |
| 67 | | - module_name, ext = os.path.splitext(f) |
| 68 | | - if ext == '.py': |
| | 58 | + for filename in os.listdir(os.path.abspath(path)): |
| | 59 | + module_name, ext = os.path.splitext(filename) |
| | 60 | + if ext == '.py' and module_name != '__init__': |
| 69 | 61 | module = __import__(module_name) |
| 70 | 62 | plugins[module_name] = module |
| 71 | 63 | |
| Index: trunk/tools/editor_trends/analyses/analyzer.py |
| — | — | @@ -74,9 +74,9 @@ |
| 75 | 75 | |
| 76 | 76 | |
| 77 | 77 | def write_output(ds, rts, stopwatch): |
| 78 | | - ds.create_filename() |
| | 78 | + filename = ds.create_filename() |
| 79 | 79 | print 'Storing dataset: %s' % os.path.join(rts.dataset_location, |
| 80 | | - ds.filename) |
| | 80 | + filename) |
| 81 | 81 | ds.write(format='csv') |
| 82 | 82 | print 'Serializing dataset to %s_%s' % (rts.dbname, 'charts') |
| 83 | 83 | #log.log_to_mongo(rts, 'chart', 'storing', stopwatch, event='start') |
| — | — | @@ -97,15 +97,15 @@ |
| 98 | 98 | |
| 99 | 99 | feedback(func, rts) |
| 100 | 100 | |
| 101 | | - obs = dict() |
| 102 | 101 | tasks = JoinableQueue() |
| 103 | 102 | result = JoinableQueue() |
| 104 | 103 | |
| 105 | 104 | mgr = Manager() |
| 106 | 105 | lock = mgr.RLock() |
| | 106 | + obs = dict() |
| 107 | 107 | obs_proxy = mgr.dict(obs) |
| 108 | 108 | |
| 109 | | - db = storage.Database(rts.storage, rts.dbname, rts.editors_dataset) |
| | 109 | + db = storage.init_database(rts.storage, rts.dbname, rts.editors_dataset) |
| 110 | 110 | editors = db.retrieve_distinct_keys('editor') |
| 111 | 111 | min_year, max_year = determine_project_year_range(db, 'new_wikipedian') |
| 112 | 112 | |
| — | — | @@ -118,11 +118,12 @@ |
| 119 | 119 | var = dataset.Variable('count', time_unit, lock, obs_proxy, **kwargs) |
| 120 | 120 | |
| 121 | 121 | try: |
| 122 | | - print 'Preloading data...' |
| | 122 | + print 'Determinging whether plugin requires preloaded data...' |
| 123 | 123 | preloader = getattr(plugin, 'preload') |
| 124 | 124 | data = preloader(rts) |
| | 125 | + print 'Finished preloading data...' |
| 125 | 126 | except Exception, error: |
| 126 | | - print error |
| | 127 | + print Exception, error |
| 127 | 128 | data = None |
| 128 | 129 | finally: |
| 129 | 130 | print 'Finished preloading data.' |
| — | — | @@ -131,7 +132,7 @@ |
| 132 | 133 | for editor in editors: |
| 133 | 134 | tasks.put(analytics.Task(plugin, editor)) |
| 134 | 135 | |
| 135 | | - consumers = [analytics.Analyzer(rts, tasks, result, var, data) for |
| | 136 | + analyzers = [analytics.Analyzer(rts, tasks, result, var, data) for |
| 136 | 137 | x in xrange(rts.number_of_processes)] |
| 137 | 138 | |
| 138 | 139 | |
| — | — | @@ -139,12 +140,11 @@ |
| 140 | 141 | tasks.put(None) |
| 141 | 142 | |
| 142 | 143 | pbar = progressbar.ProgressBar(maxval=len(editors)).start() |
| 143 | | - for w in consumers: |
| 144 | | - w.start() |
| | 144 | + for analyzer in analyzers: |
| | 145 | + analyzer.start() |
| 145 | 146 | |
| 146 | 147 | |
| 147 | 148 | ppills = rts.number_of_processes |
| 148 | | - vars = [] |
| 149 | 149 | while True: |
| 150 | 150 | while ppills > 0: |
| 151 | 151 | try: |