| Index: trunk/tools/editor_trends/cronjobs.py |
| — | — | @@ -19,14 +19,16 @@ |
| 20 | 20 | |
| 21 | 21 | from multiprocessing import Process |
| 22 | 22 | |
| | 23 | +import manage as manager |
| 23 | 24 | |
| 24 | 25 | from database import db |
| 25 | 26 | from classes import wikiprojects |
| 26 | | -import manage as manager |
| | 27 | +from analyses import analyzer |
| 27 | 28 | |
| | 29 | + |
| 28 | 30 | def launch_editor_trends_toolkit(task): |
| 29 | 31 | ''' |
| 30 | | - This function should only be called as a cronjob. |
| | 32 | + This function should only be called as a cronjob and not directly. |
| 31 | 33 | ''' |
| 32 | 34 | parser, settings, wiki = manager.init_args_parser() |
| 33 | 35 | args = parser.parse_args(['django']) |
| — | — | @@ -34,37 +36,77 @@ |
| 35 | 37 | args.project = task['project'] |
| 36 | 38 | print args |
| 37 | 39 | wiki = wikiprojects.Wiki(settings, args) |
| 38 | | - manager.all_launcher(wiki, settings, None) |
| 39 | | - #wiki.update_language(language) |
| 40 | | - #wiki.update_project(project) |
| | 40 | + res = manager.all_launcher(wiki, settings, None) |
| | 41 | + return res |
| 41 | 42 | |
| 42 | | - #p = Process(target=manager.all_launcher, args=(wiki, settings, None)) |
| 43 | | - #p.start() |
| 44 | 43 | |
| 45 | | -def launch_chart(project, language): |
| 46 | | - pass |
| | 44 | +def launch_chart(task): |
| | 45 | + ''' |
| | 46 | + This function should only be called as a cronjob and not directly. |
| | 47 | + ''' |
| | 48 | + res = True |
| | 49 | + try: |
| | 50 | + project = task['project'] |
| | 51 | + language_code = task['language_code'] |
| | 52 | + func = task['jobtype'] |
| 47 | 53 | |
| | 54 | + collection = 'editors_dataset' #FIXME hardcoded string |
| | 55 | + time_unit = 'month' #FIXME hardcoded string |
| | 56 | + cutoff = 1 #FIXME hardcoded string |
| | 57 | + cum_cutoff = 50 #FIXME hardcoded string |
| 48 | 58 | |
| | 59 | + analyzer.generate_chart_data(project, |
| | 60 | + collection, |
| | 61 | + language_code, |
| | 62 | + func, |
| | 63 | + time_unit=time_unit, |
| | 64 | + cutoff=cutoff, |
| | 65 | + cum_cutoff=cum_cutoff) |
| | 66 | + except AttributeError, e: |
| | 67 | + res = False |
| | 68 | + print e #need to capture more fine grained errors but not quite what errors are going to happen. |
| | 69 | + except Exception, e: |
| | 70 | + res = False |
| | 71 | + print e #need to capture more fine grained errors but not quite what errors are going to happen. |
| | 72 | + |
| | 73 | + return res |
| | 74 | + |
| | 75 | + |
| 49 | 76 | def launcher(): |
| | 77 | + ''' |
| | 78 | + This is the main entry point, it creates a queue with jobs and determines |
| | 79 | + the type of job and fires it off |
| | 80 | + ''' |
| 50 | 81 | mongo = db.init_mongo_db('wikilytics') |
| 51 | 82 | coll = mongo['jobs'] |
| 52 | 83 | tasks = [] |
| 53 | | - #'in_progress': False} |
| 54 | | - jobs = coll.find({'finished': False, 'in_progress': False}) |
| | 84 | + jobs = coll.find({'finished': False, 'in_progress': False, 'error': False}) |
| 55 | 85 | for job in jobs: |
| 56 | 86 | tasks.append(job) |
| 57 | 87 | |
| 58 | 88 | for task in tasks: |
| 59 | 89 | if task['jobtype'] == 'dataset': |
| 60 | | - launch_editor_trends_toolkit(task) |
| 61 | | - elif task['jobtype'] == 'chart': |
| 62 | | - launch_chart(task) |
| | 90 | + print 'Launching the Editor Trends Analytics Toolkit.' |
| | 91 | + res = launch_editor_trends_toolkit(task) |
| | 92 | + #res = False |
| | 93 | + else: |
| | 94 | + print 'Launching %s.' % task['jobtype'] |
| | 95 | + res = launch_chart(task) |
| 63 | 96 | |
| | 97 | + if res: |
| | 98 | + coll.update({'_id': task['_id']}, {'$set': {'finished': True}}) |
| | 99 | + else: |
| | 100 | + ''' |
| | 101 | + To prevent jobs from recurring non-stop, set error to True. These |
| | 102 | + jobs will be excluded and need to be investigated to see what's |
| | 103 | + happening. |
| | 104 | + ''' |
| | 105 | + coll.update({'_id': task['_id']}, {'$set': {'error': True}}) |
| | 106 | + |
| | 107 | + |
| 64 | 108 | def debug(): |
| 65 | | - project = 'wiki' |
| 66 | | - language = 'en' |
| 67 | | - #launch_editor_trends_toolkit(project, language) |
| 68 | 109 | launcher() |
| 69 | 110 | |
| | 111 | + |
| 70 | 112 | if __name__ == '__main__': |
| 71 | 113 | debug() |