| Index: trunk/tools/editor_trends/etl/enricher.py |
| — | — | @@ -238,29 +238,30 @@ |
| 239 | 239 | #print '%s articles took %s' % (len(self.articles.keys()), (t1 - t0)) |
| 240 | 240 | |
| 241 | 241 | def write_revisions(self): |
| 242 | | - #t0 = datetime.datetime.now() |
| | 242 | + t0 = datetime.datetime.now() |
| 243 | 243 | self.group_revisions_by_fileid() |
| 244 | 244 | file_ids = self.revisions.keys() |
| 245 | | - while len(self.revisions.keys()) > 0: |
| 246 | | - print len(self.revisions.keys()) |
| 247 | | - for file_id in file_ids: |
| 248 | | - for i, revision in enumerate(self.revisions[file_id]): |
| 249 | | - if i == 0: |
| 250 | | - #file_id = self.get_hash(revision[2]) |
| | 245 | + for file_id in file_ids: |
| | 246 | + wait = True |
| | 247 | + for i, revision in enumerate(self.revisions[file_id]): |
| | 248 | + if i == 0: |
| | 249 | + while wait: |
| | 250 | + print file_id, self.lock |
| 251 | 251 | if self.lock.available(file_id): |
| 252 | 252 | fh = self.filehandles[file_id] |
| 253 | | - #print editor, file_id, fh |
| 254 | | - else: |
| 255 | | - break |
| 256 | | - try: |
| 257 | | - file_utils.write_list_to_csv(revision, fh) |
| 258 | | - except Exception, error: |
| 259 | | - print '''Encountered the following error while writing |
| | 253 | + wait = False |
| | 254 | + try: |
| | 255 | + file_utils.write_list_to_csv(revision, fh) |
| | 256 | + except Exception, error: |
| | 257 | + print '''Encountered the following error while writing |
| 260 | 258 | revision data to %s: %s''' % (fh, error) |
| 261 | | - self.lock.release(file_id) |
| 262 | | - del self.revisions[file_id] |
| 263 | | - #t1 = datetime.datetime.now() |
| 264 | | - #print '%s revisions took %s' % (len(self.revisions), (t1 - t0)) |
| | 259 | + self.lock.release(file_id) |
| | 260 | + del self.revisions[file_id] |
| | 261 | + wait = True |
| | 262 | + t1 = datetime.datetime.now() |
| | 263 | + print 'Worker %s: %s revisions took %s' % (self.process_id, |
| | 264 | + len(self.revisions), |
| | 265 | + (t1 - t0)) |
| 265 | 266 | |
| 266 | 267 | def extract_categories(): |
| 267 | 268 | ''' |