| Index: trunk/tools/editor_trends/classes/buffer.py |
| — | — | @@ -80,7 +80,7 @@ |
| 81 | 81 | self.n = 0 |
| 82 | 82 | self.filehandles = [file_utils.create_txt_filehandle(self.rts.txt, |
| 83 | 83 | file_id, 'a', 'utf-8') for file_id in xrange(self.rts.max_filehandles)] |
| 84 | | - self.keys = ['revision_id', 'article_id', 'id', 'username', 'namespace', |
| | 84 | + self.keys = ['id', 'article_id', 'revision_id', 'username', 'namespace', |
| 85 | 85 | 'title', 'timestamp', 'hash', 'revert', 'bot', 'cur_size', |
| 86 | 86 | 'delta'] |
| 87 | 87 | self.fh_articles = file_utils.create_txt_filehandle(self.rts.txt, |
| Index: trunk/tools/editor_trends/classes/storage.py |
| — | — | @@ -120,14 +120,14 @@ |
| 121 | 121 | assert isinstance(data, dict), 'You need to feed me dictionaries.' |
| 122 | 122 | return self.db[self.collection].save(data) |
| 123 | 123 | |
| 124 | | - def insert(self, data, qualifiers=None): |
| | 124 | + def insert(self, data, qualifiers=None, safe=False): |
| 125 | 125 | assert isinstance(data, dict), 'You need to feed me dictionaries.' |
| 126 | 126 | data = self.stringify_keys(data) |
| 127 | 127 | try: |
| 128 | 128 | if qualifiers: |
| 129 | | - self.db[self.collection].insert(data, qualifiers, safe=True) |
| | 129 | + self.db[self.collection].insert(data, qualifiers, safe=safe) |
| 130 | 130 | else: |
| 131 | | - self.db[self.collection].insert(data, safe=True) |
| | 131 | + self.db[self.collection].insert(data, safe=safe) |
| 132 | 132 | except bson.errors.InvalidDocument, error: |
| 133 | 133 | print error |
| 134 | 134 | print 'BSON document too large, unable to store %s' % \ |
| Index: trunk/tools/editor_trends/database/cache.py |
| — | — | @@ -54,6 +54,7 @@ |
| 55 | 55 | def add(self, key, value): |
| 56 | 56 | if value == 'NEXT': |
| 57 | 57 | self.n += 1 |
| | 58 | + print self.editors |
| 58 | 59 | edits = self.drop_years_no_obs(self.editors[key]['edits']) |
| 59 | 60 | self.insert(key, edits, self.editors[key]['username']) |
| 60 | 61 | del self.editors[key] |
| — | — | @@ -72,19 +73,7 @@ |
| 73 | 74 | |
| 74 | 75 | def insert(self, editor, values, username): |
| 75 | 76 | data = {'editor': editor, 'edits': values, 'username': username} |
| 76 | | - self.db.insert(data) |
| 77 | | -# ''' |
| 78 | | -# Adding the safe=True statement slows down the insert process but this |
| 79 | | -# assures that all data will be written. |
| 80 | | -# ''' |
| 81 | | -# try: |
| 82 | | -# self.collection.insert({'editor': editor, 'edits': values, 'username': username}, safe=True) |
| 83 | | -# except bson.errors.InvalidDocument: |
| 84 | | -# print 'BSON document too large, unable to store %s' % (username) |
| 85 | | -# except OperationFailure, error: |
| 86 | | -# print error |
| 87 | | -# print 'It seems that you are running out of disk space.' |
| 88 | | -# sys.exit(-1) |
| | 77 | + self.db.insert(data, safe=False) |
| 89 | 78 | |
| 90 | 79 | def store(self): |
| 91 | 80 | file_utils.store_object(self, settings.binary_location, self.__repr__()) |