| Index: trunk/tools/editor_trends/etl/enricher.py |
| — | — | @@ -35,7 +35,6 @@ |
| 36 | 36 | try: |
| 37 | 37 | from database import cassandra |
| 38 | 38 | import pycassa |
| 39 | | - |
| 40 | 39 | except ImportError: |
| 41 | 40 | pass |
| 42 | 41 | |
| — | — | @@ -699,6 +698,7 @@ |
| 700 | 699 | filename = input_queue.get() |
| 701 | 700 | input_queue.task_done() |
| 702 | 701 | if filename == None: |
| | 702 | + print '%s files left in the queue' % input_queue.qsize() |
| 703 | 703 | break |
| 704 | 704 | |
| 705 | 705 | fh = file_utils.create_streaming_buffer(filename) |
| — | — | @@ -765,14 +765,13 @@ |
| 766 | 766 | else: |
| 767 | 767 | processors = len(files) |
| 768 | 768 | |
| 769 | | - #files = files[0:1] |
| 770 | | - print rts.input_location, rts.location |
| 771 | 769 | for filename in files: |
| 772 | 770 | filename = os.path.join(rts.location, filename) |
| 773 | 771 | print filename |
| 774 | 772 | input_queue.put(filename) |
| 775 | 773 | |
| 776 | 774 | for x in xrange(processors): |
| | 775 | + print 'Inserting poison pill %s...' % x |
| 777 | 776 | input_queue.put(None) |
| 778 | 777 | |
| 779 | 778 | extracters = [Process(target=stream_raw_xml, args=[input_queue, storage, |