| Index: branches/ariel/xmldumps-backup/incrementals/IncrDumpLib.py |
| — | — | @@ -10,6 +10,7 @@ |
| 11 | 11 | import subprocess |
| 12 | 12 | from subprocess import Popen, PIPE |
| 13 | 13 | import shutil |
| | 14 | +import time |
| 14 | 15 | |
| 15 | 16 | class ContentFile(object): |
| 16 | 17 | def __init__(self, config, date, wikiName): |
| — | — | @@ -114,6 +115,18 @@ |
| 115 | 116 | except: |
| 116 | 117 | return False |
| 117 | 118 | |
| | 119 | + def isStaleLock(self): |
| | 120 | + if not self.isLocked(): |
| | 121 | + return False |
| | 122 | + try: |
| | 123 | + timestamp = os.stat(self.lockFile.getPath()).st_mtime |
| | 124 | + except: |
| | 125 | + return False |
| | 126 | + if (time.time() - timestamp) > self._config.staleInterval: |
| | 127 | + return True |
| | 128 | + else: |
| | 129 | + return False |
| | 130 | + |
| 118 | 131 | def unlock(self): |
| 119 | 132 | os.remove(self.lockFile.getPath()) |
| 120 | 133 | |
| — | — | @@ -161,6 +174,7 @@ |
| 162 | 175 | "webroot": "http://localhost/dumps/incr", |
| 163 | 176 | "fileperms": "0640", |
| 164 | 177 | "delay": "43200", |
| | 178 | + "maxrevidstaleinterval": "3600", |
| 165 | 179 | #"database": { |
| 166 | 180 | "user": "root", |
| 167 | 181 | "password": "", |
| — | — | @@ -206,6 +220,8 @@ |
| 207 | 221 | self.fileperms = int(self.fileperms,0) |
| 208 | 222 | self.delay = self.conf.get("output", "delay") |
| 209 | 223 | self.delay = int(self.delay,0) |
| | 224 | + self.staleInterval = self.conf.get("output", "maxrevidstaleinterval") |
| | 225 | + self.staleInterval = int(self.staleInterval,0) |
| 210 | 226 | |
| 211 | 227 | if not self.conf.has_section('tools'): |
| 212 | 228 | self.conf.add_section('tools') |
| Index: branches/ariel/xmldumps-backup/incrementals/generatemaxrevids.py |
| — | — | @@ -40,11 +40,12 @@ |
| 41 | 41 | return exists(self.maxRevIdFile.getPath()) |
| 42 | 42 | |
| 43 | 43 | class MaxIDDump(object): |
| 44 | | - def __init__(self,config, date, verbose): |
| | 44 | + def __init__(self,config, date, verbose, cleanupStale): |
| 45 | 45 | self._config = config |
| 46 | 46 | self.date = date |
| 47 | 47 | self.incrDir = IncrementDir(self._config, self.date) |
| 48 | 48 | self.verbose = verbose |
| | 49 | + self.cleanupStale = cleanupStale |
| 49 | 50 | |
| 50 | 51 | def doOneWiki(self, w): |
| 51 | 52 | success = True |
| — | — | @@ -52,7 +53,23 @@ |
| 53 | 54 | if not exists(self.incrDir.getIncDir(w)): |
| 54 | 55 | os.makedirs(self.incrDir.getIncDir(w)) |
| 55 | 56 | lock = MaxRevIDLock(self._config, self.date, w) |
| 56 | | - if lock.getLock(): |
| | 57 | + lockResult = lock.getLock() |
| | 58 | + if not lockResult: |
| | 59 | + if (self.verbose): |
| | 60 | + print "failed to get lock for wiki", w |
| | 61 | + if lock.isStaleLock(): |
| | 62 | + if (self.verbose): |
| | 63 | + print "lock is stale for wiki", w |
| | 64 | + # this option should be given to one process only, or you could have trouble. |
| | 65 | + if (self.cleanupStale): |
| | 66 | + lock.unlock() |
| | 67 | + lockResult = lock.getLock() |
| | 68 | + if (self.verbose): |
| | 69 | + print "stale lock removed and trying again to get for wiki", w |
| | 70 | + if lockResult: |
| | 71 | + if (self.verbose): |
| | 72 | + print "got lock ",lock.lockFile.getFileName() |
| | 73 | + print "checking max rev id for wiki", w |
| 57 | 74 | try: |
| 58 | 75 | maxRevID = MaxRevID(self._config, w, self.date) |
| 59 | 76 | if not maxRevID.exists(): |
| — | — | @@ -66,10 +83,10 @@ |
| 67 | 84 | else: |
| 68 | 85 | if (self.verbose): |
| 69 | 86 | print "Wiki ", w, "failed to get lock." |
| 70 | | - traceback.print_exc(file=sys.stdout) |
| | 87 | + success = False |
| 71 | 88 | if success: |
| 72 | 89 | if (self.verbose): |
| 73 | | - print "Success! Wiki", w, "adds/changes dump complete." |
| | 90 | + print "Success! Wiki", w, "rev max id for adds/changes dump complete." |
| 74 | 91 | return success |
| 75 | 92 | |
| 76 | 93 | def doRunOnAllWikis(self): |
| — | — | @@ -96,11 +113,13 @@ |
| 97 | 114 | print message |
| 98 | 115 | print "Usage: python generateincrementals.py [options] [wikidbname]" |
| 99 | 116 | print "Options: --configfile, --date, --verbose" |
| 100 | | - print "--configfile: Specify an alternate config file to read. Default file is 'dumpincr.conf' in the current directory." |
| 101 | | - print "--date: (Re)run incremental of a given date (use with care)." |
| 102 | | - print "--verbose: Print error messages and other informative messages (normally the" |
| 103 | | - print " script runs silently)." |
| 104 | | - print "wikiname: Run the dumps only for the specific wiki." |
| | 117 | + print "--configfile: Specify an alternate config file to read. Default file is 'dumpincr.conf' in the current directory." |
| | 118 | + print "--cleanupstale: Clean up stale lock files as they are encountered. Only one process running should ever have this" |
| | 119 | + print " option set." |
| | 120 | + print "--date: (Re)run incremental of a given date (use with care)." |
| | 121 | + print "--verbose: Print error messages and other informative messages (normally the" |
| | 122 | + print " script runs silently)." |
| | 123 | + print "wikiname: Run the dumps only for the specific wiki." |
| 105 | 124 | sys.exit(1) |
| 106 | 125 | |
| 107 | 126 | if __name__ == "__main__": |
| — | — | @@ -108,10 +127,11 @@ |
| 109 | 128 | result = False |
| 110 | 129 | date = None |
| 111 | 130 | verbose = False |
| | 131 | + cleanupStale = False |
| 112 | 132 | |
| 113 | 133 | try: |
| 114 | 134 | (options, remainder) = getopt.gnu_getopt(sys.argv[1:], "", |
| 115 | | - ['date=', 'configfile=', 'verbose' ]) |
| | 135 | + ['date=', 'configfile=', 'verbose', 'cleanupstale' ]) |
| 116 | 136 | except: |
| 117 | 137 | usage("Unknown option specified") |
| 118 | 138 | |
| — | — | @@ -122,6 +142,8 @@ |
| 123 | 143 | configFile = val |
| 124 | 144 | elif opt == "--verbose": |
| 125 | 145 | verbose = True |
| | 146 | + elif opt == "--cleanupstale": |
| | 147 | + cleanupStale = True |
| 126 | 148 | |
| 127 | 149 | if (configFile): |
| 128 | 150 | config = Config(configFile) |
| — | — | @@ -131,7 +153,7 @@ |
| 132 | 154 | if not date: |
| 133 | 155 | date = TimeUtils.today() |
| 134 | 156 | |
| 135 | | - dump = MaxIDDump(config, date, verbose) |
| | 157 | + dump = MaxIDDump(config, date, verbose, cleanupStale) |
| 136 | 158 | if len(remainder) > 0: |
| 137 | 159 | dump.doOneWiki(remainder[0]) |
| 138 | 160 | else: |