| Index: branches/ariel/xmldumps-backup/WikiDump.py |
| — | — | @@ -9,75 +9,120 @@ |
| 10 | 10 | import time |
| 11 | 11 | import tempfile |
| 12 | 12 | |
| 13 | | -def fileAge(filename): |
| 14 | | - return time.time() - os.stat(filename).st_mtime |
| | 13 | +class FileUtils(object): |
| 15 | 14 | |
| 16 | | -def atomicCreate(filename, mode='w'): |
| 17 | | - """Create a file, aborting if it already exists...""" |
| 18 | | - fd = os.open(filename, os.O_EXCL + os.O_CREAT + os.O_WRONLY) |
| 19 | | - return os.fdopen(fd, mode) |
| | 15 | + def fileAge(filename): |
| | 16 | + return time.time() - os.stat(filename).st_mtime |
| 20 | 17 | |
| 21 | | -def shellEscape(param): |
| 22 | | - """Escape a string parameter, or set of strings, for the shell.""" |
| 23 | | - if isinstance(param, basestring): |
| 24 | | - return "'" + param.replace("'", "'\\''") + "'" |
| 25 | | - elif param is None: |
| 26 | | - # A blank string might actually be needed; None means we can leave it out |
| 27 | | - return "" |
| 28 | | - else: |
| 29 | | - return tuple([shellEscape(x) for x in param]) |
| | 18 | + def atomicCreate(filename, mode='w'): |
| | 19 | + """Create a file, aborting if it already exists...""" |
| | 20 | + fd = os.open(filename, os.O_EXCL + os.O_CREAT + os.O_WRONLY) |
| | 21 | + return os.fdopen(fd, mode) |
| 30 | 22 | |
| 31 | | -def prettySize(size): |
| 32 | | - """Return a string with an attractively formatted file size.""" |
| 33 | | - quanta = ("%d bytes", "%d KB", "%0.1f MB", "%0.1f GB", "%0.1f TB") |
| 34 | | - return _prettySize(size, quanta) |
| | 23 | + def writeFile(dirname, filename, text): |
| | 24 | + """Write text to a file, as atomically as possible, via a temporary file in the same directory.""" |
| | 25 | + |
| | 26 | + (fd, tempFilename ) = tempfile.mkstemp("_txt","wikidump_",dirname); |
| | 27 | + os.write(fd,text) |
| | 28 | + os.close(fd) |
| | 29 | + # This may fail across filesystems or on Windows. |
| | 30 | + # Of course nothing else will work on Windows. ;) |
| | 31 | + os.rename(tempFilename, filename) |
| 35 | 32 | |
| 36 | | -def _prettySize(size, quanta): |
| 37 | | - if size < 1024 or len(quanta) == 1: |
| 38 | | - return quanta[0] % size |
| 39 | | - else: |
| 40 | | - return _prettySize(size / 1024.0, quanta[1:]) |
| | 33 | + def readFile(filename): |
| | 34 | + """Read text from a file in one fell swoop.""" |
| | 35 | + file = open(filename, "r") |
| | 36 | + text = file.read() |
| | 37 | + file.close() |
| | 38 | + return text |
| 41 | 39 | |
| 42 | | -def today(): |
| 43 | | - return time.strftime("%Y%m%d", time.gmtime()) |
| | 40 | + def splitPath(path): |
| | 41 | + # For some reason, os.path.split only does one level. |
| | 42 | + parts = [] |
| | 43 | + (path, file) = os.path.split(path) |
| | 44 | + if not file: |
| | 45 | + # Probably a final slash |
| | 46 | + (path, file) = os.path.split(path) |
| | 47 | + while file: |
| | 48 | + parts.insert(0, file) |
| | 49 | + (path, file) = os.path.split(path) |
| | 50 | + return parts |
| 44 | 51 | |
| 45 | | -def prettyTime(): |
| 46 | | - return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) |
| | 52 | + def relativePath(path, base): |
| | 53 | + """Return a relative path to 'path' from the directory 'base'.""" |
| | 54 | + path = FileUtils.splitPath(path) |
| | 55 | + base = FileUtils.splitPath(base) |
| | 56 | + while base and path[0] == base[0]: |
| | 57 | + path.pop(0) |
| | 58 | + base.pop(0) |
| | 59 | + for prefix in base: |
| | 60 | + path.insert(0, "..") |
| | 61 | + return os.path.join(*path) |
| 47 | 62 | |
| 48 | | -def prettyDate(key): |
| 49 | | - "Prettify a MediaWiki date key" |
| 50 | | - return "-".join((key[0:4], key[4:6], key[6:8])) |
| | 63 | + def prettySize(size): |
| | 64 | + """Return a string with an attractively formatted file size.""" |
| | 65 | + quanta = ("%d bytes", "%d KB", "%0.1f MB", "%0.1f GB", "%0.1f TB") |
| | 66 | + return FileUtils._prettySize(size, quanta) |
| 51 | 67 | |
| 52 | | -def dumpFile(dirname, filename, text): |
| 53 | | - """Dump a string to a file, as atomically as possible, via a temporary file in the same directory.""" |
| 54 | | - |
| 55 | | - (fd, tempFilename ) = tempfile.mkstemp("_txt","wikidump_",dirname); |
| 56 | | - os.write(fd,text) |
| 57 | | - os.close(fd) |
| 58 | | - # This may fail across filesystems or on Windows. |
| 59 | | - # Of course nothing else will work on Windows. ;) |
| 60 | | - os.rename(tempFilename, filename) |
| | 68 | + def _prettySize(size, quanta): |
| | 69 | + if size < 1024 or len(quanta) == 1: |
| | 70 | + return quanta[0] % size |
| | 71 | + else: |
| | 72 | + return FileUtils._prettySize(size / 1024.0, quanta[1:]) |
| 61 | 73 | |
| 62 | | -def readFile(filename): |
| 63 | | - file = open(filename, "r") |
| 64 | | - text = file.read() |
| 65 | | - file.close() |
| 66 | | - return text |
| | 74 | + fileAge = staticmethod(fileAge) |
| | 75 | + atomicCreate = staticmethod(atomicCreate) |
| | 76 | + writeFile = staticmethod(writeFile) |
| | 77 | + readFile = staticmethod(readFile) |
| | 78 | + splitPath = staticmethod(splitPath) |
| | 79 | + relativePath = staticmethod(relativePath) |
| | 80 | + prettySize = staticmethod(prettySize) |
| | 81 | + _prettySize = staticmethod(_prettySize) |
| 67 | 82 | |
| 68 | | -def dbList(filename): |
| 69 | | - """Read database list from a file""" |
| 70 | | - if (not filename): |
| 71 | | - return [] |
| 72 | | - infile = open(filename) |
| 73 | | - dbs = [] |
| 74 | | - for line in infile: |
| 75 | | - line = line.strip() |
| 76 | | - if line != "": |
| 77 | | - dbs.append(line) |
| 78 | | - infile.close() |
| 79 | | - dbs.sort() |
| 80 | | - return dbs |
| | 83 | +class TimeUtils(object): |
| 81 | 84 | |
| | 85 | + def today(): |
| | 86 | + return time.strftime("%Y%m%d", time.gmtime()) |
| | 87 | + |
| | 88 | + def prettyTime(): |
| | 89 | + return time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()) |
| | 90 | + |
| | 91 | + def prettyDate(key): |
| | 92 | + "Prettify a MediaWiki date key" |
| | 93 | + return "-".join((key[0:4], key[4:6], key[6:8])) |
| | 94 | + |
| | 95 | + today = staticmethod(today) |
| | 96 | + prettyTime = staticmethod(prettyTime) |
| | 97 | + prettyDate = staticmethod(prettyDate) |
| | 98 | + |
| | 99 | +class MiscUtils(object): |
| | 100 | + def dbList(filename): |
| | 101 | + """Read database list from a file""" |
| | 102 | + if (not filename): |
| | 103 | + return [] |
| | 104 | + infile = open(filename) |
| | 105 | + dbs = [] |
| | 106 | + for line in infile: |
| | 107 | + line = line.strip() |
| | 108 | + if line != "": |
| | 109 | + dbs.append(line) |
| | 110 | + infile.close() |
| | 111 | + dbs.sort() |
| | 112 | + return dbs |
| | 113 | + |
| | 114 | + def shellEscape(param): |
| | 115 | + """Escape a string parameter, or set of strings, for the shell.""" |
| | 116 | + if isinstance(param, basestring): |
| | 117 | + return "'" + param.replace("'", "'\\''") + "'" |
| | 118 | + elif param is None: |
| | 119 | + # A blank string might actually be needed; None means we can leave it out |
| | 120 | + return "" |
| | 121 | + else: |
| | 122 | + return tuple([MiscUtils.shellEscape(x) for x in param]) |
| | 123 | + |
| | 124 | + dbList = staticmethod(dbList) |
| | 125 | + shellEscape = staticmethod(shellEscape) |
| | 126 | + |
| 82 | 127 | class Config(object): |
| 83 | 128 | def __init__(self, configFile=False): |
| 84 | 129 | home = os.path.dirname(sys.argv[0]) |
| — | — | @@ -155,11 +200,11 @@ |
| 156 | 201 | print "The mandatory setting 'dir' in the section 'wiki' was not defined." |
| 157 | 202 | raise ConfigParser.NoOptionError('wiki','dir') |
| 158 | 203 | |
| 159 | | - self.dbList = dbList(conf.get("wiki", "dblist")) |
| 160 | | - self.skipDbList = dbList(conf.get("wiki", "skipdblist")) |
| 161 | | - self.privateList = dbList(conf.get("wiki", "privatelist")) |
| 162 | | - self.bigList = dbList(conf.get("wiki", "biglist")) |
| 163 | | - self.flaggedRevsList = dbList(conf.get("wiki", "flaggedrevslist")) |
| | 204 | + self.dbList = MiscUtils.dbList(conf.get("wiki", "dblist")) |
| | 205 | + self.skipDbList = MiscUtils.dbList(conf.get("wiki", "skipdblist")) |
| | 206 | + self.privateList = MiscUtils.dbList(conf.get("wiki", "privatelist")) |
| | 207 | + self.bigList = MiscUtils.dbList(conf.get("wiki", "biglist")) |
| | 208 | + self.flaggedRevsList = MiscUtils.dbList(conf.get("wiki", "flaggedrevslist")) |
| 164 | 209 | self.wikiDir = conf.get("wiki", "dir") |
| 165 | 210 | self.forceNormal = conf.getint("wiki", "forceNormal") |
| 166 | 211 | self.halt = conf.getint("wiki", "halt") |
| — | — | @@ -248,8 +293,8 @@ |
| 249 | 294 | # tack on the file mtime so that if we have multiple wikis |
| 250 | 295 | # dumped on the same day, they get ordered properly |
| 251 | 296 | date = int(today()) - int(last) |
| 252 | | - age = fileAge(dumpStatus) |
| 253 | | - status = readFile(dumpStatus) |
| | 297 | + age = FileUtils.fileAge(dumpStatus) |
| | 298 | + status = FileUtils.readFile(dumpStatus) |
| 254 | 299 | except: |
| 255 | 300 | print "dump dir %s corrupt?" % dumpStatus |
| 256 | 301 | dumpFailed = (status == '') or ('dump aborted' in status) |
| — | — | @@ -259,7 +304,7 @@ |
| 260 | 305 | |
| 261 | 306 | def readTemplate(self, name): |
| 262 | 307 | template = os.path.join(self.templateDir, name) |
| 263 | | - return readFile(template) |
| | 308 | + return FileUtils.readFile(template) |
| 264 | 309 | |
| 265 | 310 | def mail(self, subject, body): |
| 266 | 311 | """Send out a quickie email.""" |
| — | — | @@ -330,7 +375,7 @@ |
| 331 | 376 | # Maybe it was just created (race condition)? |
| 332 | 377 | if not os.path.isdir(self.privateDir()): |
| 333 | 378 | raise |
| 334 | | - f = atomicCreate(self.lockFile(), "w") |
| | 379 | + f = FileUtils.atomicCreate(self.lockFile(), "w") |
| 335 | 380 | f.write("%s %d" % (socket.getfqdn(), os.getpid())) |
| 336 | 381 | f.close() |
| 337 | 382 | |
| — | — | @@ -358,7 +403,7 @@ |
| 359 | 404 | def writePerDumpIndex(self, html): |
| 360 | 405 | directory = os.path.join(self.publicDir(), self.date) |
| 361 | 406 | index = os.path.join(self.publicDir(), self.date, self.config.perDumpIndex) |
| 362 | | - dumpFile(directory, index, html) |
| | 407 | + FileUtils.writeFile(directory, index, html) |
| 363 | 408 | |
| 364 | 409 | def existsPerDumpIndex(self): |
| 365 | 410 | index = os.path.join(self.publicDir(), self.date, self.config.perDumpIndex) |
| — | — | @@ -367,14 +412,14 @@ |
| 368 | 413 | def writeStatus(self, message): |
| 369 | 414 | directory = os.path.join(self.publicDir(), self.date) |
| 370 | 415 | index = os.path.join(self.publicDir(), self.date, "status.html") |
| 371 | | - dumpFile(directory, index, message) |
| | 416 | + FileUtils.writeFile(directory, index, message) |
| 372 | 417 | |
| 373 | 418 | def statusLine(self): |
| 374 | 419 | date = self.latestDump() |
| 375 | 420 | if date: |
| 376 | 421 | status = os.path.join(self.publicDir(), date, "status.html") |
| 377 | 422 | try: |
| 378 | | - return readFile(status) |
| | 423 | + return FileUtils.readFile(status) |
| 379 | 424 | except: |
| 380 | 425 | return self.reportStatusLine("missing status record") |
| 381 | 426 | else: |
| — | — | @@ -383,9 +428,9 @@ |
| 384 | 429 | def reportStatusLine(self, status, error=False): |
| 385 | 430 | if error: |
| 386 | 431 | # No state information, hide the timestamp |
| 387 | | - stamp = "<span style=\"visible: none\">" + prettyTime() + "</span>" |
| | 432 | + stamp = "<span style=\"visible: none\">" + TimeUtils.prettyTime() + "</span>" |
| 388 | 433 | else: |
| 389 | | - stamp = prettyTime() |
| | 434 | + stamp = TimeUtils.prettyTime() |
| 390 | 435 | if self.isPrivate(): |
| 391 | 436 | link = "%s (private data)" % self.dbName |
| 392 | 437 | else: |
| — | — | @@ -426,7 +471,7 @@ |
| 427 | 472 | return os.path.join(self.privateDir(), "lock") |
| 428 | 473 | |
| 429 | 474 | def lockAge(self): |
| 430 | | - return fileAge(self.lockFile()) |
| | 475 | + return FileUtils.fileAge(self.lockFile()) |
| 431 | 476 | |
| 432 | 477 | class LockWatchdog(threading.Thread): |
| 433 | 478 | """Touch the given file every 10 seconds until asked to stop.""" |
| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -21,34 +21,13 @@ |
| 22 | 22 | |
| 23 | 23 | from os.path import dirname, exists, getsize, join, realpath |
| 24 | 24 | from subprocess import Popen, PIPE |
| 25 | | -from WikiDump import prettyTime, prettySize, shellEscape |
| | 25 | +#from WikiDump import FileUtils, DirUtils, MiscUtils, prettyTime, prettySize, shellEscape |
| | 26 | +from WikiDump import FileUtils, MiscUtils, TimeUtils |
| 26 | 27 | from CommandManagement import CommandPipeline, CommandSeries, CommandsInParallel |
| 27 | 28 | |
| 28 | | -def splitPath(path): |
| 29 | | - # For some reason, os.path.split only does one level. |
| 30 | | - parts = [] |
| 31 | | - (path, file) = os.path.split(path) |
| 32 | | - if not file: |
| 33 | | - # Probably a final slash |
| 34 | | - (path, file) = os.path.split(path) |
| 35 | | - while file: |
| 36 | | - parts.insert(0, file) |
| 37 | | - (path, file) = os.path.split(path) |
| 38 | | - return parts |
| 39 | | - |
| 40 | | -def relativePath(path, base): |
| 41 | | - """Return a relative path to 'path' from the directory 'base'.""" |
| 42 | | - path = splitPath(path) |
| 43 | | - base = splitPath(base) |
| 44 | | - while base and path[0] == base[0]: |
| 45 | | - path.pop(0) |
| 46 | | - base.pop(0) |
| 47 | | - for prefix in base: |
| 48 | | - path.insert(0, "..") |
| 49 | | - return os.path.join(*path) |
| 50 | | - |
| | 29 | +# FIXME test this change. |
| 51 | 30 | def xmlEscape(text): |
| 52 | | - return text.replace("&", "&").replace("<", "<").replace(">", ">") |
| | 31 | + return text.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """); |
| 53 | 32 | |
| 54 | 33 | class Logger(object): |
| 55 | 34 | |
| — | — | @@ -183,7 +162,7 @@ |
| 184 | 163 | |
| 185 | 164 | def defaultServer(self): |
| 186 | 165 | # if this fails what do we do about it? Not a bleeping thing. *ugh* FIXME!! |
| 187 | | - command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % shellEscape(( |
| | 166 | + command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % MiscUtils.shellEscape(( |
| 188 | 167 | self.config.php, self.config.wikiDir, self.dbName)) |
| 189 | 168 | return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| 190 | 169 | |
| — | — | @@ -238,7 +217,7 @@ |
| 239 | 218 | def getDBTablePrefix(self): |
| 240 | 219 | """Get the prefix for all tables for the specific wiki ($wgDBprefix)""" |
| 241 | 220 | # FIXME later full path |
| 242 | | - command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % shellEscape(( |
| | 221 | + command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % MiscUtils.shellEscape(( |
| 243 | 222 | self.config.php, self.config.wikiDir, self.dbName)) |
| 244 | 223 | return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| 245 | 224 | |
| — | — | @@ -257,37 +236,21 @@ |
| 258 | 237 | output = proc.fromchild.read() |
| 259 | 238 | retval = proc.wait() |
| 260 | 239 | while (retval and retries < maxretries): |
| 261 | | - if self.logCallback: |
| 262 | | - self.logCallback("Non-zero return code from '%s'" % command) |
| | 240 | + if logCallback: |
| | 241 | + logCallback("Non-zero return code from '%s'" % command) |
| 263 | 242 | time.sleep(5) |
| 264 | 243 | proc = popen2.Popen4(command, 64) |
| 265 | 244 | output = proc.fromchild.read() |
| 266 | 245 | retval = proc.wait() |
| 267 | 246 | retries = retries + 1 |
| 268 | 247 | if retval: |
| 269 | | - if self.logCallback: |
| 270 | | - self.logCallback("Non-zero return code from '%s'" % command) |
| | 248 | + if logCallback: |
| | 249 | + logCallback("Non-zero return code from '%s'" % command) |
| 271 | 250 | raise BackupError("Non-zero return code from '%s'" % command) |
| 272 | 251 | else: |
| 273 | 252 | return output |
| 274 | 253 | |
| 275 | | - def runAndReport(self, command, callback): |
| 276 | | - """Shell out to a command, and feed output lines to the callback function. |
| 277 | | - Returns the exit code from the program once complete. |
| 278 | | - stdout and stderr will be combined into a single stream. |
| 279 | | - """ |
| 280 | | - # FIXME convert all these calls so they just use runCommand now |
| 281 | | - proc = popen2.Popen4(command, 64) |
| 282 | | - #for line in proc.fromchild: |
| 283 | | - # callback(self, line) |
| 284 | | - line = proc.fromchild.readline() |
| 285 | | - while line: |
| 286 | | - callback(self, line) |
| 287 | | - line = proc.fromchild.readline() |
| 288 | | - return proc.wait() |
| 289 | | - |
| 290 | 254 | runAndReturn = staticmethod(runAndReturn) |
| 291 | | - runAndReport = staticmethod(runAndReport) |
| 292 | 255 | |
| 293 | 256 | class PageAndEditStats(object): |
| 294 | 257 | def __init__(self, wiki, dbName, errorCallback = None): |
| — | — | @@ -559,7 +522,7 @@ |
| 560 | 523 | def writeDumpRunInfoFile(self, text): |
| 561 | 524 | directory = self._getDumpRunInfoDirName() |
| 562 | 525 | dumpRunInfoFilename = self._getDumpRunInfoFileName() |
| 563 | | - WikiDump.dumpFile(directory, dumpRunInfoFilename, text) |
| | 526 | + FileUtils.writeFile(directory, dumpRunInfoFilename, text) |
| 564 | 527 | |
| 565 | 528 | def saveDumpRunInfoFile(self, done=False): |
| 566 | 529 | """Write out a simple text file with the status for this wiki's dump.""" |
| — | — | @@ -799,7 +762,7 @@ |
| 800 | 763 | message = self.config.readTemplate("errormail.txt") % { |
| 801 | 764 | "db": self.dbName, |
| 802 | 765 | "date": self.date, |
| 803 | | - "time": prettyTime(), |
| | 766 | + "time": TimeUtils.prettyTime(), |
| 804 | 767 | "url": "/".join((self.config.webRoot, self.dbName, self.date, ''))} |
| 805 | 768 | config.mail(subject, message) |
| 806 | 769 | |
| — | — | @@ -820,7 +783,7 @@ |
| 821 | 784 | raise(ValueException) |
| 822 | 785 | except: |
| 823 | 786 | return "No prior dumps of this database stored." |
| 824 | | - prettyDate = WikiDump.prettyDate(rawDate) |
| | 787 | + prettyDate = TimeUtils.prettyDate(rawDate) |
| 825 | 788 | if done: |
| 826 | 789 | prefix = "" |
| 827 | 790 | message = "Last dumped on" |
| — | — | @@ -870,10 +833,10 @@ |
| 871 | 834 | def reportFile(self, file, itemStatus): |
| 872 | 835 | filepath = self.dumpDir.publicPath(file) |
| 873 | 836 | if itemStatus == "in-progress" and exists (filepath): |
| 874 | | - size = prettySize(getsize(filepath)) |
| | 837 | + size = FileUtils.prettySize(getsize(filepath)) |
| 875 | 838 | return "<li class='file'>%s %s (written) </li>" % (file, size) |
| 876 | 839 | elif itemStatus == "done" and exists(filepath): |
| 877 | | - size = prettySize(getsize(filepath)) |
| | 840 | + size = FileUtils.prettySize(getsize(filepath)) |
| 878 | 841 | webpath = self.dumpDir.webPath(file) |
| 879 | 842 | return "<li class='file'><a href=\"%s\">%s</a> %s</li>" % (webpath, file, size) |
| 880 | 843 | else: |
| — | — | @@ -896,7 +859,7 @@ |
| 897 | 860 | # Override, continuing a past dump? |
| 898 | 861 | self.date = date |
| 899 | 862 | else: |
| 900 | | - self.date = WikiDump.today() |
| | 863 | + self.date = TimeUtils.today() |
| 901 | 864 | wiki.setDate(self.date) |
| 902 | 865 | |
| 903 | 866 | self.lastFailed = False |
| — | — | @@ -931,7 +894,7 @@ |
| 932 | 895 | done = log.doJobOnLogQueue() |
| 933 | 896 | |
| 934 | 897 | def logAndPrint(self, message): |
| 935 | | - if (self.log): |
| | 898 | + if hasattr(self,'log') and self.log: |
| 936 | 899 | self.log.addToLogQueue("%s\n" % message) |
| 937 | 900 | print message |
| 938 | 901 | |
| — | — | @@ -990,8 +953,8 @@ |
| 991 | 954 | return 1 |
| 992 | 955 | |
| 993 | 956 | def debug(self, stuff): |
| 994 | | - self.logAndPrint("%s: %s %s" % (prettyTime(), self.dbName, stuff)) |
| 995 | | -# print "%s: %s %s" % (prettyTime(), self.dbName, stuff) |
| | 957 | + self.logAndPrint("%s: %s %s" % (TimeUtils.prettyTime(), self.dbName, stuff)) |
| | 958 | +# print "%s: %s %s" % (MiscUtils.prettyTime(), self.dbName, stuff) |
| 996 | 959 | |
| 997 | 960 | def runHandleFailure(self): |
| 998 | 961 | if self.status.failCount < 1: |
| — | — | @@ -1226,7 +1189,7 @@ |
| 1227 | 1190 | else: |
| 1228 | 1191 | self.logAndPrint("What the hell dude, %s is not a symlink" % link) |
| 1229 | 1192 | raise BackupError("What the hell dude, %s is not a symlink" % link) |
| 1230 | | - relative = relativePath(real, dirname(link)) |
| | 1193 | + relative = FileUtils.relativePath(real, dirname(link)) |
| 1231 | 1194 | # if we removed the link cause it's obsolete, make the new one |
| 1232 | 1195 | if exists(real) and not exists(link): |
| 1233 | 1196 | self.debug("Adding symlink %s -> %s" % (link, relative)) |
| — | — | @@ -1247,7 +1210,7 @@ |
| 1248 | 1211 | "date": time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())} |
| 1249 | 1212 | directory = self.dumpDir.latestDir() |
| 1250 | 1213 | rssPath = self.dumpDir.latestPath(file + "-rss.xml") |
| 1251 | | - WikiDump.dumpFile(directory, rssPath, rssText) |
| | 1214 | + FileUtils.writeFile(directory, rssPath, rssText) |
| 1252 | 1215 | |
| 1253 | 1216 | class Dump(object): |
| 1254 | 1217 | def __init__(self, name, desc): |
| — | — | @@ -1279,7 +1242,7 @@ |
| 1280 | 1243 | def setStatus(self,status,setUpdated = True): |
| 1281 | 1244 | self.runInfo.setStatus(status) |
| 1282 | 1245 | if (setUpdated): |
| 1283 | | - self.runInfo.setUpdated(prettyTime()) |
| | 1246 | + self.runInfo.setUpdated(TimeUtils.prettyTime()) |
| 1284 | 1247 | |
| 1285 | 1248 | def setUpdated(self, updated): |
| 1286 | 1249 | self.runInfo.setUpdated(updated) |
| — | — | @@ -1345,18 +1308,18 @@ |
| 1346 | 1309 | # we assume the result is always going to be run in a subshell. |
| 1347 | 1310 | # much quicker than this script trying to read output |
| 1348 | 1311 | # and pass it to a subprocess |
| 1349 | | - outputFilenameEsc = shellEscape(outputFilename) |
| 1350 | | - headEsc = shellEscape(head) |
| 1351 | | - tailEsc = shellEscape(tail) |
| 1352 | | - grepEsc = shellEscape(grep) |
| | 1312 | + outputFilenameEsc = MiscUtils.shellEscape(outputFilename) |
| | 1313 | + headEsc = MiscUtils.shellEscape(head) |
| | 1314 | + tailEsc = MiscUtils.shellEscape(tail) |
| | 1315 | + grepEsc = MiscUtils.shellEscape(grep) |
| 1353 | 1316 | |
| 1354 | 1317 | uncompressionCommandEsc = uncompressionCommand[:] |
| 1355 | 1318 | for u in uncompressionCommandEsc: |
| 1356 | | - u = shellEscape(u) |
| | 1319 | + u = MiscUtils.shellEscape(u) |
| 1357 | 1320 | for u in compressionCommand: |
| 1358 | | - u = shellEscape(u) |
| | 1321 | + u = MiscUtils.shellEscape(u) |
| 1359 | 1322 | for f in files: |
| 1360 | | - f = shellEscape(f) |
| | 1323 | + f = MiscUtils.shellEscape(f) |
| 1361 | 1324 | |
| 1362 | 1325 | for f in files: |
| 1363 | 1326 | f = runner.dumpDir.publicPath(f) |
| — | — | @@ -1690,7 +1653,7 @@ |
| 1691 | 1654 | pipeline.append(uncompressionCommand) |
| 1692 | 1655 | # warning: we figure any header (<siteinfo>...</siteinfo>) is going to be less than 2000 lines! |
| 1693 | 1656 | head = runner.config.head |
| 1694 | | - headEsc = shellEscape(head) |
| | 1657 | + headEsc = MiscUtils.shellEscape(head) |
| 1695 | 1658 | pipeline.append([ head, "-2000"]) |
| 1696 | 1659 | # without shell |
| 1697 | 1660 | p = CommandPipeline(pipeline, quiet=True) |