| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -92,7 +92,7 @@ |
| 93 | 93 | # otherwise we get passed alist that says "here's now many for each chunk and it's this many chunks. |
| 94 | 94 | # extra pages/revs go in the last chunk, stuck on the end. too bad. :-P |
| 95 | 95 | class Chunk(object, ): |
| 96 | | - def __init__(self, wiki, dbName): |
| | 96 | + def __init__(self, wiki, dbName, errorCallback = None): |
| 97 | 97 | |
| 98 | 98 | self._dbName = dbName |
| 99 | 99 | self._chunksEnabled = wiki.config.chunksEnabled |
| — | — | @@ -102,7 +102,7 @@ |
| 103 | 103 | self._recombineHistory = wiki.config.recombineHistory |
| 104 | 104 | |
| 105 | 105 | if (self._chunksEnabled): |
| 106 | | - self.Stats = PageAndEditStats(wiki,dbName) |
| | 106 | + self.Stats = PageAndEditStats(wiki,dbName, errorCallback) |
| 107 | 107 | if (not self.Stats.totalEdits or not self.Stats.totalPages): |
| 108 | 108 | raise BackupError("Failed to get DB stats, exiting") |
| 109 | 109 | if (self._revsPerChunkHistory): |
| — | — | @@ -175,22 +175,23 @@ |
| 176 | 176 | return chunks |
| 177 | 177 | |
| 178 | 178 | class DbServerInfo(object): |
| 179 | | - def __init__(self, wiki, dbName): |
| | 179 | + def __init__(self, wiki, dbName, errorCallback = None): |
| 180 | 180 | self.config = wiki.config |
| 181 | 181 | self.dbName = dbName |
| | 182 | + self.errorCallback = errorCallback |
| 182 | 183 | self.selectDatabaseServer() |
| 183 | 184 | |
| 184 | 185 | def defaultServer(self): |
| 185 | 186 | # if this fails what do we do about it? Not a bleeping thing. *ugh* FIXME!! |
| 186 | 187 | command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % shellEscape(( |
| 187 | 188 | self.config.php, self.config.wikiDir, self.dbName)) |
| 188 | | - return RunSimpleCommand.runAndReturn(command).strip() |
| | 189 | + return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| 189 | 190 | |
| 190 | 191 | def selectDatabaseServer(self): |
| 191 | 192 | self.dbServer = self.defaultServer() |
| 192 | 193 | |
| 193 | | - def runSqlAndGetOutput(self, query): |
| 194 | | - """Pass some SQL commands to the server for this DB and save output to a file.""" |
| | 194 | + def buildSqlCommand(self, query, pipeto = None): |
| | 195 | + """Put together a command to execute an sql query to the server for this DB.""" |
| 195 | 196 | command = [ [ "/bin/echo", "%s" % query ], |
| 196 | 197 | [ "%s" % self.config.mysql, "-h", |
| 197 | 198 | "%s" % self.dbServer, |
| — | — | @@ -198,6 +199,26 @@ |
| 199 | 200 | "%s" % self.passwordOption(), |
| 200 | 201 | "%s" % self.dbName, |
| 201 | 202 | "-r" ] ] |
| | 203 | + if (pipeto): |
| | 204 | + command.append([ pipeto ]) |
| | 205 | + return command |
| | 206 | + |
| | 207 | + def buildSqlDumpCommand(self, table, pipeto = None): |
| | 208 | + """Put together a command to dump a table from the current DB with mysqldump |
| | 209 | + and save to a gzipped sql file.""" |
| | 210 | + command = [ [ "%s" % self.config.mysqldump, "-h", |
| | 211 | + "%s" % self.dbServer, "-u", |
| | 212 | + "%s" % self.config.dbUser, |
| | 213 | + "%s" % self.passwordOption(), "--opt", "--quick", |
| | 214 | + "--skip-add-locks", "--skip-lock-tables", |
| | 215 | + "%s" % self.dbName, |
| | 216 | + "%s" % self.getDBTablePrefix() + table ] ] |
| | 217 | + if (pipeto): |
| | 218 | + command.append([ pipeto ]) |
| | 219 | + return command |
| | 220 | + |
| | 221 | + def runSqlAndGetOutput(self, query): |
| | 222 | + command = self.buildSqlCommand(query) |
| 202 | 223 | p = CommandPipeline(command, quiet=True) |
| 203 | 224 | p.runPipelineAndGetOutput() |
| 204 | 225 | # fixme best to put the return code someplace along with any errors.... |
| — | — | @@ -214,15 +235,18 @@ |
| 215 | 236 | else: |
| 216 | 237 | return "-p" + self.config.dbPassword |
| 217 | 238 | |
| | 239 | + def getDBTablePrefix(self): |
| | 240 | + """Get the prefix for all tables for the specific wiki ($wgDBprefix)""" |
| | 241 | + # FIXME later full path |
| | 242 | + command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % shellEscape(( |
| | 243 | + self.config.php, self.config.wikiDir, self.dbName)) |
| | 244 | + return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| | 245 | + |
| 218 | 246 | |
| 219 | 247 | class RunSimpleCommand(object): |
| 220 | 248 | |
| 221 | | - def log(message, logInfo = None): |
| 222 | | - if (logInfo): |
| 223 | | - logInfo.addToLogQueue("%s\n" % message) |
| 224 | | - |
| 225 | 249 | # FIXME rewrite to not use popen2 |
| 226 | | - def runAndReturn(command, logInfo = None): |
| | 250 | + def runAndReturn(command, logCallback = None): |
| 227 | 251 | """Run a command and return the output as a string. |
| 228 | 252 | Raises BackupError on non-zero return code.""" |
| 229 | 253 | # FIXME convert all these calls so they just use runCommand now |
| — | — | @@ -233,14 +257,16 @@ |
| 234 | 258 | output = proc.fromchild.read() |
| 235 | 259 | retval = proc.wait() |
| 236 | 260 | while (retval and retries < maxretries): |
| 237 | | - RunSimpleCommand.log("Non-zero return code from '%s'" % command, logInfo) |
| | 261 | + if self.logCallback: |
| | 262 | + self.logCallback("Non-zero return code from '%s'" % command) |
| 238 | 263 | time.sleep(5) |
| 239 | 264 | proc = popen2.Popen4(command, 64) |
| 240 | 265 | output = proc.fromchild.read() |
| 241 | 266 | retval = proc.wait() |
| 242 | 267 | retries = retries + 1 |
| 243 | 268 | if retval: |
| 244 | | -# RunSimpleCommand.log("Non-zero return code from '%s'" % command, logInfo) |
| | 269 | + if self.logCallback: |
| | 270 | + self.logCallback("Non-zero return code from '%s'" % command) |
| 245 | 271 | raise BackupError("Non-zero return code from '%s'" % command) |
| 246 | 272 | else: |
| 247 | 273 | return output |
| — | — | @@ -262,15 +288,14 @@ |
| 263 | 289 | |
| 264 | 290 | runAndReturn = staticmethod(runAndReturn) |
| 265 | 291 | runAndReport = staticmethod(runAndReport) |
| 266 | | - log = staticmethod(log) |
| 267 | 292 | |
| 268 | 293 | class PageAndEditStats(object): |
| 269 | | - def __init__(self, wiki, dbName): |
| | 294 | + def __init__(self, wiki, dbName, errorCallback = None): |
| 270 | 295 | self.totalPages = None |
| 271 | 296 | self.totalEdits = None |
| 272 | 297 | self.config = wiki.config |
| 273 | 298 | self.dbName = dbName |
| 274 | | - self.dbServerInfo = DbServerInfo(wiki, dbName) |
| | 299 | + self.dbServerInfo = DbServerInfo(wiki, dbName, errorCallback) |
| 275 | 300 | self.getStatistics(config,dbName) |
| 276 | 301 | |
| 277 | 302 | def getStatistics(self, dbName, ignore): |
| — | — | @@ -862,7 +887,7 @@ |
| 863 | 888 | self.dbName = wiki.dbName |
| 864 | 889 | self.prefetch = prefetch |
| 865 | 890 | self.spawn = spawn |
| 866 | | - self.chunkInfo = Chunk(wiki, self.dbName) |
| | 891 | + self.chunkInfo = Chunk(wiki, self.dbName, self.logAndPrint) |
| 867 | 892 | self.restart = restart |
| 868 | 893 | self.loggingEnabled = loggingEnabled |
| 869 | 894 | self.log = None |
| — | — | @@ -879,7 +904,7 @@ |
| 880 | 905 | self.checkpoint = checkpoint |
| 881 | 906 | |
| 882 | 907 | self.jobRequested = job |
| 883 | | - self.dbServerInfo = DbServerInfo(self.wiki, self.dbName) |
| | 908 | + self.dbServerInfo = DbServerInfo(self.wiki, self.dbName, self.logAndPrint) |
| 884 | 909 | |
| 885 | 910 | self.dumpDir = DumpDir(self.wiki, self.dbName, self.date) |
| 886 | 911 | |
| — | — | @@ -916,37 +941,15 @@ |
| 917 | 942 | else: |
| 918 | 943 | return "" |
| 919 | 944 | |
| 920 | | - def getDBTablePrefix(self): |
| 921 | | - """Get the prefix for all tables for the specific wiki ($wgDBprefix)""" |
| 922 | | - # FIXME later full path |
| 923 | | - command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % shellEscape(( |
| 924 | | - self.config.php, self.config.wikiDir, self.dbName)) |
| 925 | | - return RunSimpleCommand.runAndReturn(command, self.log).strip() |
| 926 | | - |
| 927 | 945 | # returns 0 on success, 1 on error |
| 928 | 946 | def saveTable(self, table, outfile): |
| 929 | 947 | """Dump a table from the current DB with mysqldump, save to a gzipped sql file.""" |
| 930 | | - commands = [ [ "%s" % self.config.mysqldump, "-h", |
| 931 | | - "%s" % self.dbServerInfo.dbServer, "-u", |
| 932 | | - "%s" % self.config.dbUser, |
| 933 | | - "%s" % self.dbServerInfo.passwordOption(), "--opt", "--quick", |
| 934 | | - "--skip-add-locks", "--skip-lock-tables", |
| 935 | | - "%s" % self.dbName, |
| 936 | | - "%s" % self.getDBTablePrefix() + table ], |
| 937 | | - [ "%s" % self.config.gzip ] ] |
| 938 | | - |
| | 948 | + commands = self.dbServerInfo.buildSqlDumpCommand(table, self.config.gzip) |
| 939 | 949 | return self.saveCommand(commands, outfile) |
| 940 | 950 | |
| 941 | 951 | def saveSql(self, query, outfile): |
| 942 | | - """Pass some SQL commands to the server for this DB and save output to a file.""" |
| 943 | | - command = [ [ "/bin/echo", "%s" % query ], |
| 944 | | - [ "%s" % self.config.mysql, "-h", |
| 945 | | - "%s" % self.dbServerInfo.dbServer, |
| 946 | | - "-u", "%s" % self.config.dbUser, |
| 947 | | - "%s" % self.dbServerInfo.passwordOption(), |
| 948 | | - "%s" % self.dbName, |
| 949 | | - "-r" ], |
| 950 | | - [ "%s" % self.config.gzip ] ] |
| | 952 | + """Pass some SQL commands to the server for this DB and save output to a gzipped file.""" |
| | 953 | + command = self.dbServerInfo.buildSqlCommand(query, self.config.gzip) |
| 951 | 954 | return self.saveCommand(command, outfile) |
| 952 | 955 | |
| 953 | 956 | # returns 0 on success, 1 on error |
| — | — | @@ -993,7 +996,7 @@ |
| 994 | 997 | def runHandleFailure(self): |
| 995 | 998 | if self.status.failCount < 1: |
| 996 | 999 | # Email the site administrator just once per database |
| 997 | | - self.reportFailure() |
| | 1000 | + self.status.reportFailure() |
| 998 | 1001 | self.status.failCount += 1 |
| 999 | 1002 | self.lastFailed = True |
| 1000 | 1003 | |
| — | — | @@ -1049,8 +1052,6 @@ |
| 1050 | 1053 | self.cleanOldDumps() |
| 1051 | 1054 | self.showRunnerState("Starting backup of %s" % self.dbName) |
| 1052 | 1055 | |
| 1053 | | - self.DbServerInfo = DbServerInfo(self.wiki,self.dbName) |
| 1054 | | - |
| 1055 | 1056 | files = self.listFilesFor(self.dumpItemList.dumpItems) |
| 1056 | 1057 | |
| 1057 | 1058 | if (self.jobRequested): |
| — | — | @@ -2182,11 +2183,11 @@ |
| 2183 | 2184 | # try this initially and see how it goes |
| 2184 | 2185 | maxretries = 3 |
| 2185 | 2186 | query="select page_title from page where page_namespace=0;" |
| 2186 | | - error = runner.saveSql(query, runner.dumpDir.publicPath("all-titles-in-ns0.gz")) |
| | 2187 | + error = runner.dbServerInfo.saveSql(query, runner.dumpDir.publicPath("all-titles-in-ns0.gz")) |
| 2187 | 2188 | while (error and retries < maxretries): |
| 2188 | 2189 | retries = retries + 1 |
| 2189 | 2190 | time.sleep(5) |
| 2190 | | - error = runner.saveSql(query, runner.dumpDir.publicPath("all-titles-in-ns0.gz")) |
| | 2191 | + error = runner.dbServerInfo.saveSql(query, runner.dumpDir.publicPath("all-titles-in-ns0.gz")) |
| 2191 | 2192 | return error |
| 2192 | 2193 | |
| 2193 | 2194 | def listFiles(self, runner): |