| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -169,32 +169,108 @@ |
| 170 | 170 | return 0 |
| 171 | 171 | return chunks |
| 172 | 172 | |
| | 173 | +class DbServerInfo(object): |
| | 174 | + def __init__(self, wiki, dbName): |
| | 175 | + self.config = wiki.config |
| | 176 | + self.dbName = dbName |
| | 177 | + self.selectDatabaseServer() |
| | 178 | + |
| | 179 | + def defaultServer(self): |
| | 180 | + # if this fails what do we do about it? Not a bleeping thing. *ugh* FIXME!! |
| | 181 | + command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % shellEscape(( |
| | 182 | + self.config.php, self.config.wikiDir, self.dbName)) |
| | 183 | + return RunSimpleCommand.runAndReturn(command).strip() |
| | 184 | + |
| | 185 | + def selectDatabaseServer(self): |
| | 186 | + self.dbServer = self.defaultServer() |
| | 187 | + |
| | 188 | + def runSqlAndGetOutput(self, query): |
| | 189 | + """Pass some SQL commands to the server for this DB and save output to a file.""" |
| | 190 | + command = [ [ "/bin/echo", "%s" % query ], |
| | 191 | + [ "%s" % self.config.mysql, "-h", |
| | 192 | + "%s" % self.dbServer, |
| | 193 | + "-u", "%s" % self.config.dbUser, |
| | 194 | + "%s" % self.passwordOption(), |
| | 195 | + "%s" % self.dbName, |
| | 196 | + "-r" ] ] |
| | 197 | + p = CommandPipeline(command, quiet=True) |
| | 198 | + p.runPipelineAndGetOutput() |
| | 199 | + # fixme best to put the return code someplace along with any errors.... |
| | 200 | + if (p.output()): |
| | 201 | + return(p.output()) |
| | 202 | + else: |
| | 203 | + return None |
| | 204 | + |
| | 205 | + def passwordOption(self): |
| | 206 | + """If you pass '-pfoo' mysql uses the password 'foo', |
| | 207 | + but if you pass '-p' it prompts. Sigh.""" |
| | 208 | + if self.config.dbPassword == "": |
| | 209 | + return None |
| | 210 | + else: |
| | 211 | + return "-p" + self.config.dbPassword |
| | 212 | + |
| | 213 | + |
| | 214 | +class RunSimpleCommand(object): |
| | 215 | + |
| | 216 | + def log(self, message, log = None): |
| | 217 | + if (log): |
| | 218 | + log.addToLogQueue("%s\n" % message) |
| | 219 | + |
| | 220 | + # FIXME rewrite to not use popen2 |
| | 221 | + def runAndReturn(command, log = None): |
| | 222 | + """Run a command and return the output as a string. |
| | 223 | + Raises BackupError on non-zero return code.""" |
| | 224 | + # FIXME convert all these calls so they just use runCommand now |
| | 225 | + proc = popen2.Popen4(command, 64) |
| | 226 | + output = proc.fromchild.read() |
| | 227 | + retval = proc.wait() |
| | 228 | + if retval: |
| | 229 | + self.log("Non-zero return code from '%s'" % command, log) |
| | 230 | + raise BackupError("Non-zero return code from '%s'" % command) |
| | 231 | + else: |
| | 232 | + return output |
| | 233 | + |
| | 234 | + def runAndReport(self, command, callback): |
| | 235 | + """Shell out to a command, and feed output lines to the callback function. |
| | 236 | + Returns the exit code from the program once complete. |
| | 237 | + stdout and stderr will be combined into a single stream. |
| | 238 | + """ |
| | 239 | + # FIXME convert all these calls so they just use runCommand now |
| | 240 | + proc = popen2.Popen4(command, 64) |
| | 241 | + #for line in proc.fromchild: |
| | 242 | + # callback(self, line) |
| | 243 | + line = proc.fromchild.readline() |
| | 244 | + while line: |
| | 245 | + callback(self, line) |
| | 246 | + line = proc.fromchild.readline() |
| | 247 | + return proc.wait() |
| | 248 | + |
| | 249 | + runAndReturn = staticmethod(runAndReturn) |
| | 250 | + runAndReport = staticmethod(runAndReport) |
| | 251 | + |
| 173 | 252 | class PageAndEditStats(object): |
| 174 | 253 | def __init__(self, wiki, dbName): |
| 175 | 254 | self.totalPages = None |
| 176 | 255 | self.totalEdits = None |
| 177 | 256 | self.config = wiki.config |
| 178 | 257 | self.dbName = dbName |
| | 258 | + self.dbServerInfo = DbServerInfo(wiki, dbName) |
| 179 | 259 | (self.totalPages, totalEdits) = self.getStatistics(config,dbName) |
| 180 | 260 | |
| 181 | | - def getStatistics(self, config,dbName): |
| | 261 | + def getStatistics(self, dbName, ignore): |
| 182 | 262 | """Get (cached) statistics for the wiki""" |
| 183 | 263 | totalPages = None |
| 184 | 264 | totalEdits = None |
| 185 | | - statsCommand = """%s -q %s/maintenance/showStats.php --wiki=%s """ % shellEscape(( |
| 186 | | - self.config.php, self.config.wikiDir, self.dbName)) |
| 187 | | - # FIXME runAndReturn? defined somewhere else |
| 188 | | - results = self.runAndReturn(statsCommand) |
| | 265 | + query = "select MAX(page_id) from page;" |
| | 266 | + results = self.dbServerInfo.runSqlAndGetOutput(query) |
| 189 | 267 | lines = results.splitlines() |
| 190 | | - if (lines): |
| 191 | | - for line in lines: |
| 192 | | - (name,value) = line.split(':') |
| 193 | | - name = name.replace(' ','') |
| 194 | | - value = value.replace(' ','') |
| 195 | | - if (name == "Totalpages"): |
| 196 | | - totalPages = int(value) |
| 197 | | - elif (name == "Totaledits"): |
| 198 | | - totalEdits = int(value) |
| | 268 | + if (lines and lines[1]): |
| | 269 | + totalPages = int(lines[1]) |
| | 270 | + query = "select MAX(rev_id) from revision;" |
| | 271 | + results = self.dbServerInfo.runSqlAndGetOutput(query) |
| | 272 | + lines = results.splitlines() |
| | 273 | + if (lines and lines[1]): |
| | 274 | + totalEdits = int(lines[1]) |
| 199 | 275 | return(totalPages, totalEdits) |
| 200 | 276 | |
| 201 | 277 | def getTotalPages(self): |
| — | — | @@ -203,19 +279,6 @@ |
| 204 | 280 | def getTotalEdits(self): |
| 205 | 281 | return self.totalEdits |
| 206 | 282 | |
| 207 | | - # FIXME should rewrite this I guess and also move it elsewhere, phooey |
| 208 | | - def runAndReturn(self, command): |
| 209 | | - """Run a command and return the output as a string. |
| 210 | | - Raises BackupError on non-zero return code.""" |
| 211 | | - # FIXME convert all these calls so they just use runCommand now |
| 212 | | - proc = popen2.Popen4(command, 64) |
| 213 | | - output = proc.fromchild.read() |
| 214 | | - retval = proc.wait() |
| 215 | | - if retval: |
| 216 | | - raise BackupError("Non-zero return code from '%s'" % command) |
| 217 | | - else: |
| 218 | | - return output |
| 219 | | - |
| 220 | 283 | class BackupError(Exception): |
| 221 | 284 | pass |
| 222 | 285 | |
| — | — | @@ -618,6 +681,8 @@ |
| 619 | 682 | self.checkpoint = checkpoint |
| 620 | 683 | |
| 621 | 684 | self.jobRequested = job |
| | 685 | + self.dbServerInfo = DbServerInfo(self.wiki, self.dbName) |
| | 686 | + |
| 622 | 687 | self.dumpDir = DumpDir(self.wiki, self.dbName, self.date) |
| 623 | 688 | |
| 624 | 689 | # this must come after the dumpdir setup so we know which directory we are in |
| — | — | @@ -645,14 +710,6 @@ |
| 646 | 711 | self.log.addToLogQueue("%s\n" % message) |
| 647 | 712 | print message |
| 648 | 713 | |
| 649 | | - def passwordOption(self): |
| 650 | | - """If you pass '-pfoo' mysql uses the password 'foo', |
| 651 | | - but if you pass '-p' it prompts. Sigh.""" |
| 652 | | - if self.config.dbPassword == "": |
| 653 | | - return None |
| 654 | | - else: |
| 655 | | - return "-p" + self.config.dbPassword |
| 656 | | - |
| 657 | 714 | def forceNormalOption(self): |
| 658 | 715 | if self.config.forceNormal: |
| 659 | 716 | return "--force-normal" |
| — | — | @@ -664,14 +721,14 @@ |
| 665 | 722 | # FIXME later full path |
| 666 | 723 | command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % shellEscape(( |
| 667 | 724 | self.config.php, self.config.wikiDir, self.dbName)) |
| 668 | | - return self.runAndReturn(command).strip() |
| | 725 | + return RunSimpleCommand.runAndReturn(command, self.log).strip() |
| 669 | 726 | |
| 670 | 727 | def saveTable(self, table, outfile): |
| 671 | 728 | """Dump a table from the current DB with mysqldump, save to a gzipped sql file.""" |
| 672 | 729 | commands = [ [ "%s" % self.config.mysqldump, "-h", |
| 673 | | - "%s" % self.dbServer, "-u", |
| | 730 | + "%s" % self.dbServerInfo.dbServer, "-u", |
| 674 | 731 | "%s" % self.config.dbUser, |
| 675 | | - "%s" % self.passwordOption(), "--opt", "--quick", |
| | 732 | + "%s" % self.dbServerInfo.passwordOption(), "--opt", "--quick", |
| 676 | 733 | "--skip-add-locks", "--skip-lock-tables", |
| 677 | 734 | "%s" % self.dbName, |
| 678 | 735 | "%s" % self.getDBTablePrefix() + table ], |
| — | — | @@ -683,9 +740,9 @@ |
| 684 | 741 | """Pass some SQL commands to the server for this DB and save output to a file.""" |
| 685 | 742 | command = [ [ "/bin/echo", "%s" % query ], |
| 686 | 743 | [ "%s" % self.config.mysql, "-h", |
| 687 | | - "%s" % self.dbServer, |
| | 744 | + "%s" % self.dbServerInfo.dbServer, |
| 688 | 745 | "-u", "%s" % self.config.dbUser, |
| 689 | | - "%s" % self.passwordOption(), |
| | 746 | + "%s" % self.dbServerInfo.passwordOption(), |
| 690 | 747 | "%s" % self.dbName, |
| 691 | 748 | "-r" ], |
| 692 | 749 | [ "%s" % self.config.gzip ] ] |
| — | — | @@ -697,25 +754,6 @@ |
| 698 | 755 | series = [ commands ] |
| 699 | 756 | return self.runCommand([ series ]) |
| 700 | 757 | |
| 701 | | - def getStatistics(self, dbName): |
| 702 | | - """Get (cached) statistics for the wiki""" |
| 703 | | - totalPages = None |
| 704 | | - totalEdits = None |
| 705 | | - statsCommand = """%s -q %s/maintenance/showStats.php --wiki=%s """ % shellEscape(( |
| 706 | | - self.config.php, self.config.wikiDir, self.dbName)) |
| 707 | | - results = self.runAndReturn(statsCommand) |
| 708 | | - lines = results.splitlines() |
| 709 | | - if (lines): |
| 710 | | - for line in lines: |
| 711 | | - (name,value) = line.split(':') |
| 712 | | - name = name.replace(' ','') |
| 713 | | - value = value.replace(' ','') |
| 714 | | - if (name == "Totalpages"): |
| 715 | | - totalPages = int(value) |
| 716 | | - elif (name == "Totaledits"): |
| 717 | | - totalEdits = int(value) |
| 718 | | - return(totalPages, totalEdits) |
| 719 | | - |
| 720 | 758 | # command series list: list of (commands plus args) is one pipeline. list of pipelines = 1 series. |
| 721 | 759 | # this function wants a list of series. |
| 722 | 760 | # be a list (the command name and the various args) |
| — | — | @@ -735,7 +773,6 @@ |
| 736 | 774 | if commands.exitedSuccessfully(): |
| 737 | 775 | return 0 |
| 738 | 776 | else: |
| 739 | | - #print "***** BINGBING retval is '%s' ********" % retval |
| 740 | 777 | problemCommands = commands.commandsWithErrors() |
| 741 | 778 | errorString = "Error from command(s): " |
| 742 | 779 | for cmd in problemCommands: |
| — | — | @@ -744,34 +781,6 @@ |
| 745 | 782 | raise BackupError(errorString) |
| 746 | 783 | return 1 |
| 747 | 784 | |
| 748 | | - def runAndReport(self, command, callback): |
| 749 | | - """Shell out to a command, and feed output lines to the callback function. |
| 750 | | - Returns the exit code from the program once complete. |
| 751 | | - stdout and stderr will be combined into a single stream. |
| 752 | | - """ |
| 753 | | - # FIXME convert all these calls so they just use runCommand now |
| 754 | | - proc = popen2.Popen4(command, 64) |
| 755 | | - #for line in proc.fromchild: |
| 756 | | - # callback(self, line) |
| 757 | | - line = proc.fromchild.readline() |
| 758 | | - while line: |
| 759 | | - callback(self, line) |
| 760 | | - line = proc.fromchild.readline() |
| 761 | | - return proc.wait() |
| 762 | | - |
| 763 | | - def runAndReturn(self, command): |
| 764 | | - """Run a command and return the output as a string. |
| 765 | | - Raises BackupError on non-zero return code.""" |
| 766 | | - # FIXME convert all these calls so they just use runCommand now |
| 767 | | - proc = popen2.Popen4(command, 64) |
| 768 | | - output = proc.fromchild.read() |
| 769 | | - retval = proc.wait() |
| 770 | | - if retval: |
| 771 | | - self.logAndPrint("Non-zero return code from '%s'" % command) |
| 772 | | - raise BackupError("Non-zero return code from '%s'" % command) |
| 773 | | - else: |
| 774 | | - return output |
| 775 | | - |
| 776 | 785 | def debug(self, stuff): |
| 777 | 786 | self.logAndPrint("%s: %s %s" % (prettyTime(), self.dbName, stuff)) |
| 778 | 787 | # print "%s: %s %s" % (prettyTime(), self.dbName, stuff) |
| — | — | @@ -783,14 +792,6 @@ |
| 784 | 793 | self.debug("Creating %s ..." % dir) |
| 785 | 794 | os.makedirs(dir) |
| 786 | 795 | |
| 787 | | - def selectDatabaseServer(self): |
| 788 | | - self.dbServer = self.defaultServer() |
| 789 | | - |
| 790 | | - def defaultServer(self): |
| 791 | | - command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % shellEscape(( |
| 792 | | - self.config.php, self.config.wikiDir, self.dbName)) |
| 793 | | - return self.runAndReturn(command).strip() |
| 794 | | - |
| 795 | 796 | def runHandleFailure(self): |
| 796 | 797 | if self.failCount < 1: |
| 797 | 798 | # Email the site administrator just once per database |
| — | — | @@ -850,7 +851,7 @@ |
| 851 | 852 | self.cleanOldDumps() |
| 852 | 853 | self.showRunnerState("Starting backup of %s" % self.dbName) |
| 853 | 854 | |
| 854 | | - self.selectDatabaseServer() |
| | 855 | + self.DbServerInfo = DbServerInfo(self.wiki,self.dbName) |
| 855 | 856 | |
| 856 | 857 | files = self.listFilesFor(self.dumpItemList.dumpItems) |
| 857 | 858 | |