| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -154,6 +154,20 @@ |
| 155 | 155 | return 0 |
| 156 | 156 | return chunks |
| 157 | 157 | |
| | 158 | +class MultiVersion(object): |
| | 159 | + def MWScriptAsString(config, maintenanceScript): |
| | 160 | + return(" ".join(MultiVersion.MWScriptAsArray(config, maintenanceScript))) |
| | 161 | + |
| | 162 | + def MWScriptAsArray(config, maintenanceScript): |
| | 163 | + MWScriptLocation = os.path.join(config.wikiDir,"..","multiversion","MWScript.php") |
| | 164 | + if exists(MWScriptLocation): |
| | 165 | + return [ MWScriptLocation, maintenanceScript ] |
| | 166 | + else: |
| | 167 | + return [ "%s/maintenance/%s" % (config.wikiDir, maintenanceScript) ] |
| | 168 | + |
| | 169 | + MWScriptAsString = staticmethod(MWScriptAsString) |
| | 170 | + MWScriptAsArray = staticmethod(MWScriptAsArray) |
| | 171 | + |
| 158 | 172 | class DbServerInfo(object): |
| 159 | 173 | def __init__(self, wiki, dbName, errorCallback = None): |
| 160 | 174 | self.wiki = wiki |
| — | — | @@ -165,8 +179,14 @@ |
| 166 | 180 | # if this fails what do we do about it? Not a bleeping thing. *ugh* FIXME!! |
| 167 | 181 | if (not exists( self.wiki.config.php ) ): |
| 168 | 182 | raise BackupError("php command %s not found" % self.wiki.config.php) |
| 169 | | - command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % MiscUtils.shellEscape(( |
| 170 | | - self.wiki.config.php, self.wiki.config.wikiDir, self.dbName)) |
| | 183 | + commandList = MultiVersion.MWScriptAsArray(self.wiki.config, "getSlaveServer.php") |
| | 184 | +# command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % MiscUtils.shellEscape(( |
| | 185 | + for i in range(0,len(commandList)): |
| | 186 | + phpCommand = MiscUtils.shellEscape(self.wiki.config.php) |
| | 187 | + dbName = MiscUtils.shellEscape(self.dbName) |
| | 188 | + commandList[i] = MiscUtils.shellEscape(commandList[i]) |
| | 189 | + command = " ".join(commandList) |
| | 190 | + command = "%s -q %s --wiki=%s --group=dump" % (phpCommand, command, dbName) |
| 171 | 191 | return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| 172 | 192 | |
| 173 | 193 | def selectDatabaseServer(self): |
| — | — | @@ -226,8 +246,14 @@ |
| 227 | 247 | # FIXME later full path |
| 228 | 248 | if (not exists( self.wiki.config.php ) ): |
| 229 | 249 | raise BackupError("php command %s not found" % self.wiki.config.php) |
| 230 | | - command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % MiscUtils.shellEscape(( |
| 231 | | - self.wiki.config.php, self.wiki.config.wikiDir, self.dbName)) |
| | 250 | + commandList = MultiVersion.MWScriptAsArray(self.wiki.config, "eval.php") |
| | 251 | +# command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % MiscUtils.shellEscape(( |
| | 252 | + for i in range(0,len(commandList)): |
| | 253 | + phpCommand = MiscUtils.shellEscape(self.wiki.config.php) |
| | 254 | + dbName = MiscUtils.shellEscape(self.dbName) |
| | 255 | + commandList[i] = MiscUtils.shellEscape(commandList[i]) |
| | 256 | + command = " ".join(commandList) |
| | 257 | + command = "echo 'print $wgDBprefix; ' | %s -q %s --wiki=%s" % (phpCommand, command, dbName) |
| 232 | 258 | return RunSimpleCommand.runAndReturn(command, self.errorCallback).strip() |
| 233 | 259 | |
| 234 | 260 | |
| — | — | @@ -2637,15 +2663,17 @@ |
| 2638 | 2664 | articlesFile = runner.dumpDir.filenamePublicPath(f) |
| 2639 | 2665 | historyFile = runner.dumpDir.filenamePublicPath(DumpFilename(runner.wiki, f.date, self.historyDumpName, f.fileType, f.fileExt, f.chunk, f.checkpoint, f.temp)) |
| 2640 | 2666 | currentFile = runner.dumpDir.filenamePublicPath(DumpFilename(runner.wiki, f.date, self.currentDumpName, f.fileType, f.fileExt, f.chunk, f.checkpoint, f.temp)) |
| 2641 | | - command = [ "%s" % runner.wiki.config.php, |
| 2642 | | - "-q", "%s/maintenance/dumpBackup.php" % runner.wiki.config.wikiDir, |
| 2643 | | - "--wiki=%s" % runner.dbName, |
| 2644 | | - "--full", "--stub", "--report=10000", |
| 2645 | | - "%s" % runner.forceNormalOption(), |
| 2646 | | - "--output=gzip:%s" % historyFile, |
| 2647 | | - "--output=gzip:%s" % currentFile, |
| 2648 | | - "--filter=latest", "--output=gzip:%s" % articlesFile, |
| 2649 | | - "--filter=latest", "--filter=notalk", "--filter=namespace:!NS_USER" ] |
| | 2667 | + scriptCommand = MultiVersion.MWScriptAsArray(runner.wiki.config, "dumpBackup.php") |
| | 2668 | + command = [ "%s" % runner.wiki.config.php, "-q" ] |
| | 2669 | + command.extend(scriptCommand) |
| | 2670 | + command.extend(["--wiki=%s" % runner.dbName, |
| | 2671 | + "--full", "--stub", "--report=10000", |
| | 2672 | + "%s" % runner.forceNormalOption(), |
| | 2673 | + "--output=gzip:%s" % historyFile, |
| | 2674 | + "--output=gzip:%s" % currentFile, |
| | 2675 | + "--filter=latest", "--output=gzip:%s" % articlesFile, |
| | 2676 | + "--filter=latest", "--filter=notalk", "--filter=namespace:!NS_USER" |
| | 2677 | + ]) |
| 2650 | 2678 | |
| 2651 | 2679 | if (f.chunk): |
| 2652 | 2680 | # set up start end end pageids for this piece |
| — | — | @@ -2760,12 +2788,13 @@ |
| 2761 | 2789 | logging = runner.dumpDir.filenamePublicPath(files[0]) |
| 2762 | 2790 | if (not exists( runner.wiki.config.php ) ): |
| 2763 | 2791 | raise BackupError("php command %s not found" % runner.wiki.config.php) |
| 2764 | | - command = [ "%s" % runner.wiki.config.php, |
| 2765 | | - "-q", "%s/maintenance/dumpBackup.php" % runner.wiki.config.wikiDir, |
| 2766 | | - "--wiki=%s" % runner.dbName, |
| | 2792 | + scriptCommand = MultiVersion.MWScriptAsArray(runner.wiki.config, "dumpBackup.php") |
| | 2793 | + command = [ "%s" % runner.wiki.config.php, "-q" ] |
| | 2794 | + command.extend(scriptCommand) |
| | 2795 | + command.extend(["--wiki=%s" % runner.dbName, |
| 2767 | 2796 | "--logs", "--report=10000", |
| 2768 | 2797 | "%s" % runner.forceNormalOption(), |
| 2769 | | - "--output=gzip:%s" % logging ] |
| | 2798 | + "--output=gzip:%s" % logging ]) |
| 2770 | 2799 | pipeline = [ command ] |
| 2771 | 2800 | series = [ pipeline ] |
| 2772 | 2801 | error = runner.runCommand([ series ], callbackStderr=self.progressCallback, callbackStderrArg=runner) |
| — | — | @@ -2839,6 +2868,7 @@ |
| 2840 | 2869 | commands.append(series) |
| 2841 | 2870 | else: |
| 2842 | 2871 | for f in inputFiles: |
| | 2872 | + print "input file ", f.filename, "for buildcommand from xml dump" |
| 2843 | 2873 | # we should convert the input file to an output file I guess |
| 2844 | 2874 | # we write regular files |
| 2845 | 2875 | outputFile = DumpFilename(self.wiki, f.date, f.dumpName, f.fileType, self.fileExt) |
| — | — | @@ -3003,16 +3033,18 @@ |
| 3004 | 3034 | else: |
| 3005 | 3035 | checkpointTime = "" |
| 3006 | 3036 | checkpointFile = "" |
| 3007 | | - dumpCommand = [ "%s" % self.wiki.config.php, |
| 3008 | | - "-q", "%s/maintenance/dumpTextPass.php" % self.wiki.config.wikiDir, |
| 3009 | | - "--wiki=%s" % runner.dbName, |
| 3010 | | - "%s" % stubOption, |
| 3011 | | - "%s" % prefetch, |
| 3012 | | - "%s" % runner.forceNormalOption(), |
| 3013 | | - "%s" % checkpointTime, |
| 3014 | | - "%s" % checkpointFile, |
| 3015 | | - "--report=1000", |
| 3016 | | - "%s" % spawn ] |
| | 3037 | + scriptCommand = MultiVersion.MWScriptAsArray(runner.wiki.config, "dumpTextPass.php") |
| | 3038 | + dumpCommand = [ "%s" % self.wiki.config.php, "-q" ] |
| | 3039 | + dumpCommand.extend(scriptCommand) |
| | 3040 | + dumpCommand.extend(["--wiki=%s" % runner.dbName, |
| | 3041 | + "%s" % stubOption, |
| | 3042 | + "%s" % prefetch, |
| | 3043 | + "%s" % runner.forceNormalOption(), |
| | 3044 | + "%s" % checkpointTime, |
| | 3045 | + "%s" % checkpointFile, |
| | 3046 | + "--report=1000", |
| | 3047 | + "%s" % spawn |
| | 3048 | + ]) |
| 3017 | 3049 | |
| 3018 | 3050 | dumpCommand = filter(None, dumpCommand) |
| 3019 | 3051 | command = dumpCommand |
| — | — | @@ -3384,12 +3416,13 @@ |
| 3385 | 3417 | def buildCommand(self, runner, f): |
| 3386 | 3418 | if (not exists( runner.wiki.config.php ) ): |
| 3387 | 3419 | raise BackupError("php command %s not found" % runner.wiki.config.php) |
| 3388 | | - command = [ "%s" % runner.wiki.config.php, |
| 3389 | | - "-q", "%s/maintenance/dumpBackup.php" % runner.wiki.config.wikiDir, |
| 3390 | | - "--wiki=%s" % runner.dbName, |
| | 3420 | + scriptCommand = MultiVersion.MWScriptAsArray(runner.wiki.config, "dumpBackup.php") |
| | 3421 | + command = [ "%s" % runner.wiki.config.php, "-q" ] |
| | 3422 | + command.extend(scriptCommand) |
| | 3423 | + command.extend([ "--wiki=%s" % runner.dbName, |
| 3391 | 3424 | "--plugin=AbstractFilter:%s/extensions/ActiveAbstract/AbstractFilter.php" % runner.wiki.config.wikiDir, |
| 3392 | 3425 | "--current", "--report=1000", "%s" % runner.forceNormalOption(), |
| 3393 | | - ] |
| | 3426 | + ]) |
| 3394 | 3427 | |
| 3395 | 3428 | for v in self._variants(): |
| 3396 | 3429 | variantOption = self._variantOption(v) |