| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -179,14 +179,26 @@ |
| 180 | 180 | return(" ".join(MultiVersion.MWScriptAsArray(config, maintenanceScript))) |
| 181 | 181 | |
| 182 | 182 | def MWScriptAsArray(config, maintenanceScript): |
| 183 | | - MWScriptLocation = os.path.join(config.wikiDir,"..","multiversion","MWScript.php") |
| | 183 | + MWScriptLocation = os.path.join(config.wikiDir,"multiversion","MWScript.php") |
| 184 | 184 | if exists(MWScriptLocation): |
| 185 | 185 | return [ MWScriptLocation, maintenanceScript ] |
| 186 | 186 | else: |
| 187 | 187 | return [ "%s/maintenance/%s" % (config.wikiDir, maintenanceScript) ] |
| 188 | 188 | |
| | 189 | + def MWVersion(config, dbName): |
| | 190 | + getVersionLocation = os.path.join(config.wikiDir,"multiversion","getMWVersion") |
| | 191 | + if exists(getVersionLocation): |
| | 192 | + # run the command for the wiki and get the version |
| | 193 | + command = getVersionLocation + " " + dbName |
| | 194 | + version = RunSimpleCommand.runAndReturn(command) |
| | 195 | + if version: |
| | 196 | + version = version.rstrip() |
| | 197 | + return version |
| | 198 | + return None |
| | 199 | + |
| 189 | 200 | MWScriptAsString = staticmethod(MWScriptAsString) |
| 190 | 201 | MWScriptAsArray = staticmethod(MWScriptAsArray) |
| | 202 | + MWVersion = staticmethod(MWVersion) |
| 191 | 203 | |
| 192 | 204 | class DbServerInfo(object): |
| 193 | 205 | def __init__(self, wiki, dbName, errorCallback = None): |
| — | — | @@ -200,7 +212,6 @@ |
| 201 | 213 | if (not exists( self.wiki.config.php ) ): |
| 202 | 214 | raise BackupError("php command %s not found" % self.wiki.config.php) |
| 203 | 215 | commandList = MultiVersion.MWScriptAsArray(self.wiki.config, "getSlaveServer.php") |
| 204 | | -# command = "%s -q %s/maintenance/getSlaveServer.php --wiki=%s --group=dump" % MiscUtils.shellEscape(( |
| 205 | 216 | for i in range(0,len(commandList)): |
| 206 | 217 | phpCommand = MiscUtils.shellEscape(self.wiki.config.php) |
| 207 | 218 | dbName = MiscUtils.shellEscape(self.dbName) |
| — | — | @@ -267,7 +278,6 @@ |
| 268 | 279 | if (not exists( self.wiki.config.php ) ): |
| 269 | 280 | raise BackupError("php command %s not found" % self.wiki.config.php) |
| 270 | 281 | commandList = MultiVersion.MWScriptAsArray(self.wiki.config, "eval.php") |
| 271 | | -# command = "echo 'print $wgDBprefix; ' | %s -q %s/maintenance/eval.php --wiki=%s" % MiscUtils.shellEscape(( |
| 272 | 282 | for i in range(0,len(commandList)): |
| 273 | 283 | phpCommand = MiscUtils.shellEscape(self.wiki.config.php) |
| 274 | 284 | dbName = MiscUtils.shellEscape(self.dbName) |
| — | — | @@ -891,11 +901,14 @@ |
| 892 | 902 | if not date: |
| 893 | 903 | date = self._wiki.date |
| 894 | 904 | directory = os.path.join(self._wiki.publicDir(), date) |
| 895 | | - dirTimeStamp = os.stat(directory).st_mtime |
| 896 | | - if (not date in self._dirCache or dirTimeStamp > self._dirCacheTime[date]): |
| | 905 | + if exists(directory): |
| | 906 | + dirTimeStamp = os.stat(directory).st_mtime |
| | 907 | + if (not date in self._dirCache or dirTimeStamp > self._dirCacheTime[date]): |
| | 908 | + return True |
| | 909 | + else: |
| | 910 | + return False |
| | 911 | + else: |
| 897 | 912 | return True |
| 898 | | - else: |
| 899 | | - return False |
| 900 | 913 | |
| 901 | 914 | # warning: date can also be "latest" |
| 902 | 915 | def getFilesInDir(self, date = None): |
| — | — | @@ -903,15 +916,18 @@ |
| 904 | 917 | date = self._wiki.date |
| 905 | 918 | if (self.dirCacheOutdated(date)): |
| 906 | 919 | directory = os.path.join(self._wiki.publicDir(),date) |
| 907 | | - dirTimeStamp = os.stat(directory).st_mtime |
| 908 | | - files = os.listdir(directory) |
| 909 | | - fileObjs = [] |
| 910 | | - for f in files: |
| 911 | | - fileObj = DumpFilename(self._wiki) |
| 912 | | - fileObj.newFromFilename(f) |
| 913 | | - fileObjs.append(fileObj) |
| 914 | | - self._dirCache[date] = fileObjs |
| 915 | | - self._dirCacheTime[date] = dirTimeStamp |
| | 920 | + if exists(directory): |
| | 921 | + dirTimeStamp = os.stat(directory).st_mtime |
| | 922 | + files = os.listdir(directory) |
| | 923 | + fileObjs = [] |
| | 924 | + for f in files: |
| | 925 | + fileObj = DumpFilename(self._wiki) |
| | 926 | + fileObj.newFromFilename(f) |
| | 927 | + fileObjs.append(fileObj) |
| | 928 | + self._dirCache[date] = fileObjs |
| | 929 | + self._dirCacheTime[date] = dirTimeStamp |
| | 930 | + else: |
| | 931 | + self._dirCache[date] = [] |
| 916 | 932 | return(self._dirCache[date]) |
| 917 | 933 | |
| 918 | 934 | # list all files that exist, filtering by the given args. |
| — | — | @@ -1874,32 +1890,33 @@ |
| 1875 | 1891 | self.symLinks.cleanupSymLinks() |
| 1876 | 1892 | |
| 1877 | 1893 | for item in self.dumpItemList.dumpItems: |
| 1878 | | - dumpNames = item.getDumpName() |
| 1879 | | - if type(dumpNames).__name__!='list': |
| 1880 | | - dumpNames = [ dumpNames ] |
| | 1894 | + if (item.toBeRun()): |
| | 1895 | + dumpNames = item.getDumpName() |
| | 1896 | + if type(dumpNames).__name__!='list': |
| | 1897 | + dumpNames = [ dumpNames ] |
| 1881 | 1898 | |
| 1882 | | - if (item._chunksEnabled): |
| 1883 | | - # if there is a specific chunk, we want to only clear out |
| 1884 | | - # old files for that piece, because new files for the other |
| 1885 | | - # pieces may not have been generated yet. |
| 1886 | | - chunk = item._chunkToDo |
| 1887 | | - else: |
| 1888 | | - chunk = None |
| | 1899 | + if (item._chunksEnabled): |
| | 1900 | + # if there is a specific chunk, we want to only clear out |
| | 1901 | + # old files for that piece, because new files for the other |
| | 1902 | + # pieces may not have been generated yet. |
| | 1903 | + chunk = item._chunkToDo |
| | 1904 | + else: |
| | 1905 | + chunk = None |
| 1889 | 1906 | |
| 1890 | | - checkpoint = None |
| 1891 | | - if (item._checkpointsEnabled): |
| 1892 | | - if (item.checkpointFile): |
| 1893 | | - # if there's a specific checkpoint file we are |
| 1894 | | - # rerunning, we would only clear out old copies |
| 1895 | | - # of that very file. meh. how likely is it that we |
| 1896 | | - # have one? these files are time based and the start/end pageids |
| 1897 | | - # are going to fluctuate. whatever |
| 1898 | | - checkpoint = item.checkpointFile.checkpoint |
| | 1907 | + checkpoint = None |
| | 1908 | + if (item._checkpointsEnabled): |
| | 1909 | + if (item.checkpointFile): |
| | 1910 | + # if there's a specific checkpoint file we are |
| | 1911 | + # rerunning, we would only clear out old copies |
| | 1912 | + # of that very file. meh. how likely is it that we |
| | 1913 | + # have one? these files are time based and the start/end pageids |
| | 1914 | + # are going to fluctuate. whatever |
| | 1915 | + checkpoint = item.checkpointFile.checkpoint |
| 1899 | 1916 | |
| 1900 | | - for d in dumpNames: |
| 1901 | | - self.symLinks.removeSymLinksFromOldRuns(self.wiki.date, d, chunk, checkpoint ) |
| | 1917 | + for d in dumpNames: |
| | 1918 | + self.symLinks.removeSymLinksFromOldRuns(self.wiki.date, d, chunk, checkpoint ) |
| 1902 | 1919 | |
| 1903 | | - self.feeds.cleanupFeeds() |
| | 1920 | + self.feeds.cleanupFeeds() |
| 1904 | 1921 | |
| 1905 | 1922 | def makeDir(self, dir): |
| 1906 | 1923 | if self._makeDirEnabled: |
| — | — | @@ -1974,6 +1991,7 @@ |
| 1975 | 1992 | # earlier dates. checkpoint ranges change, and configuration of chunks changes too, so maybe |
| 1976 | 1993 | # old files still exist and the links need to be removed because we have newer files for the |
| 1977 | 1994 | # same phase of the dump. |
| | 1995 | + |
| 1978 | 1996 | if (self._enabled): |
| 1979 | 1997 | latestDir = self.dumpDir.latestDir() |
| 1980 | 1998 | files = os.listdir(latestDir) |
| — | — | @@ -2854,7 +2872,6 @@ |
| 2855 | 2873 | # we don't checkpoint the checkpoint file. |
| 2856 | 2874 | self._checkpointsEnabled = False |
| 2857 | 2875 | self.pageIDRange = pageIDRange |
| 2858 | | - |
| 2859 | 2876 | Dump.__init__(self, name, desc) |
| 2860 | 2877 | |
| 2861 | 2878 | def getDumpNameBase(self): |
| — | — | @@ -2898,9 +2915,6 @@ |
| 2899 | 2916 | commands.append(series) |
| 2900 | 2917 | else: |
| 2901 | 2918 | for f in inputFiles: |
| 2902 | | - print "input file ", f.filename, "for buildcommand from xml dump" |
| 2903 | | - # we should convert the input file to an output file I guess |
| 2904 | | - # we write regular files |
| 2905 | 2919 | outputFile = DumpFilename(self.wiki, f.date, f.dumpName, f.fileType, self.fileExt) |
| 2906 | 2920 | series = self.buildCommand(runner, f) |
| 2907 | 2921 | commands.append(series) |
| — | — | @@ -3449,10 +3463,15 @@ |
| 3450 | 3464 | scriptCommand = MultiVersion.MWScriptAsArray(runner.wiki.config, "dumpBackup.php") |
| 3451 | 3465 | command = [ "%s" % runner.wiki.config.php, "-q" ] |
| 3452 | 3466 | command.extend(scriptCommand) |
| | 3467 | + version = MultiVersion.MWVersion(runner.wiki.config, runner.dbName) |
| | 3468 | + if version: |
| | 3469 | + abstractFilterCommand = "--plugin=AbstractFilter:%s/%s/extensions/ActiveAbstract/AbstractFilter.php" % (runner.wiki.config.wikiDir, version) |
| | 3470 | + else: |
| | 3471 | + abstractFilterCommand = "--plugin=AbstractFilter:%s/extensions/ActiveAbstract/AbstractFilter.php" % runner.wiki.config.wikiDir |
| 3453 | 3472 | command.extend([ "--wiki=%s" % runner.dbName, |
| 3454 | | - "--plugin=AbstractFilter:%s/extensions/ActiveAbstract/AbstractFilter.php" % runner.wiki.config.wikiDir, |
| 3455 | | - "--current", "--report=1000", "%s" % runner.forceNormalOption(), |
| 3456 | | - ]) |
| | 3473 | + abstractFilterCommand, |
| | 3474 | + "--current", "--report=1000", "%s" % runner.forceNormalOption(), |
| | 3475 | + ]) |
| 3457 | 3476 | |
| 3458 | 3477 | for v in self._variants(): |
| 3459 | 3478 | variantOption = self._variantOption(v) |