| Index: branches/ariel/xmldumps-backup/worker.py |
| — | — | @@ -1959,7 +1959,9 @@ |
| 1960 | 1960 | if message: |
| 1961 | 1961 | print message |
| 1962 | 1962 | print "Usage: python worker.py [options] [wikidbname]" |
| 1963 | | - print "Options: --date, --checkpoint, --job, --force, --noprefetch, --nospawn" |
| | 1963 | + print "Options: --configfile, --date, --checkpoint, --job, --force, --noprefetch, --nospawn" |
| | 1964 | + print "--configfile: Specify an alternative configuration file to read." |
| | 1965 | + print " Default config file name: wikidump.conf" |
| 1964 | 1966 | print "--date: Rerun dump of a given date (probably unwise)" |
| 1965 | 1967 | print "--checkpoint: Run just the specified step (deprecated)" |
| 1966 | 1968 | print "--job: Run just the specified step or set of steps; for the list," |
| — | — | @@ -1978,10 +1980,9 @@ |
| 1979 | 1981 | |
| 1980 | 1982 | if __name__ == "__main__": |
| 1981 | 1983 | try: |
| 1982 | | - config = WikiDump.Config() |
| 1983 | | - |
| 1984 | 1984 | date = None |
| 1985 | 1985 | checkpoint = None |
| | 1986 | + configFile = False |
| 1986 | 1987 | forceLock = False |
| 1987 | 1988 | prefetch = True |
| 1988 | 1989 | spawn = True |
| — | — | @@ -1989,7 +1990,7 @@ |
| 1990 | 1991 | |
| 1991 | 1992 | try: |
| 1992 | 1993 | (options, remainder) = getopt.gnu_getopt(sys.argv[1:], "", |
| 1993 | | - ['date=', 'checkpoint=', 'job=', 'force', 'noprefetch', 'nospawn']) |
| | 1994 | + ['date=', 'checkpoint=', 'job=', 'configfile=', 'force', 'noprefetch', 'nospawn']) |
| 1994 | 1995 | except: |
| 1995 | 1996 | usage("Unknown option specified") |
| 1996 | 1997 | |
| — | — | @@ -1998,6 +1999,8 @@ |
| 1999 | 2000 | date = val |
| 2000 | 2001 | elif opt == "--checkpoint": |
| 2001 | 2002 | checkpoint = val |
| | 2003 | + elif opt == "--configfile": |
| | 2004 | + configFile = val |
| 2002 | 2005 | elif opt == "--force": |
| 2003 | 2006 | forceLock = True |
| 2004 | 2007 | elif opt == "--noprefetch": |
| — | — | @@ -2012,6 +2015,12 @@ |
| 2013 | 2016 | if (jobRequested and forceLock): |
| 2014 | 2017 | usage("--force cannot be used with --job option") |
| 2015 | 2018 | |
| | 2019 | + # allow alternate config file |
| | 2020 | + if (configFile): |
| | 2021 | + config = WikiDump.Config(configFile) |
| | 2022 | + else: |
| | 2023 | + config = WikiDump.Config() |
| | 2024 | + |
| 2016 | 2025 | if len(remainder) > 0: |
| 2017 | 2026 | wiki = WikiDump.Wiki(config, remainder[0]) |
| 2018 | 2027 | # if we are doing one piece only of the dump, we don't try to grab a lock. |
| Index: branches/ariel/xmldumps-backup/worker |
| — | — | @@ -2,8 +2,16 @@ |
| 3 | 3 | |
| 4 | 4 | WIKIDUMP_BASE=`dirname "$0"` |
| 5 | 5 | |
| | 6 | +if [ ! -z "$1" ]; then |
| | 7 | + configFile="$1" |
| | 8 | +fi |
| | 9 | + |
| 6 | 10 | while true; do |
| | 11 | + if [ ! -z "$configFile" ]; then |
| | 12 | + python $WIKIDUMP_BASE/worker.py "--configfile" "$configFile" |
| | 13 | + else |
| 7 | 14 | python $WIKIDUMP_BASE/worker.py |
| 8 | | - echo "sleeping" |
| 9 | | - sleep 30 |
| | 15 | + fi |
| | 16 | + echo "sleeping" |
| | 17 | + sleep 30 |
| 10 | 18 | done |
| Index: branches/ariel/xmldumps-backup/monitor.py |
| — | — | @@ -1,9 +1,14 @@ |
| 2 | 2 | # Wiki dump-generation monitor |
| 3 | 3 | |
| 4 | 4 | import os |
| | 5 | +import sys |
| 5 | 6 | import WikiDump |
| 6 | 7 | |
| 7 | | -config = WikiDump.Config() |
| | 8 | +# can specify name of alternate config file |
| | 9 | +if (sys.argv[1]): |
| | 10 | + config = WikiDump.Config(sys.argv[1]) |
| | 11 | +else: |
| | 12 | + config = WikiDump.Config() |
| 8 | 13 | |
| 9 | 14 | def generateIndex(): |
| 10 | 15 | running = False |
| Index: branches/ariel/xmldumps-backup/WikiDump.py |
| — | — | @@ -80,10 +80,12 @@ |
| 81 | 81 | return dbs |
| 82 | 82 | |
| 83 | 83 | class Config(object): |
| 84 | | - def __init__(self): |
| | 84 | + def __init__(self, configFile=False): |
| 85 | 85 | home = os.path.dirname(sys.argv[0]) |
| | 86 | + if (not configFile): |
| | 87 | + configFile = "wikidump.conf" |
| 86 | 88 | files = [ |
| 87 | | - os.path.join(home, "wikidump.conf"), |
| | 89 | + os.path.join(home,configFile), |
| 88 | 90 | "/etc/wikidump.conf", |
| 89 | 91 | os.path.join(os.getenv("HOME"), ".wikidump.conf")] |
| 90 | 92 | defaults = { |
| — | — | @@ -142,8 +144,17 @@ |
| 143 | 145 | conf = ConfigParser.SafeConfigParser(defaults) |
| 144 | 146 | conf.read(files) |
| 145 | 147 | |
| 146 | | - self.dbList = dbList(conf.get("wiki", "dblist")) |
| 147 | | - self.skipDbList = dbList(conf.get("wiki", "skipdblist")) |
| | 148 | + try: |
| | 149 | + self.dbList = dbList(conf.get("wiki", "dblist")) |
| | 150 | + except ConfigParser.NoSectionError: |
| | 151 | + print "The mandatory configuration section 'wiki' was not defined." |
| | 152 | + print "Either the section was ommitted or none of the files in the list" |
| | 153 | + print "%s exists. Giving up." % files |
| | 154 | + raise |
| | 155 | + try: |
| | 156 | + self.skipDbList = dbList(conf.get("wiki", "skipdblist")) |
| | 157 | + except ConfigParser.NoSectionError: |
| | 158 | + self.skipDbList = [] |
| 148 | 159 | self.dbList = list(set(self.dbList) - set(self.skipDbList)) |
| 149 | 160 | |
| 150 | 161 | self.privateList = dbList(conf.get("wiki", "privatelist")) |
| Index: branches/ariel/xmldumps-backup/monitor |
| — | — | @@ -2,10 +2,16 @@ |
| 3 | 3 | |
| 4 | 4 | WIKIDUMP_BASE=`dirname "$0"` |
| 5 | 5 | |
| | 6 | +if [ ! -z "$1" ]; then |
| | 7 | + configFile="$1" |
| | 8 | +else |
| | 9 | + configFile="" |
| | 10 | +fi |
| | 11 | + |
| 6 | 12 | while true; do |
| 7 | 13 | echo "" |
| 8 | 14 | echo "Sweeping!" |
| 9 | | - python $WIKIDUMP_BASE/monitor.py |
| | 15 | + python $WIKIDUMP_BASE/monitor.py "$configFile" |
| 10 | 16 | echo "sleeping" |
| 11 | 17 | sleep 15 |
| 12 | 18 | done |