r53928 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r53927‎ | r53928 | r53929 >
Date:11:16, 29 July 2009
Author:daniel
Status:deferred
Tags:
Comment:
error handling and termination
Modified paths:
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/CliApp.java (modified) (history)
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/StoreBackedApp.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportDump.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/ExtractFromDump.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/XmlDumpDriver.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java
——@@ -7,6 +7,7 @@
88
99 import de.brightbyte.application.Agenda;
1010 import de.brightbyte.application.Agenda.Monitor;
 11+import de.brightbyte.db.DatabaseTask;
1112 import de.brightbyte.io.Prompt;
1213 import de.brightbyte.util.PersistenceException;
1314 import de.brightbyte.wikiword.StoreBackedApp;
——@@ -14,6 +15,7 @@
1516 import de.brightbyte.wikiword.store.WikiWordStore;
1617 import de.brightbyte.wikiword.store.WikiWordStoreFactory;
1718 import de.brightbyte.wikiword.store.builder.DatabaseConceptStoreBuilders;
 19+import de.brightbyte.wikiword.store.builder.DatabaseWikiWordStoreBuilder;
1820 import de.brightbyte.wikiword.store.builder.WikiWordConceptStoreBuilder;
1921 import de.brightbyte.wikiword.store.builder.WikiWordStoreBuilder;
2022
——@@ -36,7 +38,7 @@
3739 protected Operation operation = null;
3840 private Monitor agendaMonitor;
3941 protected String[] baseTasks = new String[] {};
40 - protected InputFileHelper inputHelper;
 42+ protected InputFileHelper inputHelper;
4143
4244 public ImportApp(String agendaTask, boolean allowGlobal, boolean allowLocal) { //TODO: agenda-params!
4345 super(allowGlobal, allowLocal);
——@@ -268,6 +270,10 @@
269271 protected void createStores(WikiWordStoreFactory<? extends S> factory) throws IOException, PersistenceException {
270272 super.createStores(factory);
271273
 274+ if (conceptStore instanceof DatabaseWikiWordStoreBuilder) {
 275+ ((DatabaseWikiWordStoreBuilder)conceptStore).setBackgroundErrorHandler(new FatalBackgroundErrorHandler<Runnable, Throwable, Error>());
 276+ }
 277+
272278 boolean noimport = args.isSet("noimport");
273279
274280 if (!noimport && useAgenda) {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
——@@ -21,6 +21,7 @@
2222 import de.brightbyte.wikiword.model.LocalConceptReference;
2323 import de.brightbyte.wikiword.processor.ImportProgressTracker;
2424 import de.brightbyte.wikiword.schema.AliasScope;
 25+import de.brightbyte.wikiword.schema.ConceptStoreSchemas;
2526 import de.brightbyte.wikiword.store.builder.IncrementalStoreBuilder;
2627 import de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder;
2728 import de.brightbyte.wikiword.store.builder.PropertyStoreBuilder;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportDump.java
——@@ -17,7 +17,7 @@
1818 super(agendaTask, false, true);
1919 }
2020
21 - private URL dumpFile;
 21+ protected URL dumpFile;
2222
2323 @Override
2424 protected boolean applyArguments() {
——@@ -107,7 +107,7 @@
108108
109109 ///////////////////////// main import run ////////////////////////////////////
110110 if (agenda.beginTask("ImportDump.run", "analysis")) {
111 - DataSourceDriver driver = new XmlDumpDriver(getDumpFileURL(), inputHelper, getLogOutput(), tweaks);
 111+ DataSourceDriver driver = new XmlDumpDriver(getDumpFileURL(), inputHelper, getLogOutput(), new FatalBackgroundErrorHandler<XmlDumpDriver, Throwable, RuntimeException>(), tweaks);
112112
113113 importer.reset();
114114 importer.prepare();
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/ExtractFromDump.java
——@@ -6,7 +6,6 @@
77
88 import de.brightbyte.util.PersistenceException;
99 import de.brightbyte.wikiword.analyzer.WikiTextAnalyzer;
10 -import de.brightbyte.wikiword.builder.InputFileHelper;
1110 import de.brightbyte.wikiword.output.DataOutput;
1211 import de.brightbyte.wikiword.processor.DataSourceDriver;
1312 import de.brightbyte.wikiword.processor.WikiWordPageProcessor;
——@@ -61,7 +60,7 @@
6261 processor.setLogOutput(getLogOutput());
6362 processor.configure(args);
6463
65 - DataSourceDriver driver = new XmlDumpDriver(dumpFile, inputHelper, getLogOutput(), tweaks);
 64+ DataSourceDriver driver = new XmlDumpDriver(dumpFile, inputHelper, getLogOutput(), new FatalBackgroundErrorHandler<XmlDumpDriver, Throwable, RuntimeException>(), tweaks);
6665
6766 processor.reset();
6867 processor.prepare();
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
——@@ -1342,12 +1342,14 @@
13431343 @Override
13441344 protected DatabaseConceptInfoStoreBuilder<LocalConcept> newConceptInfoStoreBuilder() throws SQLException {
13451345 ConceptInfoStoreSchema schema = new ConceptInfoStoreSchema(getDatasetIdentifier(), getDatabaseAccess().getConnection(), true, tweaks, false, true);
 1346+ schema.setBackgroundErrorHandler(database.getBackgroundErrorHandler());
13461347 return new DatabaseLocalConceptInfoStoreBuilder(schema, tweaks, getAgenda());
13471348 }
13481349
13491350 @Override
13501351 protected DatabaseStatisticsStoreBuilder newStatisticsStoreBuilder() throws SQLException {
13511352 StatisticsStoreSchema schema = new LocalStatisticsStoreSchema(getDatasetIdentifier(), getDatabaseAccess().getConnection(), tweaks, false);
 1353+ schema.setBackgroundErrorHandler(database.getBackgroundErrorHandler());
13521354 return new DatabaseLocalStatisticsStoreBuilder(schema, tweaks, getAgenda());
13531355 }
13541356
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java
——@@ -12,7 +12,6 @@
1313 public abstract void prepareImport() throws PersistenceException;
1414 public void finalizeImport() throws PersistenceException;
1515
16 -
1716 public abstract void close(boolean flush) throws PersistenceException;
1817
1918 public abstract void open() throws PersistenceException;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java
——@@ -40,14 +40,14 @@
4141 }
4242
4343 public DatabaseTextStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException, PersistenceException {
44 - this(new LocalConceptStoreSchema(conceptStore.getCorpus(),
45 - conceptStore.getDatabaseAccess().getConnection(),
46 - tweaks, true),
 44+ this((LocalConceptStoreSchema)conceptStore.getDatabaseAccess(),
4745 new TextStoreSchema(conceptStore.getCorpus(),
4846 conceptStore.getDatabaseAccess().getConnection(),
4947 tweaks, true),
5048 tweaks,
5149 conceptStore.getAgenda());
 50+
 51+ database.setBackgroundErrorHandler(conceptStore.getDatabaseAccess().getBackgroundErrorHandler());
5252 }
5353
5454 protected DatabaseTextStoreBuilder(LocalConceptStoreSchema conceptStoreSchema, TextStoreSchema database, TweakSet tweaks, Agenda agenda) throws SQLException, PersistenceException {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
——@@ -36,15 +36,15 @@
3737 }
3838
3939 public DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException, PersistenceException {
40 - this(new LocalConceptStoreSchema(conceptStore.getCorpus(),
41 - conceptStore.getDatabaseAccess().getConnection(),
42 - tweaks, true),
 40+ this((LocalConceptStoreSchema)conceptStore.getDatabaseAccess(),
4341 new PropertyStoreSchema(conceptStore.getCorpus(),
4442 conceptStore.getDatabaseAccess().getConnection(),
4543 tweaks, true),
4644 conceptStore.idManager,
4745 tweaks,
4846 conceptStore.getAgenda());
 47+
 48+ database.setBackgroundErrorHandler(conceptStore.getDatabaseAccess().getBackgroundErrorHandler());
4949 }
5050
5151 protected DatabasePropertyStoreBuilder(LocalConceptStoreSchema conceptStoreSchema, PropertyStoreSchema database, PersistentIdManager idManager, TweakSet tweaks, Agenda agenda) throws SQLException, PersistenceException {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
——@@ -19,6 +19,7 @@
2020 import de.brightbyte.db.ReferenceField;
2121 import de.brightbyte.db.RelationTable;
2222 import de.brightbyte.db.StatementBasedInserter;
 23+import de.brightbyte.util.ErrorHandler;
2324 import de.brightbyte.util.PersistenceException;
2425 import de.brightbyte.util.StringUtils;
2526 import de.brightbyte.wikiword.TweakSet;
——@@ -64,6 +65,14 @@
6566 warningInserter.disableAutomaticField();
6667 }
6768
 69+ public void setBackgroundErrorHandler(ErrorHandler<Runnable, Throwable, Error> handler) {
 70+ database.setBackgroundErrorHandler(handler);
 71+ }
 72+
 73+ public ErrorHandler<Runnable, Throwable, Error> getBackgroundErrorHandler() {
 74+ return database.getBackgroundErrorHandler();
 75+ }
 76+
6877 protected Inserter configureTable(String name, int bufferLength, int bufferWidth) throws SQLException {
6978 DatabaseTable t = database.getTable(name);
7079 return configureTable(t, bufferLength, bufferWidth);
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/XmlDumpDriver.java
——@@ -17,6 +17,7 @@
1818
1919 import de.brightbyte.io.LeveledOutput;
2020 import de.brightbyte.job.BlockingJobQueue;
 21+import de.brightbyte.util.ErrorHandler;
2122 import de.brightbyte.util.PersistenceException;
2223 import de.brightbyte.wikiword.Namespace;
2324 import de.brightbyte.wikiword.NamespaceSet;
——@@ -33,6 +34,7 @@
3435 protected InputStream in;
3536 protected TweakSet tweaks;
3637 protected LeveledOutput log;
 38+ protected ErrorHandler<XmlDumpDriver, Throwable, RuntimeException> errorHandler;
3739
3840 protected class Sink implements DumpWriter {
3941
——@@ -145,9 +147,8 @@
146148 try {
147149 importPage(p, r);
148150 } catch (Throwable e) {
149 - log.error("exception while processing page `"+p.Title.toString()+"` (id="+p.Id+", rev="+r.Id+", date="+r.Timestamp.getTime()+")", e);
150 - error = e;
151151 executor.shutdownNow();
 152+ handleImportError("exception while processing page `"+p.Title.toString()+"` (id="+p.Id+", rev="+r.Id+", date="+r.Timestamp.getTime()+")", e);
152153
153154 if (e instanceof Error) throw (Error)e;
154155 }
——@@ -197,24 +198,29 @@
198199 importer.handlePage(page.Id, page.Title.Namespace, page.Title.Text, revision.Text, revision.Timestamp.getTime());
199200 }
200201
 202+ protected void handleImportError(String message, Throwable e) {
 203+ error = e;
 204+ if (errorHandler!=null) errorHandler.handleError(XmlDumpDriver.this, message, e);
 205+ else log.error(message, e);
 206+ }
201207 }
202208
203 - public XmlDumpDriver(URL dump, InputFileHelper inputHelper, LeveledOutput log, TweakSet tweaks) {
 209+ public XmlDumpDriver(URL dump, InputFileHelper inputHelper, LeveledOutput log, ErrorHandler<XmlDumpDriver, Throwable, RuntimeException> errorHandler, TweakSet tweaks) {
204210 if (dump==null) throw new NullPointerException();
205211 this.dump= dump;
206 - init(inputHelper, log, tweaks);
 212+ init(inputHelper, log, errorHandler, tweaks);
207213 }
208214
209 - public XmlDumpDriver(InputStream in, LeveledOutput log, TweakSet tweaks) {
 215+ public XmlDumpDriver(InputStream in, LeveledOutput log, ErrorHandler<XmlDumpDriver, Throwable, RuntimeException> errorHandler, TweakSet tweaks) {
210216 if (in==null) throw new NullPointerException();
211217 this.in= in;
212 - init(null, log, tweaks);
 218+ init(null, log, errorHandler, tweaks);
213219 }
214220
215221 private int importQueueCapacity = 0;
216222 private InputFileHelper inputHelper;
217223
218 - private void init(InputFileHelper inputHelper, LeveledOutput log, TweakSet tweaks) {
 224+ private void init(InputFileHelper inputHelper, LeveledOutput log, ErrorHandler<XmlDumpDriver, Throwable, RuntimeException> errorHandler, TweakSet tweaks) {
219225 if (log==null) throw new NullPointerException();
220226 if (tweaks==null) throw new NullPointerException();
221227 if (inputHelper==null && in==null) throw new NullPointerException();
——@@ -222,6 +228,7 @@
223229 this.tweaks = tweaks;
224230 this.log = log;
225231 this.inputHelper = inputHelper;
 232+ this.errorHandler = errorHandler;
226233
227234 importQueueCapacity = tweaks.getTweak("dumpdriver.pageImportQueue", 8);
228235 }
——@@ -243,4 +250,5 @@
244251 sink.close(); //NOTE: make sure the executor queue is terminated
245252 }
246253 }
 254+
247255 }
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/CliApp.java
——@@ -16,6 +16,7 @@
1717 import de.brightbyte.io.LeveledOutput;
1818 import de.brightbyte.io.LogOutput;
1919 import de.brightbyte.io.Output;
 20+import de.brightbyte.util.ErrorHandler;
2021 import de.brightbyte.util.PersistenceException;
2122 import de.brightbyte.util.SystemUtils;
2223 import de.brightbyte.wikiword.rdf.WikiWordIdentifiers;
——@@ -25,6 +26,20 @@
2627 */
2728 public abstract class CliApp {
2829
 30+ public class FatalBackgroundErrorHandler<T, X extends Throwable, Y extends Throwable> implements
 31+ ErrorHandler<T, X, Y> {
 32+
 33+ public void handleError(T context, String message, X error) throws Y {
 34+ String m = message;
 35+ if (context!=null) m = m + " ("+context+")";
 36+
 37+ out.error(m, error);
 38+
 39+ terminate();
 40+ }
 41+
 42+ }
 43+
2944 public static final String VERSION_INFO = "WikiWord by Daniel Kinzler, brightbyte.de, 2007-2009";
3045 public static final String LICENSE_INFO = "Originally created at the University of Leipzig,\n\tdevelopment supported by Wikimedia Deutschland.\n\tFree Software, GNU LGPL. See COPYING for details.";
3146
——@@ -415,4 +430,8 @@
416431 return exitCode;
417432 }
418433
 434+ protected void terminate() {
 435+ info("exiting application");
 436+ exit(55);
 437+ }
419438 }
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/StoreBackedApp.java
——@@ -185,4 +185,31 @@
186186 }
187187 }
188188
 189+ protected void terminate() {
 190+ info("terminating application");
 191+
 192+ //XXX: note: flushing first would be nice, but
 193+ // can deadlock, especially when called from a worker of the database's background flush queue!
 194+ // would need to detect that.
 195+ /*
 196+ try {
 197+ closeStores(true);
 198+ } catch (Exception ex) {
 199+ error("failed to flush stores", ex);
 200+ try {
 201+ closeStores(false);
 202+ } catch (Exception exx) {
 203+ error("failed to close stores", exx);
 204+ }
 205+ } */
 206+
 207+ try {
 208+ closeStores(false);
 209+ } catch (Exception exx) {
 210+ error("failed to close stores", exx);
 211+ }
 212+
 213+ super.terminate();
 214+ }
 215+
189216 }

Status & tagging log