r46734 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r46733‎ | r46734 | r46735 >
Date:20:36, 2 February 2009
Author:daniel
Status:deferred
Tags:
Comment:
init all stores (WORK IN PROGRESS)
Modified paths:
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/FauxStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java
@@ -256,10 +256,11 @@
257257
258258 if (operation == Operation.FRESH) {
259259 section("-- purge --------------------------------------------------");
260 - store.prepare(true, getDropWarnings()); //FIXME: don't purge warning always... but when?!
 260+ store.initialize(true, getDropWarnings()); //FIXME: don't purge warning always... but when?!
 261+ ...all stores...
261262 }
262263 else {
263 - store.prepare(false, getDropWarnings());
 264+ store.initialize(false, getDropWarnings());
264265 }
265266
266267 if (!useAgenda || agenda.beginTask("ImportApp.launch", agendaTask)) {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
@@ -49,6 +49,8 @@
5050 @Override
5151 public void prepare() throws PersistenceException {
5252 store.prepare();
 53+ if (storeProperties) propertyStore.prepare();
 54+ if (storeFlatText) textStore.prepare();
5355 }
5456
5557 @Override
@@ -58,6 +60,16 @@
5961 endTask("ConceptImporter.finish", "finishImport");
6062 }
6163
 64+ if (beginTask("ConceptImporter.finish", "finishImport")) {
 65+ propertyStore.finishImport();
 66+ endTask("ConceptImporter.finish", "finishImport");
 67+ }
 68+
 69+ if (beginTask("ConceptImporter.finish", "finishImport")) {
 70+ textStore.finishImport();
 71+ endTask("ConceptImporter.finish", "finishImport");
 72+ }
 73+
6274 if (beginTask("ConceptImporter.finish", "finishBadLinks")) {
6375 store.finishBadLinks();
6476 endTask("ConceptImporter.finish", "finishBadLinks");
@@ -86,11 +98,13 @@
8799 endTask("ConceptImporter.finish", "finishAliases");
88100 }
89101
90 - if (beginTask("ConceptImporter.finish", "propertyStore#finishAliases")) {
 102+ if (propertyStore!=null && beginTask("ConceptImporter.finish", "propertyStore#finishAliases")) {
91103 propertyStore.finishAliases();
92104 endTask("ConceptImporter.finish", "propertyStore#finishAliases");
93105 }
94106
 107+ //TODO: finish aliases for textStore!
 108+
95109 if (beginTask("ConceptImporter.finish", "finishRelations")) {
96110 store.finishRelations();
97111 endTask("ConceptImporter.finish", "finishRelations");
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java
@@ -79,7 +79,7 @@
8080 log("* optimize *");
8181 }
8282
83 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 83+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
8484 log("* prepare("+purge+", "+dropAll+") *");
8585 }
8686
@@ -152,7 +152,7 @@
153153 log("* optimize *");
154154 }
155155
156 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 156+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
157157 log("* prepare *");
158158 }
159159
@@ -229,7 +229,7 @@
230230 log("* optimize *");
231231 }
232232
233 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 233+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
234234 log("* prepare *");
235235 }
236236
@@ -299,7 +299,7 @@
300300 log("* optimize *");
301301 }
302302
303 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 303+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
304304 log("* prepare *");
305305 }
306306
@@ -387,7 +387,7 @@
388388 log("* close *");
389389 }
390390
391 - public void prepare(boolean purge, boolean dropAll) {
 391+ public void initialize(boolean purge, boolean dropAll) {
392392 log("* prepare *");
393393 }
394394
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
@@ -20,6 +20,7 @@
2121 import de.brightbyte.util.PersistenceException;
2222 import de.brightbyte.util.StringUtils;
2323 import de.brightbyte.wikiword.TweakSet;
 24+import de.brightbyte.wikiword.schema.AliasScope;
2425 import de.brightbyte.wikiword.schema.WikiWordStoreSchema;
2526 import de.brightbyte.wikiword.store.DatabaseWikiWordStore;
2627
@@ -343,7 +344,7 @@
344345 }
345346 */
346347
347 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 348+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
348349 if (purge) {
349350 dropTables(true, dropAll);
350351 createTables(false);
@@ -449,5 +450,32 @@
450451 throw new PersistenceException(e);
451452 }
452453 }
 454+
 455+ protected int resolveRedirects(RelationTable aliasTable, DatabaseTable table, String relNameField, String relIdField, AliasScope scope, int chunkFactor) throws PersistenceException {
 456+ if (relIdField==null && relNameField==null) throw new IllegalArgumentException("relNameFields and relIdField can't both be null");
 457+
 458+ DatabaseField nmField = relNameField ==null ? null : table.getField(relNameField);
 459+ DatabaseField idField = relIdField == null ? null : table.getField(relIdField);
 460+
 461+ if (nmField!=null && !(nmField instanceof ReferenceField)) throw new IllegalArgumentException(relNameField+" is not a reference field in table "+table.getName());
 462+ if (idField!=null && !(idField instanceof ReferenceField)) throw new IllegalArgumentException(relIdField+" is not a reference field in table "+table.getName());
 463+
 464+ String nmTable = nmField == null ? null : ((ReferenceField)nmField).getTargetTable();
 465+ String idTable = idField == null ? null : ((ReferenceField)idField).getTargetTable();
 466+
 467+ if (idTable!=null && nmTable!=null && !nmTable.equals(idTable)) throw new IllegalArgumentException(relNameField+" and "+relIdField+" in table "+table.getName()+" do not reference the same table: "+nmTable+" != "+idTable);
453468
 469+ String sql = "UPDATE "+table.getSQLName()+" as R JOIN "+aliasTable.getSQLName()+" as E "
 470+ + ((idField!=null) ? " ON R."+relIdField+" = E.source " : " ON R."+relNameField+" = E.source_name ")
 471+ + " SET ";
 472+
 473+ if (nmField!=null) sql += " R."+relNameField+" = E.target_name";
 474+ if (nmField!=null && idField!=null) sql += ", ";
 475+ if (idField!=null) sql += " R."+relIdField+" = E.target";
 476+
 477+ String where = scope == null ? null : " scope = "+scope.ordinal();
 478+
 479+ return executeChunkedUpdate("resolveRedirects", table.getName()+"."+relNameField+"+"+relIdField, sql, where, aliasTable, "source", chunkFactor);
 480+ }
 481+
454482 }
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
@@ -35,9 +35,7 @@
3636 import de.brightbyte.wikiword.schema.ConceptInfoStoreSchema;
3737 import de.brightbyte.wikiword.schema.LocalConceptStoreSchema;
3838 import de.brightbyte.wikiword.schema.LocalStatisticsStoreSchema;
39 -import de.brightbyte.wikiword.schema.PropertyStoreSchema;
4039 import de.brightbyte.wikiword.schema.StatisticsStoreSchema;
41 -import de.brightbyte.wikiword.schema.TextStoreSchema;
4240 import de.brightbyte.wikiword.store.DatabaseLocalConceptStore;
4341
4442 /**
@@ -160,7 +158,7 @@
161159 }
162160
163161 @Override
164 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 162+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
165163 if (purge) {
166164 if (idManager!=null) idManager.clear();
167165 }
@@ -175,7 +173,7 @@
176174 }
177175 }
178176
179 - super.prepare(purge, dropAll);
 177+ super.initialize(purge, dropAll);
180178 }
181179
182180 public ConceptType getConceptType(int type) {
@@ -671,7 +669,7 @@
672670 //NOTE: need to resolve category-aliases here, so no concepts are generated for aliased categories!
673671 //NOTE: bad category redirs have been droped in finishBadLinks
674672 if (beginTask("finishMissingConcpets", "resolveRedirects:broader")) {
675 - int n = resolveRedirects(broaderTable, "broad_name", idManager==null ? null : "broad", AliasScope.CATEGORY, 1);
 673+ int n = resolveRedirects(aliasTable, broaderTable, "broad_name", idManager==null ? null : "broad", AliasScope.CATEGORY, 1);
676674 endTask("finishMissingConcpets", "resolveRedirects:broader", n+" entries");
677675 }
678676
@@ -743,24 +741,24 @@
744742 if (beginTask("finishAliases", "resolveRedirects:link")) {
745743 //XXX: SLOW!
746744 //TODO: smaller chunks? chunk on target table, not alias table? force index?
747 - int n = resolveRedirects(linkTable, "target_name", "target", AliasScope.REDIRECT, 8);
 745+ int n = resolveRedirects(aliasTable, linkTable, "target_name", "target", AliasScope.REDIRECT, 8);
748746 endTask("finishAliases", "resolveRedirects:link", n+" entries");
749747 }
750748
751749 //NOTE: broader.broad_name already done in finishMissingConcepts for AliasScope.BROADER
752750
753751 if (beginTask("finishAliases", "resolveRedirects:about")) {
754 - int n = resolveRedirects(aboutTable, null, "concept", null, 1);
 752+ int n = resolveRedirects(aliasTable, aboutTable, null, "concept", null, 1);
755753 endTask("finishAliases", "resolveRedirects:about", n+" entries");
756754 }
757755
758756 if (beginTask("finishAliases", "resolveRedirects:narrow")) {
759 - int n = resolveRedirects(broaderTable, "narrow_name", "narrow", null, 1);
 757+ int n = resolveRedirects(aliasTable, broaderTable, "narrow_name", "narrow", null, 1);
760758 endTask("finishAliases", "resolveRedirects:narrow", n+" entries");
761759 }
762760
763761 if (beginTask("finishAliases", "resolveRedirects:broad")) {
764 - int n = resolveRedirects(broaderTable, "broad_name", "broad", null, 1);
 762+ int n = resolveRedirects(aliasTable, broaderTable, "broad_name", "broad", null, 1);
765763 endTask("finishAliases", "resolveRedirects:broad", n+" entries");
766764 }
767765
@@ -1148,29 +1146,6 @@
11491147 return executeChunkedUpdate("buildIdLinks", table.getName()+"."+relNameField, sql, where, target, targetIdField, chunkFactor);
11501148 }
11511149
1152 - protected int resolveRedirects(DatabaseTable table, String relNameField, String relIdField, AliasScope scope, int chunkFactor) throws PersistenceException {
1153 - DatabaseField nmField = table.getField(relNameField);
1154 - DatabaseField idField = relIdField == null ? null : table.getField(relIdField);
1155 -
1156 - if (!(nmField instanceof ReferenceField)) throw new IllegalArgumentException(relNameField+" is not a reference field in table "+table.getName());
1157 - if (idField!=null && !(idField instanceof ReferenceField)) throw new IllegalArgumentException(relIdField+" is not a reference field in table "+table.getName());
1158 -
1159 - String nmTable = ((ReferenceField)nmField).getTargetTable();
1160 - String idTable = idField == null ? null : ((ReferenceField)idField).getTargetTable();
1161 -
1162 - if (idTable!=null && !nmTable.equals(idTable)) throw new IllegalArgumentException(relNameField+" and "+relIdField+" in table "+table.getName()+" do not reference the same table: "+nmTable+" != "+idTable);
1163 -
1164 - String sql = "UPDATE "+table.getSQLName()+" as R JOIN "+aliasTable.getSQLName()+" as E "
1165 - + ((idField!=null) ? " ON R."+relIdField+" = E.source " : " ON R."+relNameField+" = E.source_name ")
1166 - + " SET R."+relNameField+" = E.target_name";
1167 -
1168 - if (idField!=null) sql += ", R."+relIdField+" = E.target";
1169 -
1170 - String where = scope == null ? null : " scope = "+scope.ordinal();
1171 -
1172 - return executeChunkedUpdate("resolveRedirects", table.getName()+"."+relNameField+"+"+relIdField, sql, where, aliasTable, "source", chunkFactor);
1173 - }
1174 -
11751150 /////////////////////////////////////////////////////////////////////////////////////////////
11761151 protected class DatabaseLocalStatisticsStoreBuilder extends DatabaseStatisticsStoreBuilder {
11771152 protected EntityTable termTable;
@@ -1262,13 +1237,11 @@
12631238 private DatabasePropertyStoreBuilder propertyStore;
12641239
12651240 protected DatabaseTextStoreBuilder newTextStoreBuilder() throws SQLException, PersistenceException {
1266 - TextStoreSchema schema = new TextStoreSchema(getCorpus(), getDatabaseAccess().getConnection(), tweaks, false);
1267 - return new DatabaseTextStoreBuilder(schema, tweaks);
 1241+ return new DatabaseTextStoreBuilder(this, tweaks);
12681242 }
12691243
12701244 protected DatabasePropertyStoreBuilder newPropertyStoreBuilder() throws SQLException, PersistenceException {
1271 - PropertyStoreSchema schema = new PropertyStoreSchema(getCorpus(), getDatabaseAccess().getConnection(), tweaks, false);
1272 - return new DatabasePropertyStoreBuilder(schema, tweaks);
 1245+ return new DatabasePropertyStoreBuilder(this, tweaks);
12731246 }
12741247
12751248 public TextStoreBuilder getTextStoreBuilder() throws PersistenceException {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java
@@ -7,7 +7,7 @@
88
99 public interface WikiWordStoreBuilder extends WikiWordStore {
1010
11 - public abstract void prepare(boolean purge, boolean dropAll) throws PersistenceException;
 11+ public abstract void initialize(boolean purge, boolean dropAll) throws PersistenceException;
1212
1313 public abstract void close(boolean flush) throws PersistenceException;
1414
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/FauxStoreBuilder.java
@@ -41,7 +41,7 @@
4242 flush();
4343 }
4444
45 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
 45+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
4646 //noop
4747 }
4848
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java
@@ -31,7 +31,13 @@
3232
3333 protected Inserter plainTextInserter;
3434 protected Inserter rawTextInserter;
 35+
 36+ public DatabaseTextStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException {
 37+ this(new TextStoreSchema(conceptStore.getCorpus(),
 38+ conceptStore.getDatabaseAccess().getConnection(), tweaks, true), tweaks);
 39+ }
3540
 41+
3642 /**
3743 * Constructs a DatabaseWikiStore, soring information from/about the given Corpus
3844 * into the database defined by the DatabaseConnectionInfo.
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
@@ -1,24 +1,22 @@
22 package de.brightbyte.wikiword.store.builder;
33
4 -import java.sql.Connection;
54 import java.sql.SQLException;
65
7 -import javax.sql.DataSource;
8 -
96 import de.brightbyte.db.Inserter;
107 import de.brightbyte.db.RelationTable;
118 import de.brightbyte.util.PersistenceException;
129 import de.brightbyte.wikiword.Corpus;
1310 import de.brightbyte.wikiword.TweakSet;
1411 import de.brightbyte.wikiword.schema.AliasScope;
 12+import de.brightbyte.wikiword.schema.ConceptStoreSchemas;
 13+import de.brightbyte.wikiword.schema.LocalConceptStoreSchema;
1514 import de.brightbyte.wikiword.schema.PropertyStoreSchema;
1615
1716 public class DatabasePropertyStoreBuilder extends DatabaseLocalStoreBuilder implements PropertyStoreBuilder {
1817
19 - protected DatabaseLocalConceptStoreBuilder conceptStore;
20 -
2118 protected RelationTable propertyTable;
2219 protected Inserter propertyInserter;
 20+ private LocalConceptStoreSchema conceptStoreSchema;
2321
2422 /**
2523 * Constructs a DatabaseWikiStore, soring information from/about the given Corpus
@@ -28,10 +26,11 @@
2927 * Used to determin the table names (from Corpus.getDbPrefix) and to generate URIs.
3028 * @param dbInfo database connection info, used to connect to the database
3129 * @param tweaks a tweak set from which additional options can be taken (see description at the top).
 30+ * @throws PersistenceException
3231 */
33 - public DatabasePropertyStoreBuilder(Corpus corpus, DataSource dbInfo, TweakSet tweaks) throws SQLException {
 32+ /*public DatabasePropertyStoreBuilder(Corpus corpus, DataSource dbInfo, TweakSet tweaks) throws SQLException, PersistenceException {
3433 this(new PropertyStoreSchema(corpus, dbInfo, tweaks, true), tweaks);
35 - }
 34+ }*/
3635
3736 /**
3837 * Constructs a DatabaseWikiStore, soring information from/about the given Corpus
@@ -41,12 +40,13 @@
4241 * Used to determin the table names (from Corpus.getDbPrefix) and to generate URIs.
4342 * @param db a database connection
4443 * @param tweaks a tweak set from which additional options can be taken (see description at the top).
 44+ * @throws PersistenceException
4545 */
46 - public DatabasePropertyStoreBuilder(Corpus corpus, Connection db, TweakSet tweaks) throws SQLException {
 46+ /*public DatabasePropertyStoreBuilder(Corpus corpus, Connection db, TweakSet tweaks) throws SQLException, PersistenceException {
4747 this(new PropertyStoreSchema(corpus, db, tweaks, true), tweaks);
48 - }
 48+ }*/
4949
50 - public DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException {
 50+ public DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException, PersistenceException {
5151 this(conceptStore,
5252 new PropertyStoreSchema(conceptStore.getCorpus(),
5353 conceptStore.getDatabaseAccess().getConnection(),
@@ -63,37 +63,41 @@
6464 * @param db empty DatabaseSchema, wrapping a database connection. Will be configured with the appropriate table defitions
6565 * @param tweaks a tweak set from which additional options can be taken (see description at the top).
6666 * @throws SQLException
 67+ * @throws PersistenceException
6768 */
68 - protected DatabasePropertyStoreBuilder(PropertyStoreSchema database, TweakSet tweaks) throws SQLException {
 69+ /*protected DatabasePropertyStoreBuilder(PropertyStoreSchema database, TweakSet tweaks) throws SQLException, PersistenceException {
6970 this( new DatabaseLocalConceptStoreBuilder(database.getCorpus(), database.getConnection(), tweaks),
7071 database, tweaks);
71 - }
 72+ }*/
7273
73 - protected DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, PropertyStoreSchema database, TweakSet tweaks) throws SQLException {
 74+ protected DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, PropertyStoreSchema database, TweakSet tweaks) throws SQLException, PersistenceException {
7475 super(database, tweaks);
7576
76 - this.conceptStore = conceptStore;
 77+ //this.conceptStore = conceptStore;
7778
7879 this.propertyInserter = configureTable("property", 128, 3*32);
7980 this.propertyTable = (RelationTable)propertyInserter.getTable();
 81+
 82+ this.conceptStoreSchema = new LocalConceptStoreSchema(conceptStore.getCorpus(),
 83+ conceptStore.getDatabaseAccess().getConnection(),
 84+ tweaks, true);
 85+
 86+ this.agenda = conceptStore.getAgenda();
8087 }
8188
8289 @Override
8390 protected void deleteDataFrom(int rcId, String op) throws PersistenceException {
84 - conceptStore.deleteDataFrom(rcId, op);
8591 deleteDataFrom(rcId, op, propertyTable, "concept");
8692 }
8793
8894 @Override
89 - public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
90 - super.prepare(purge, dropAll);
91 - conceptStore.prepare(purge, dropAll);
 95+ public void initialize(boolean purge, boolean dropAll) throws PersistenceException {
 96+ super.initialize(purge, dropAll);
9297 }
9398
9499 @Override
95100 public void flush() throws PersistenceException {
96101 super.flush();
97 - conceptStore.flush();
98102 }
99103
100104 protected int getConceptId(String title) throws SQLException {
@@ -142,8 +146,11 @@
143147 }
144148
145149 public void finishAliases() throws PersistenceException {
 150+
146151 if (beginTask("finishAliases", "resolveRedirects:property")) {
147 - int n = conceptStore.resolveRedirects(propertyTable, "concept_name", "concept", AliasScope.REDIRECT, 3);
 152+ RelationTable aliasTable = (RelationTable)conceptStoreSchema.getTable("alias");
 153+
 154+ int n = resolveRedirects(aliasTable, propertyTable, "concept_name", "concept", AliasScope.REDIRECT, 3);
148155 endTask("finishAliases", "resolveRedirects:property", n+" entries");
149156 }
150157 }

Status & tagging log