r66609 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r66608‎ | r66609 | r66610 >
Date:13:57, 18 May 2010
Author:daniel
Status:deferred
Tags:
Comment:
always make sure all keys are enabled
Modified paths:
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseIncrementalStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java (modified) (history)
  • /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java
——@@ -148,14 +148,10 @@
149149 @Override
150150 public void finish() throws PersistenceException {
151151 ConceptBasedStoreBuilder store = buildConcepts ? this.store : this.propertyStore;
152 - boolean resolveIdsFirst = buildConcepts ? true : false;
153152
154 - if (beginTask("PropertyImporter.finish", "finishImport")) {
155 - store.preparePostProcessing();
156 - endTask("PropertyImporter.finish", "finishImport");
157 - }
 153+ store.prepareMassProcessing(); //NOTE: always make sure the DB is ready for mass processing
158154
159 - if (resolveIdsFirst && beginTask("PropertyImporter.finish", "finishIdReferences#1")) {
 155+ if (beginTask("PropertyImporter.finish", "finishIdReferences#1")) {
160156 store.finishIdReferences();
161157 endTask("PropertyImporter.finish", "finishIdReferences#1");
162158 }
——@@ -165,7 +161,7 @@
166162 endTask("PropertyImporter.finish", "finishAliases");
167163 }
168164
169 - if (!resolveIdsFirst && beginTask("PropertyImporter.finish", "finishIdReferences#2")) {
 165+ if (beginTask("PropertyImporter.finish", "finishIdReferences#2")) { //NOTE: resolve IDs again, some may only be known know, after resolving aliases.
170166 store.finishIdReferences();
171167 endTask("PropertyImporter.finish", "finishIdReferences#2");
172168 }
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
——@@ -47,10 +47,7 @@
4848
4949 @Override
5050 public void finish() throws PersistenceException {
51 - if (beginTask("ConceptImporter.finish", "finishImport")) {
52 - store.preparePostProcessing();
53 - endTask("ConceptImporter.finish", "finishImport");
54 - }
 51+ store.prepareMassProcessing(); //NOTE: always make sure the DB is ready for mass processing
5552
5653 if (beginTask("ConceptImporter.finish", "finishBadLinks")) {
5754 store.finishBadLinks();
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java
——@@ -867,11 +867,6 @@
868868 return dataset;
869869 }
870870
871 - public void preparePostProcessing() throws PersistenceException {
872 - log("* preparePostProcessing *");
873 - }
874 -
875 -
876871 public void prepareMassInsert() throws PersistenceException {
877872 log("* prepareMassInsert *");
878873 }
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
——@@ -517,7 +517,7 @@
518518 }
519519 }
520520
521 - protected int resolveRedirects(RelationTable aliasTable, DatabaseTable table, String relNameField, String relIdField, AliasScope scope, int chunkFactor, String forceRIndex, String forceEIndex) throws PersistenceException {
 521+ protected int resolveRedirects(RelationTable aliasTable, DatabaseTable table, String relNameField, String relIdField, AliasScope scope, int chunkFactor, String where, String forceRIndex, String forceEIndex) throws PersistenceException {
522522 if (relIdField==null && relNameField==null) throw new IllegalArgumentException("relNameFields and relIdField can't both be null");
523523
524524 if (forceRIndex==null) {
——@@ -558,8 +558,13 @@
559559 if (nmField!=null && idField!=null) sql += ", ";
560560 if (idField!=null) sql += " R."+relIdField+" = E.target";
561561
562 - String where = scope == null ? null : " scope = "+scope.ordinal();
 562+ String w = scope == null ? null : " scope = "+scope.ordinal();
563563
 564+ if (w!=null) {
 565+ if (where == null) where = w;
 566+ else where = "("+where+") AND ("+w+")";
 567+ }
 568+
564569 return executeChunkedUpdate("resolveRedirects", table.getName()+"."+relNameField+"+"+relIdField, sql, where, aliasTable, "source", chunkFactor);
565570 }
566571
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseIncrementalStoreBuilder.java
——@@ -55,16 +55,21 @@
5656 Runtime.getRuntime().gc(); //run garbage collection
5757 }
5858
59 - public void preparePostProcessing() throws PersistenceException {
 59+ public void prepareMassInsert() throws PersistenceException {
6060 try {
61 - flush();
62 - if (beginTask("DatabaseLocalConceptStore.preparePostProcessing", "enableKeys")) {
 61+ database.disableKeys();
 62+ } catch (SQLException e) {
 63+ throw new PersistenceException(e);
 64+ }
 65+ }
 66+
 67+ public void prepareMassProcessing() throws PersistenceException {
 68+ try {
 69+ flush();
6370 database.enableKeys();
64 - endTask("DatabaseLocalConceptStore.preparePostProcessing", "enableKeys");
65 - }
6671 } catch (SQLException e) {
6772 throw new PersistenceException(e);
68 - }
 73+ }
6974 }
7075
7176 }
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
——@@ -632,6 +632,7 @@
633633 }
634634
635635 public void prepareMassProcessing() throws PersistenceException {
 636+ this.flush();
636637 this.enableKeys();
637638
638639 if (propertyStore!=null) {
——@@ -674,21 +675,6 @@
675676 }
676677 }
677678
678 - public void preparePostProcessing() throws PersistenceException {
679 - flush();
680 - enableKeys();
681 -
682 - if (propertyStore!=null && beginTask("preparePostProcessing", "propertyStore.preparePostProcessing")) {
683 - propertyStore.preparePostProcessing();
684 - endTask("preparePostProcessing", "propertyStore.preparePostProcessing");
685 - }
686 -
687 - if (textStore!=null && beginTask("preparePostProcessing", "textStore.preparePostProcessing")) {
688 - textStore.preparePostProcessing();
689 - endTask("preparePostProcessing", "textStore.preparePostProcessing");
690 - }
691 - }
692 -
693679 public void finishSections() throws PersistenceException {
694680 if (beginTask("finishSections", "buildSectionConcepts")) {
695681 int n = buildSectionConcepts();
——@@ -841,7 +827,7 @@
842828 //NOTE: need to resolve category-aliases here, so no concepts are generated for aliased categories!
843829 //NOTE: bad category redirs have been droped in finishBadLinks
844830 if (beginTask("finishMissingConcpets", "resolveRedirects:broader")) {
845 - int n = resolveRedirects(aliasTable, broaderTable, "broad_name", idManager==null ? null : "broad", AliasScope.CATEGORY, 1, idManager==null ? "broad_name" : "broad_narrow", null);
 831+ int n = resolveRedirects(aliasTable, broaderTable, "broad_name", idManager==null ? null : "broad", AliasScope.CATEGORY, 1, null, idManager==null ? "broad_name" : "broad_narrow", null);
846832 endTask("finishMissingConcpets", "resolveRedirects:broader", n+" entries");
847833 }
848834
——@@ -939,24 +925,24 @@
940926 if (beginTask("finishAliases", "resolveRedirects:link")) {
941927 //XXX: SLOW!
942928 //TODO: smaller chunks? chunk on target table, not alias table? force index?
943 - int n = resolveRedirects(aliasTable, linkTable, "target_name", "target", AliasScope.REDIRECT, 8, "target_anchor", null);
 929+ int n = resolveRedirects(aliasTable, linkTable, "target_name", "target", AliasScope.REDIRECT, 8, null, "target_anchor", null);
944930 endTask("finishAliases", "resolveRedirects:link", n+" entries");
945931 }
946932
947933 //NOTE: broader.broad_name already done in finishMissingConcepts for AliasScope.BROADER
948934
949935 if (beginTask("finishAliases", "resolveRedirects:about")) {
950 - int n = resolveRedirects(aliasTable, aboutTable, "concept_name", "concept", null, 1, null, null);
 936+ int n = resolveRedirects(aliasTable, aboutTable, "concept_name", "concept", null, 1, null, null, null);
951937 endTask("finishAliases", "resolveRedirects:about", n+" entries");
952938 }
953939
954940 if (beginTask("finishAliases", "resolveRedirects:narrow")) {
955 - int n = resolveRedirects(aliasTable, broaderTable, "narrow_name", "narrow", null, 1, null, null);
 941+ int n = resolveRedirects(aliasTable, broaderTable, "narrow_name", "narrow", null, 1, null, null, null);
956942 endTask("finishAliases", "resolveRedirects:narrow", n+" entries");
957943 }
958944
959945 if (beginTask("finishAliases", "resolveRedirects:broad")) {
960 - int n = resolveRedirects(aliasTable, broaderTable, "broad_name", "broad", null, 1, null, null);
 946+ int n = resolveRedirects(aliasTable, broaderTable, "broad_name", "broad", null, 1, null, null, null);
961947 endTask("finishAliases", "resolveRedirects:broad", n+" entries");
962948 }
963949
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java
——@@ -4,8 +4,6 @@
55
66 public interface IncrementalStoreBuilder extends WikiWordStoreBuilder {
77
8 - public void preparePostProcessing() throws PersistenceException;
9 -
108 public void deleteDataAfter(int delAfter, boolean inclusive) throws PersistenceException;
119
1210 public void prepareMassProcessing() throws PersistenceException;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
——@@ -110,35 +110,66 @@
111111 return (Corpus)database.getDataset();
112112 }
113113
114 - public void finishAliases() throws PersistenceException {
115 - if (beginTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property")) {
116 - RelationTable aliasTable = (RelationTable)conceptStoreSchema.getTable("alias");
117 - int n = resolveRedirects(aliasTable, propertyTable, "concept_name", idManager!=null ? "concept" : null, AliasScope.REDIRECT, 3, null, null);
118 - endTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property", n+" entries");
 114+ protected boolean hasUnresolvedConceptReferences() throws PersistenceException{
 115+ try {
 116+ Number c = (Number)database.executeSingleValueQuery("DatabasePropertyStoreBuilder.finishAliases#hasNull?", "select exists(select * from wmde10apr_en_property where concept is null)");
 117+ return c.intValue() > 0;
 118+ } catch (SQLException e) {
 119+ throw new PersistenceException(e);
119120 }
120121 }
121 -
122 - public void finishIdReferences() throws PersistenceException {
123 - if (idManager==null && beginTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property")) {
124 - int n = buildIdLinks(propertyTable, "concept_name", "concept", 1);
125 - endTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property", n+" references");
126 - }
127 - }
128122
129 - public void prepareMassInsert() throws PersistenceException {
 123+ protected boolean hasResolvedConceptReferences() throws PersistenceException {
130124 try {
131 - database.disableKeys();
 125+ Number c = (Number)database.executeSingleValueQuery("DatabasePropertyStoreBuilder.finishAliases#hasNull?", "select exists(select * from wmde10apr_en_property where concept is not null)");
 126+ return c.intValue() > 0;
132127 } catch (SQLException e) {
133128 throw new PersistenceException(e);
134129 }
135130 }
136131
 132+ public void finishAliases() throws PersistenceException {
 133+ RelationTable aliasTable = (RelationTable)conceptStoreSchema.getTable("alias");
 134+
 135+ if (beginTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property#id")) {
 136+ int n = 0;
 137+ if (hasResolvedConceptReferences()) {
 138+ n = resolveRedirects(aliasTable, propertyTable, "concept_name", "concept", AliasScope.REDIRECT, 3, null, null, null);
 139+ }
 140+
 141+ endTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property#id", n+" entries");
 142+ }
 143+
 144+ if (beginTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property#name")) {
 145+ int n = 0;
 146+ if (hasUnresolvedConceptReferences()) {
 147+ n = resolveRedirects(aliasTable, propertyTable, "concept_name", null, AliasScope.REDIRECT, 3, propertyTable.getSQLName()+".concept is null", null, null);
 148+ }
 149+
 150+ endTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property#name", n+" entries");
 151+ }
 152+ }
 153+
 154+ public void finishIdReferences() throws PersistenceException {
 155+ if (beginTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property")) {
 156+ int n = 0;
 157+ if (hasUnresolvedConceptReferences()) {
 158+ n = buildIdLinks(propertyTable, "concept_name", "concept", 1);
 159+ }
 160+
 161+ endTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property", n+" references");
 162+ }
 163+ }
 164+
137165 public void prepareMassProcessing() throws PersistenceException {
138166 try {
139 - database.enableKeys();
 167+ this.conceptStoreSchema.enableKeys();
140168 } catch (SQLException e) {
141169 throw new PersistenceException(e);
142170 }
 171+
 172+ super.prepareMassProcessing();
143173 }
144174
 175+
145176 }
\ No newline at end of file

Status & tagging log