r46640 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r46639‎ \| r46640 \| r46641 >
Date:	12:40, 31 January 2009
Author:	daniel
Status:	deferred
Tags:
Comment:	handle resource <-> concept link as n:m relation
Modified paths:	/trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/schema/LocalConceptStoreSchema.java (modified) (history) /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/schema/WikiWordConceptStoreSchema.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/AbstractImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseGlobalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalStoreBuilder.java (added) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java (added) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/LocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/PropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/schema/LocalConceptStoreSchema.java
—	—	@@ -23,6 +23,7 @@
24	24
25	25	protected RelationTable meaningTable;
26	26	protected RelationTable aliasTable;
	27	+ protected RelationTable aboutTable;
27	28
28	29	//protected EntityTable conceptDescriptionTable;
29	30
—	—	@@ -41,8 +42,6 @@
42	43	private void init(Corpus corpus, TweakSet tweaks) {
43	44	this.corpus = corpus;
44	45
45		~~- conceptTable.addField( new ReferenceField(this, "resource", "INT", null, false, KeyType.INDEX, "resource", "id", null ) );~~
46		-
47	46	broaderTable.addField( new ReferenceField(this, "resource", "INT", null, false, KeyType.INDEX, "resource", "id", null ) ); //NOTE: not required. see buildSectionBroader.
48	47	broaderTable.addField( new DatabaseField(this, "rule", "INT", null, true, KeyType.INDEX) );
49	48	broaderTable.addField( new ReferenceField(this, "narrow_name", getTextType(255), null, true, KeyType.INDEX, "concept", "name", null ) );
—	—	@@ -69,6 +68,7 @@
70	69	resourceTable.addField( new DatabaseField(this, "name", getTextType(255), null, true, KeyType.UNIQUE ) );
71	70	resourceTable.addField( new DatabaseField(this, "type", "INT", null, true, KeyType.INDEX ) ); //TODO: enum
72	71	resourceTable.addField( new DatabaseField(this, "timestamp", "CHAR(14)", null, true, null ) ); //TODO: which type in which db?
	72	+
73	73	resourceTable.setAutomaticField("id");
74	74	addTable(resourceTable);
75	75
—	—	@@ -100,6 +100,14 @@
101	101	//aliasTable.addKey( new DatabaseKey(this, KeyType.UNIQUE, "ident", new String[] {"resource", "concept_name", "term_text"}) );
102	102	addTable(aliasTable);
103	103
	104	+ aboutTable = new RelationTable(this, "about", defaultTableAttributes);
	105	+ //aliasTable.addField( new DatabaseField(this, "id", "INT", "AUTO_INCREMENT", false, KeyType.PRIMARY) );
	106	+ aboutTable.addField( new ReferenceField(this, "resource", "INT", null, true, null, "resource", "id", null ) );
	107	+ aboutTable.addField( new ReferenceField(this, "concept", "INT", null, false, KeyType.INDEX, "concept", "id", null ) );
	108	+ aboutTable.addField( new ReferenceField(this, "concept_name", getTextType(255), null, true, KeyType.INDEX, "concept", "name", null ) );
	109	+ aboutTable.addKey( new DatabaseKey(this, KeyType.PRIMARY, "about", new String[] {"resource", "concept"}) );
	110	+ addTable(aliasTable);
	111	+
104	112	meaningTable = new RelationTable(this, "meaning", defaultTableAttributes);
105	113	//meaningTable.addField( new DatabaseField(this, "id", "INT", "AUTO_INCREMENT", false, KeyType.PRIMARY) );
106	114	meaningTable.addField( new ReferenceField(this, "concept", "INT", null, true, KeyType.INDEX, "concept", "id", null ) );
—	—	@@ -133,6 +141,8 @@
134	142	public void checkConsistency() throws SQLException {
135	143	super.checkConsistency();
136	144
	145	+ checkReferentialIntegrity(conceptTable, "resource", true); //NOTE: red links generate concepts with no resource assigned
	146	+
137	147	checkReferentialIntegrity(meaningTable, "concept", false);
138	148
139	149	checkIdSanity(resourceTable, "id");
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/schema/WikiWordConceptStoreSchema.java
—	—	@@ -82,8 +82,6 @@
83	83	public void checkConsistency() throws SQLException {
84	84	checkIdSanity(conceptTable, "id"); //FIXME: this barfs spuriously. something insconsistent about th db state?!
85	85
86		~~- checkReferentialIntegrity(conceptTable, "resource", true); //NOTE: red links generate concepts with no resource assigned~~
87		-
88	86	checkReferentialIntegrity(broaderTable, "narrow", false);
89	87	checkReferentialIntegrity(broaderTable, "broad", false);
90	88
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/AbstractImporter.java
—	—	@@ -19,6 +19,7 @@
20	20	import de.brightbyte.wikiword.TweakSet;
21	21	import de.brightbyte.wikiword.analyzer.WikiTextAnalyzer;
22	22	import de.brightbyte.wikiword.analyzer.WikiTextSniffer;
	23	+import de.brightbyte.wikiword.store.builder.IncrementalStoreBuilder;
23	24	import de.brightbyte.wikiword.store.builder.WikiWordStoreBuilder;
24	25
25	26	public abstract class AbstractImporter implements WikiWordImporter {
—	—	@@ -168,12 +169,12 @@
169	170	doit = false;
170	171	}
171	172	else {
172		~~- if (getAgenda().isTaskDirty()) {~~
	173	+ if (getAgenda().isTaskDirty() && store instanceof IncrementalStoreBuilder) {
173	174	Agenda.Record rec = getAgenda().getCurrentRecord();
174	175
175	176	int delAfter = (Integer)rec.parameters.get("lastRcId_");
176	177	out.info("=== DIRTY BLOCK FOR SAFEPOINT#"+safepointNumber+", Deleting entries starting after id: #"+delAfter+" ===");
177		~~- store.deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!~~
	178	+ ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!
178	179	}
179	180
180	181	out.info("=== BEGINNING BLOCK FOR SAFEPOINT#"+safepointNumber+": "+getAgenda().getCurrentRecord().parameters+" ===");
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java
—	—	@@ -108,103 +108,71 @@
109	109	log("* finishAliases *");
110	110	}
111	111
112		~~- public int storeConcept(int rcId, String name, ConceptType ctype) throws PersistenceException {~~
113		~~- // TODO Auto-generated method stub~~
114		~~- return 0;~~
115		~~- }~~
116		-
117		~~- public void storeConceptAlias(int rcId, int source, String sourceName, int target, String targetName, AliasScope scope) throws PersistenceException {~~
118		~~- // TODO Auto-generated method stub~~
119		-
120		~~- }~~
121		-
122	112	public void storeProperty(int resourceId, int conceptId, String concept, String property, String value) throws PersistenceException {
123		~~- // TODO Auto-generated method stub~~
124		-
	113	+ log("* storeProperty("+resourceId+", "+conceptId+", "+concept+", "+property+", "+value+") *");
125	114	}
126	115
127		~~- public int storeResource(String name, ResourceType ptype, Date time) throws PersistenceException {~~
128		~~- // TODO Auto-generated method stub~~
129		~~- return 0;~~
130		~~- }~~
131		-
132	116	public void checkConsistency() throws PersistenceException {
133		~~- // TODO Auto-generated method stub~~
134		-
	117	+ log("* checkConsistency *");
135	118	}
136	119
137	120	public void close(boolean flush) throws PersistenceException {
138		~~- // TODO Auto-generated method stub~~
139		-
	121	+ log("* close *");
140	122	}
141	123
142	124	public void deleteDataAfter(int lastId, boolean inclusive) throws PersistenceException {
143		~~- // TODO Auto-generated method stub~~
144		-
	125	+ log("* deleteDataAfter("+lastId+", "+inclusive+") *");
145	126	}
146	127
147	128	public void deleteDataFrom(int lastId) throws PersistenceException {
148		~~- // TODO Auto-generated method stub~~
149		-
	129	+ log("* deleteDataFrom("+lastId+") *");
150	130	}
151	131
152	132	public void dumpTableStats(Output out) throws PersistenceException {
153		~~- // TODO Auto-generated method stub~~
154		-
	133	+ log("* dumpTableStats *");
155	134	}
156	135
157	136	public void flush() throws PersistenceException {
158		~~- // TODO Auto-generated method stub~~
159		-
	137	+ log("* flush *");
160	138	}
161	139
162	140	public Agenda getAgenda() throws PersistenceException {
163		~~- // TODO Auto-generated method stub~~
164	141	return null;
165	142	}
166	143
167	144	public int getNumberOfWarnings() throws PersistenceException {
168		~~- // TODO Auto-generated method stub~~
169	145	return 0;
170	146	}
171	147
172	148	public void open() throws PersistenceException {
173		~~- // TODO Auto-generated method stub~~
174		-
	149	+ log("* open *");
175	150	}
176	151
177	152	public void optimize() throws PersistenceException {
178		~~- // TODO Auto-generated method stub~~
179		-
	153	+ log("* optimize *");
180	154	}
181	155
182	156	public void prepare(boolean purge, boolean dropAll) throws PersistenceException {
183		~~- // TODO Auto-generated method stub~~
184		-
	157	+ log("* prepare *");
185	158	}
186	159
187	160	public void setLogLevel(int loglevel) {
188		~~- // TODO Auto-generated method stub~~
189		-
	161	+ // noop
190	162	}
191	163
192	164	public void storeWarning(int rcId, String problem, String details) throws PersistenceException {
193		~~- // TODO Auto-generated method stub~~
194		-
	165	+ log("+ warning: rcId = "+rcId+", problem = "+problem+", details = "+details);
195	166	}
196	167
197	168	public Map<String, ? extends Number> getTableStats() throws PersistenceException {
198		~~- // TODO Auto-generated method stub~~
199	169	return null;
200	170	}
201	171
202	172	public boolean isComplete() throws PersistenceException {
203		~~- // TODO Auto-generated method stub~~
204	173	return false;
205	174	}
206	175
207	176	public Corpus getCorpus() {
208		~~- // TODO Auto-generated method stub~~
209	177	return null;
210	178	}
211	179
—	—	@@ -459,7 +427,13 @@
460	428	return resourceCounter;
461	429	}
462	430
	431	+ public int storeResourceAbout(String name, ResourceType ptype, Date time, int conceptId, String conceptName) {
	432	+ int resourceId = storeResource(name, ptype, time);
	433	+ storeAbout(resourceId, conceptId, conceptName);
	434	+ return resourceId;
	435	+ }
463	436
	437	+
464	438	public void storeDefinition(int rcId, int conceptId, String definition) {
465	439	definitionCounter++;
466	440	trace("+ storeDefinition: conceptId = "+conceptId+": "+definition);
—	—	@@ -653,5 +627,13 @@
654	628	//noop
655	629	return 0;
656	630	}
	631	+
	632	+ public void storeAbout(int resource, String conceptName) {
	633	+ trace("+ storeAbout: resource = "+resource+", conceptName = "+conceptName);
	634	+ }
	635	+
	636	+ public void storeAbout(int resource, int concept, String conceptName) {
	637	+ trace("+ storeAbout: resource = "+resource+", concept = "+concept+", conceptName = "+conceptName);
	638	+ }
657	639
658	640	}
\ No newline at end of file
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalStoreBuilder.java
—	—	@@ -0,0 +1,29 @@
	2	+package de.brightbyte.wikiword.store.builder;
	3	+
	4	+import java.sql.SQLException;
	5	+
	6	+import de.brightbyte.util.PersistenceException;
	7	+import de.brightbyte.wikiword.TweakSet;
	8	+import de.brightbyte.wikiword.schema.WikiWordStoreSchema;
	9	+
	10	+public abstract class DatabaseLocalStoreBuilder extends DatabaseWikiWordStoreBuilder implements IncrementalStoreBuilder {
	11	+
	12	+ public DatabaseLocalStoreBuilder(WikiWordStoreSchema database, TweakSet tweaks) throws SQLException {
	13	+ super(database, tweaks);
	14	+ }
	15	+
	16	+ protected abstract void deleteDataFrom(int rcId, String op) throws PersistenceException;
	17	+
	18	+
	19	+ public void deleteDataFrom(int rcId) throws PersistenceException {
	20	+ log("deleting data from "+rcId);
	21	+ deleteDataFrom(rcId, "=");
	22	+ }
	23	+
	24	+ public void deleteDataAfter(int rcId, boolean inclusive) throws PersistenceException {
	25	+ String op = inclusive ? ">=" : ">";
	26	+ log("deleting data from with id "+op+" "+rcId);
	27	+ deleteDataFrom(rcId, op);
	28	+ }
	29	+
	30	+}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/PropertyStoreBuilder.java
—	—	@@ -1,11 +1,6 @@
2	2	package de.brightbyte.wikiword.store.builder;
3	3
4		~~-import java.util.Date;~~
5		-
6	4	import de.brightbyte.util.PersistenceException;
7		~~-import de.brightbyte.wikiword.ConceptType;~~
8		~~-import de.brightbyte.wikiword.ResourceType;~~
9		~~-import de.brightbyte.wikiword.schema.AliasScope;~~
10	5	import de.brightbyte.wikiword.store.WikiWordLocalStore;
11	6
12	7	public interface PropertyStoreBuilder extends WikiWordStoreBuilder, WikiWordLocalStore {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
—	—	@@ -10,9 +10,11 @@
11	11	import de.brightbyte.db.BufferBasedInserter;
12	12	import de.brightbyte.db.DatabaseAccess;
13	13	import de.brightbyte.db.DatabaseAgendaPersistor;
	14	+import de.brightbyte.db.DatabaseField;
14	15	import de.brightbyte.db.DatabaseTable;
15	16	import de.brightbyte.db.Inserter;
16	17	import de.brightbyte.db.InserterFactory;
	18	+import de.brightbyte.db.ReferenceField;
17	19	import de.brightbyte.db.RelationTable;
18	20	import de.brightbyte.db.StatementBasedInserter;
19	21	import de.brightbyte.util.PersistenceException;
—	—	@@ -260,6 +262,23 @@
261	263	trace("deleted "+c+" rows from "+rel.getName()+" where "+field+" "+op+" "+rcId+(via!=null?" via "+via.getName():"")+", took "+(System.currentTimeMillis()-t)/1000+" sec");
262	264	}
263	265
	266	+ protected void deleteOrphansFrom(int rcId, String op, DatabaseTable table, RelationTable ref, String refField) throws PersistenceException {
	267	+ String sql;
	268	+
	269	+ ReferenceField f = (ReferenceField)ref.getField(refField);
	270	+ String field = f.getTargetField();
	271	+
	272	+ sql = "DELETE FROM " + table.getSQLName() + " AS T ";
	273	+ sql += " WHERE NOT EXISTS ( ";
	274	+ sql += " SELECT * FROM " + ref.getSQLName() + " AS R ";
	275	+ sql += " WHERE R." + refField + " = T." + field;
	276	+ sql += " )";
	277	+
	278	+ long t = System.currentTimeMillis();
	279	+ int c = executeUpdate("deleteDataFrom", sql);
	280	+ trace("deleted "+c+" orphan rows from "+table.getName()+" where no reference exists from "+ref.getSQLName()+"."+refField+", took "+(System.currentTimeMillis()-t)/1000+" sec");
	281	+ }
	282	+
264	283	protected int executeUpdate(String name, String sql) throws PersistenceException {
265	284	try {
266	285	return database.executeUpdate(name, sql);
—	—	@@ -311,28 +330,12 @@
312	331	}
313	332	}
314	333
315		~~- protected void deleteDataFrom(int rcId, String op) throws PersistenceException {~~
316		~~- throw new UnsupportedOperationException();~~
317		~~- }~~
318		-
319		-
320	334	protected int deleteLoops(DatabaseTable t, String id1, String id2) throws SQLException {
321	335	String sql = "DELETE FROM "+t.getSQLName()+" WHERE "+id1+" = "+id2;
322	336
323	337	return database.executeUpdate("deleteLoops", sql);
324	338	}
325		-
326		~~- public void deleteDataFrom(int rcId) throws PersistenceException {~~
327		~~- log("deleting data from "+rcId);~~
328		~~- deleteDataFrom(rcId, "=");~~
329		~~- }~~
330	339
331		~~- public void deleteDataAfter(int rcId, boolean inclusive) throws PersistenceException {~~
332		~~- String op = inclusive ? ">=" : ">";~~
333		~~- log("deleting data from with id "+op+" "+rcId);~~
334		~~- deleteDataFrom(rcId, op);~~
335		~~- }~~
336		-
337	340	/*
338	341	public void purge() throws PersistenceException {
339	342	dropTables(true);
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
—	—	@@ -70,6 +70,9 @@
71	71	protected Inserter aliasInserter;
72	72	protected Inserter meaningInserter;
73	73
	74	+ protected Inserter aboutInserter;
	75	+ protected RelationTable aboutTable;
	76	+
74	77	protected Random random;
75	78
76	79	protected TweakSet tweaks;
—	—	@@ -133,6 +136,9 @@
134	137	aliasTable = (RelationTable)aliasInserter.getTable();
135	138	meaningTable = (RelationTable)meaningInserter.getTable();
136	139
	140	+ aboutInserter = configureTable("about", 1024, 64);
	141	+ aboutTable = (RelationTable)aboutInserter.getTable();
	142	+
137	143	long seed = tweaks.getTweak("dbstore.randomSeed", -1); //TODO: doc
138	144	if (seed>0) random = new Random(seed);
139	145	else random = new Random();
—	—	@@ -187,13 +193,17 @@
188	194	}
189	195
190	196
191		~~- @Override~~
192	197	protected void deleteDataFrom(int rcId, String op) throws PersistenceException {
193	198	deleteDataFrom(rcId, op, definitionTable, "concept", conceptTable, "resource");
194	199	//deleteDataFrom(rcId, op, conceptDescriptionTable, "concept", conceptTable, "resource");
195	200
196		~~- super.deleteDataFrom(rcId, op);~~
	201	+ deleteDataFrom(rcId, op, linkTable, "resource");
	202	+ deleteDataFrom(rcId, op, langlinkTable, "resource");
	203	+ deleteDataFrom(rcId, op, broaderTable, "resource");
197	204
	205	+ deleteDataFrom(rcId, op, aboutTable, "resource");
	206	+ deleteOrphansFrom(rcId, op, conceptTable, aboutTable, "resource");
	207	+
198	208	deleteDataFrom(rcId, op, aliasTable, "resource");
199	209	deleteDataFrom(rcId, op, sectionTable, "resource");
200	210	deleteDataFrom(rcId, op, resourceTable, "id");
—	—	@@ -245,6 +255,15 @@
246	256
247	257
248	258	/**
	259	+ * @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeResourceAbout(java.lang.String, de.brightbyte.wikiword.ResourceType, java.util.Date, int conceptId, String conceptName)
	260	+ */
	261	+ public int storeResourceAbout(String name, ResourceType ptype, Date time, int conceptId, String conceptName) throws PersistenceException {
	262	+ int rcId = storeResource(name, ptype, time);
	263	+ storeAbout(rcId, conceptId, conceptName);
	264	+ return rcId;
	265	+ }
	266	+
	267	+ /**
249	268	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeResource(java.lang.String, de.brightbyte.wikiword.ResourceType, java.util.Date)
250	269	*/
251	270	public int storeResource(String name, ResourceType ptype, Date time) throws PersistenceException {
—	—	@@ -256,7 +275,9 @@
257	276	resourceInserter.updateString("timestamp", timestampFormatter.format(time));
258	277	resourceInserter.updateRow();
259	278
260		~~- return resourceInserter.getLastId();~~
	279	+ int rcId = resourceInserter.getLastId();
	280	+
	281	+ return rcId;
261	282	} catch (SQLException e) {
262	283	throw new PersistenceException(e);
263	284	}
—	—	@@ -278,7 +299,6 @@
279	300	}
280	301
281	302	conceptInserter.updateDouble("random", random.nextDouble());
282		~~- if (rcId>=0) conceptInserter.updateInt("resource", rcId);~~
283	303	conceptInserter.updateString("name", name);
284	304	conceptInserter.updateInt("type", ctype.getCode());
285	305	conceptInserter.updateRow();
—	—	@@ -287,6 +307,8 @@
288	308	id = conceptInserter.getLastId();
289	309	}
290	310
	311	+ if (rcId>=0) storeAbout(rcId, id, name);
	312	+
291	313	return id;
292	314	} catch (SQLException e) {
293	315	throw new PersistenceException(e);
—	—	@@ -412,6 +434,33 @@
413	435	}
414	436
415	437	/**
	438	+ * @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeAbout(int, String)
	439	+ */
	440	+ public void storeAbout(int rcId, String conceptName) throws PersistenceException {
	441	+ storeAbout(rcId, -1, conceptName);
	442	+ }
	443	+
	444	+ /**
	445	+ * @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeAbout(int, int, String)
	446	+ */
	447	+ public void storeAbout(int rcId, int concept, String conceptName) throws PersistenceException {
	448	+ try {
	449	+ if (rcId<0) throw new IllegalArgumentException("bad resource id "+rcId);
	450	+ conceptName = checkName(rcId, conceptName, "concept name (resource #{0})", rcId);
	451	+
	452	+ aboutInserter.updateInt("resource", rcId);
	453	+ aboutInserter.updateString("concept_name", conceptName);
	454	+
	455	+ if (concept>0) aboutInserter.updateInt("concept", concept);
	456	+ else if (idManager!=null) aboutInserter.updateInt("concept", idManager.aquireId(conceptName));
	457	+
	458	+ aliasInserter.updateRow();
	459	+ } catch (SQLException e) {
	460	+ throw new PersistenceException(e);
	461	+ }
	462	+ }
	463	+
	464	+ /**
416	465	* @see de.brightbyte.wikiword.builder.WikiStoreBuilder#storeConceptReference(int, int, java.lang.String, java.lang.String)
417	466	*/
418	467	/*public void storeConceptReference(int rcId, int source, String sourceName, String target) throws PersistenceException {
—	—	@@ -606,6 +655,11 @@
607	656	}
608	657
609	658	public void finishMissingConcepts() throws PersistenceException {
	659	+ if (beginTask("finishMissingConcpets", "buildMissingConcepts:about")) {
	660	+ int n = buildMissingConcepts(aboutTable, "concept", "concept_name");
	661	+ endTask("finishMissingConcpets", "buildMissingConcepts:about", n+" concepts");
	662	+ }
	663	+
610	664	if (beginTask("finishMissingConcpets", "buildMissingConcepts:link")) {
611	665	int n = buildMissingConcepts(linkTable, "target", "target_name");
612	666	endTask("finishMissingConcpets", "buildMissingConcepts:link", n+" concepts");
—	—	@@ -655,10 +709,16 @@
656	710	}
657	711
658	712	public void finishIdReferences() throws PersistenceException {
	713	+ if (idManager==null && beginTask("finishIdReferences", "buildIdLinks:about")) {
	714	+ int n = buildIdLinks(aboutTable, "concept_name", "concept", 1);
	715	+ endTask("finishIdReferences", "buildIdLinks:about", n+" references");
	716	+ }
	717	+
659	718	//XXX: if (beginTask("finish.buildIdLinks:link.term_text")) buildIdLinks(useTable, "term_text", "term");
660	719	//NOTE: don't need this, anchor-id is only null if anchor_name is null too. //XXX: really?! if (beginTask("finish.buildIdLinks:link.anchor_name")) buildIdLinks(linkTable, "anchor_name", "anchor"); //Uses index _use.target (and unique key _concept.name)
	720	+
661	721	if (idManager==null && beginTask("finishIdReferences", "buildIdLinks:link.target")) {
662		~~- int n = buildIdLinks(linkTable, "target_name", "target", 3);~~
	722	+ int n = buildIdLinks(linkTable, "target_name", "target", 5);
663	723	endTask("finishIdReferences", "buildIdLinks:link.target", n+" references");
664	724	}
665	725	if (idManager==null && beginTask("finishIdReferences", "buildIdLinks:broader")) {
—	—	@@ -670,7 +730,7 @@
671	731	endTask("finishIdReferences", "buildIdLinks:narrower", n+" references");
672	732	}
673	733	if (idManager==null && beginTask("finishIdReferences", "buildIdLinks:alias")) {
674		~~- int n = buildIdLinks(aliasTable, "target_name", "target", 1);~~
	734	+ int n = buildIdLinks(aliasTable, "target_name", "target", -5);
675	735	endTask("finishIdReferences", "buildIdLinks:alias", n+" references");
676	736	}
677	737	//if (beginTask("finishIdReferences", "buildIdLinks:reference")) buildIdLinks(referenceTable, "target_name", "target");
—	—	@@ -678,12 +738,19 @@
679	739
680	740	public void finishAliases() throws PersistenceException {
681	741	if (beginTask("finishAliases", "resolveRedirects:link")) {
682		~~- int n = resolveRedirects(linkTable, "target_name", "target", AliasScope.REDIRECT, 3);~~
	742	+ //XXX: SLOW!
	743	+ //TODO: smaller chunks? chunk on target table, not alias table? force index?
	744	+ int n = resolveRedirects(linkTable, "target_name", "target", AliasScope.REDIRECT, 8);
683	745	endTask("finishAliases", "resolveRedirects:link", n+" entries");
684	746	}
685	747
686	748	//NOTE: broader.broad_name already done in finishMissingConcepts for AliasScope.BROADER
687	749
	750	+ if (beginTask("finishAliases", "resolveRedirects:about")) {
	751	+ int n = resolveRedirects(aboutTable, null, "concept", null, 1);
	752	+ endTask("finishAliases", "resolveRedirects:about", n+" entries");
	753	+ }
	754	+
688	755	if (beginTask("finishAliases", "resolveRedirects:narrow")) {
689	756	int n = resolveRedirects(broaderTable, "narrow_name", "narrow", null, 1);
690	757	endTask("finishAliases", "resolveRedirects:narrow", n+" entries");
—	—	@@ -1297,4 +1364,16 @@
1298	1365	c = executeUpdate("resetTermsForUnknownConcepts", sql); //XXX: chunk?!
1299	1366	log("deleted "+c+" entries for unknown concepts from broader table");
1300	1367	}
	1368	+
	1369	+
	1370	+ public void deleteDataFrom(int rcId) throws PersistenceException {
	1371	+ log("deleting data from "+rcId);
	1372	+ deleteDataFrom(rcId, "=");
	1373	+ }
	1374	+
	1375	+ public void deleteDataAfter(int rcId, boolean inclusive) throws PersistenceException {
	1376	+ String op = inclusive ? ">=" : ">";
	1377	+ log("deleting data from with id "+op+" "+rcId);
	1378	+ deleteDataFrom(rcId, op);
	1379	+ }
1301	1380	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/WikiWordStoreBuilder.java
—	—	@@ -33,10 +33,6 @@
34	34
35	35	//public abstract void clearStatistics() throws PersistenceException;
36	36
37		~~- public abstract void deleteDataAfter(int lastId, boolean inclusive) throws PersistenceException;~~
38		-
39		~~- public abstract void deleteDataFrom(int lastId) throws PersistenceException;~~
40		-
41	37	public abstract void checkConsistency() throws PersistenceException;
42	38
43	39	public abstract void dumpTableStats(Output out) throws PersistenceException;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordConceptStoreBuilder.java
—	—	@@ -76,14 +76,6 @@
77	77	relationTable = (RelationTable)relationInserter.getTable();
78	78	}
79	79
80		~~- @Override~~
81		~~- protected void deleteDataFrom(int rcId, String op) throws PersistenceException {~~
82		~~- deleteDataFrom(rcId, op, linkTable, "resource");~~
83		~~- deleteDataFrom(rcId, op, langlinkTable, "resource");~~
84		~~- deleteDataFrom(rcId, op, broaderTable, "resource");~~
85		~~- deleteDataFrom(rcId, op, conceptTable, "resource");~~
86		~~- }~~
87		-
88	80	protected int deleteConceptBroader(int narrow, int broad) throws SQLException {
89	81	String sql = "DELETE FROM "+broaderTable.getSQLName()+" " +
90	82	" WHERE narrow = "+narrow+" " +
—	—	@@ -482,12 +474,6 @@
483	475	degreeTable = (EntityTable)degreeInserter.getTable();
484	476	}
485	477
486		-
487		~~- @Override~~
488		~~- protected void deleteDataFrom(int rcId, String op) throws PersistenceException {~~
489		~~- throw new UnsupportedOperationException();~~
490		~~- }~~
491		-
492	478	protected int getNumberOfConcepts() throws PersistenceException {
493	479	String sql = "select count(*) from "+conceptTable.getSQLName();
494	480	try {
—	—	@@ -890,10 +876,5 @@
891	877	return executeChunkedUpdate(query, chunkFactor);
892	878	}
893	879
894		~~- @Override~~
895		~~- protected void deleteDataFrom(int rcId, String op) throws PersistenceException {~~
896		~~- //deleteDataFrom(rcId, op, conceptInfoTable, "concept", conceptTable, "resource");~~
897		~~- throw new UnsupportedOperationException();~~
898		~~- }~~
899	880	}
900	881	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseGlobalConceptStoreBuilder.java
—	—	@@ -173,11 +173,6 @@
174	174
175	175	//---------------------------------
176	176
177		~~- @Override~~
178		~~- protected void deleteDataFrom(int rcId, String op) throws PersistenceException {~~
179		~~- throw new UnsupportedOperationException();~~
180		~~- }~~
181		-
182	177	public void importConcepts() throws PersistenceException {
183	178	Corpus[] cc = getLanguages();
184	179
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java
—	—	@@ -0,0 +1,9 @@
	2	+package de.brightbyte.wikiword.store.builder;
	3	+
	4	+import de.brightbyte.util.PersistenceException;
	5	+
	6	+public interface IncrementalStoreBuilder {
	7	+
	8	+ public void deleteDataAfter(int delAfter, boolean inclusive) throws PersistenceException;
	9	+
	10	+}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/LocalConceptStoreBuilder.java
—	—	@@ -18,7 +18,7 @@
19	19	* (generally by a WikiTextAnalyzer) may be written to. It may be backed by
20	20	* a RDBMS, or some other way of storing the data.
21	21	*/
22		~~-public interface LocalConceptStoreBuilder extends WikiWordConceptStoreBuilder<LocalConcept> {~~
	22	+public interface LocalConceptStoreBuilder extends WikiWordConceptStoreBuilder<LocalConcept>, IncrementalStoreBuilder {
23	23
24	24	public abstract void storeDefinition(int rcId, int conceptId, String definition)
25	25	throws PersistenceException;
—	—	@@ -26,6 +26,9 @@
27	27	public abstract int storeResource(String name, ResourceType ptype,
28	28	Date time) throws PersistenceException;
29	29
	30	+ public abstract int storeResourceAbout(String name, ResourceType ptype,
	31	+ Date time, int concept, String conceptName) throws PersistenceException;
	32	+
30	33	public abstract int storeConcept(int rcId, String name, ConceptType ctype)
31	34	throws PersistenceException;
32	35
—	—	@@ -51,6 +54,12 @@
52	55	String sourceName, int target, String targetName, AliasScope scope)
53	56	throws PersistenceException;
54	57
	58	+ public abstract void storeAbout(int resource, String conceptName)
	59	+ throws PersistenceException;
	60	+
	61	+ public abstract void storeAbout(int resource, int concept, String conceptName)
	62	+ throws PersistenceException;
	63	+
55	64	//public abstract void storeConceptReference(int rcId, int source,
56	65	// String sourceName, String target) throws PersistenceException;
57	66
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java
—	—	@@ -22,7 +22,7 @@
23	23	* The TweakSet supplied to the constructur is used by
24	24	* {@link de.brightbyte.wikiword.store.DatabaseTextStore}, see there.
25	25	*/
26		~~-public class DatabaseTextStoreBuilder extends DatabaseWikiWordStoreBuilder implements TextStoreBuilder {~~
	26	+public class DatabaseTextStoreBuilder extends DatabaseLocalStoreBuilder implements TextStoreBuilder {
27	27
28	28	protected LocalConceptStoreSchema localConceptDatabase;
29	29
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
—	—	@@ -13,7 +13,7 @@
14	14	import de.brightbyte.wikiword.schema.AliasScope;
15	15	import de.brightbyte.wikiword.schema.PropertyStoreSchema;
16	16
17		~~-public class DatabasePropertyStoreBuilder extends DatabaseWikiWordStoreBuilder implements PropertyStoreBuilder {~~
	17	+public class DatabasePropertyStoreBuilder extends DatabaseLocalStoreBuilder implements PropertyStoreBuilder {
18	18
19	19	protected DatabaseLocalConceptStoreBuilder conceptStore;
20	20

Status & tagging log

20:26, 1 February 2009 Aaron Schulz (talk | contribs) changed the status of r46640 [removed: new added: deferred]