r50230 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r50229‎ \| r50230 \| r50231 >
Date:	18:48, 5 May 2009
Author:	daniel
Status:	deferred
Tags:
Comment:	Made PropertyExtraction independant of concept creation
Modified paths:	/trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/ConceptBasedStoreBuilder.java (added) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/LocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/PropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/TextStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java
—	—	@@ -12,10 +12,13 @@
13	13	import de.brightbyte.wikiword.TweakSet;
14	14	import de.brightbyte.wikiword.analyzer.WikiTextAnalyzer;
15	15	import de.brightbyte.wikiword.analyzer.WikiPage;
	16	+import de.brightbyte.wikiword.store.builder.ConceptBasedStoreBuilder;
16	17	import de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder;
17	18
18	19	public class PropertyImporter extends ConceptImporter {
19	20
	21	+ boolean buildConcepts = true;
	22	+
20	23	public PropertyImporter(WikiTextAnalyzer analyzer, LocalConceptStoreBuilder store, TweakSet tweaks) throws PersistenceException {
21	24	super(analyzer, store, tweaks);
22	25	}
—	—	@@ -44,24 +47,37 @@
45	48	String name = analyzerPage.getConceptName();
46	49	String rcName = analyzerPage.getResourceName();
47	50
48		~~- int rcId = storeResource(rcName, analyzerPage.getResourceType(), timestamp);~~
	51	+ int rcId = 0;
	52	+ int cid = 0;
49	53
50		~~- ConceptType ctype = analyzerPage.getConceptType();~~
51		~~- int cid = storeConcept(rcId, name, ctype);~~
	54	+ ResourceType rcType = analyzerPage.getResourceType();
52	55
53		~~- //storeProperty(rcId, cid, name, "__TYPE__", analyzerPage.getConceptType().getName()); //FIXME: remove me!~~
	56	+ if (buildConcepts) {
	57	+ rcId = storeResource(rcName, rcType, timestamp);
	58	+
	59	+ if (rcType == ResourceType.REDIRECT) {
	60	+ storeAlias(analyzerPage, rcId);
	61	+ }
	62	+
	63	+ ConceptType ctype = analyzerPage.getConceptType();
	64	+ cid = storeConcept(rcId, name, ctype);
	65	+ }
54	66
55		~~- MultiMap<String, CharSequence, Set<CharSequence>> properties = analyzerPage.getProperties();~~
56		~~- for (Map.Entry<String, Set<CharSequence>> e: properties.entrySet()) {~~
57		~~- String property = e.getKey();~~
	67	+ if (rcType == ResourceType.ARTICLE \|\| rcType == ResourceType.SUPPLEMENT) {
	68	+ MultiMap<String, CharSequence, Set<CharSequence>> properties = analyzerPage.getProperties();
	69	+ for (Map.Entry<String, Set<CharSequence>> e: properties.entrySet()) {
	70	+ String property = e.getKey();
	71	+
	72	+ for (CharSequence v: e.getValue()) {
	73	+ storeProperty(rcId, cid, name, property, v.toString());
	74	+ }
	75	+ }
58	76
59		~~- for (CharSequence v: e.getValue()) {~~
60		~~- storeProperty(rcId, cid, name, property, v.toString());~~
	77	+ if (buildConcepts) {
	78	+ storeSupplements(rcId, cid, analyzerPage);
61	79	}
62	80	}
63	81
64		~~- storeSupplements(rcId, cid, analyzerPage);~~
65		-
66	82	return cid;
67	83	}
68	84
—	—	@@ -71,12 +87,18 @@
72	88
73	89	if (t!=ResourceType.ARTICLE
74	90	&& t!=ResourceType.CATEGORY
75		~~- && t!=ResourceType.SUPPLEMENT) return false;~~
	91	+ && t!=ResourceType.SUPPLEMENT) {
	92	+ return false;
	93	+ }
76	94
77	95	if (t==ResourceType.SUPPLEMENT) {
78	96	return true;
79	97	}
80	98
	99	+ if (t==ResourceType.REDIRECT) {
	100	+ return buildConcepts;
	101	+ }
	102	+
81	103	if ( analyzerPage.getProperties().isEmpty()
82	104	&& analyzerPage.getSupplementedConcept()==null
83	105	&& analyzerPage.getSupplementLinks().isEmpty() ) {
—	—	@@ -90,11 +112,44 @@
91	113
92	114	public static void declareOptions(Arguments args) {
93	115	AbstractImporter.declareOptions(args);
	116	+
	117	+ args.declare("attach", null, false, Boolean.class, "attach properties to existing thesaurus");
94	118	}
95	119
96	120	@Override
97	121	public void configure(Arguments args) throws Exception {
98	122	super.configure(args);
	123	+
	124	+ if (args.isSet("attach")) buildConcepts = false;
99	125	}
100	126
	127	+ protected boolean getPurgeData() {
	128	+ return buildConcepts;
	129	+ }
	130	+
	131	+ @Override
	132	+ public void finish() throws PersistenceException {
	133	+ ConceptBasedStoreBuilder store = buildConcepts ? this.store : this.propertyStore;
	134	+ boolean resolveIdsFirst = buildConcepts ? true : false;
	135	+
	136	+ if (beginTask("PropertyImporter.finish", "finishImport")) {
	137	+ store.finalizeImport();
	138	+ endTask("PropertyImporter.finish", "finishImport");
	139	+ }
	140	+
	141	+ if (resolveIdsFirst && beginTask("PropertyImporter.finish", "finishIdReferences#1")) {
	142	+ store.finishIdReferences();
	143	+ endTask("PropertyImporter.finish", "finishIdReferences#1");
	144	+ }
	145	+
	146	+ if (beginTask("PropertyImporter.finish", "finishAliases")) {
	147	+ store.finishAliases();
	148	+ endTask("PropertyImporter.finish", "finishAliases");
	149	+ }
	150	+
	151	+ if (!resolveIdsFirst && beginTask("PropertyImporter.finish", "finishIdReferences#2")) {
	152	+ store.finishIdReferences();
	153	+ endTask("PropertyImporter.finish", "finishIdReferences#2");
	154	+ }
	155	+ }
101	156	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java
—	—	@@ -273,7 +273,7 @@
274	274
275	275	if (operation == Operation.FRESH) {
276	276	section("-- purge --------------------------------------------------");
277		~~- initializeStores(true, getDropWarnings()); //FIXME: don't purge warning always... but when?!~~
	277	+ initializeStores(getPurgeData(), getDropWarnings());
278	278	}
279	279	else {
280	280	initializeStores(false, getDropWarnings());
—	—	@@ -314,6 +314,10 @@
315	315	return false;
316	316	}
317	317
	318	+ protected boolean getPurgeData() {
	319	+ return true;
	320	+ }
	321	+
318	322	public int getExitCode() {
319	323	return exitCode;
320	324	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
—	—	@@ -60,16 +60,6 @@
61	61	endTask("ConceptImporter.finish", "finishImport");
62	62	}
63	63
64		~~- if (storeProperties && beginTask("ConceptImporter.finish", "finishImport")) {~~
65		~~- propertyStore.finalizeImport();~~
66		~~- endTask("ConceptImporter.finish", "finishImport");~~
67		~~- }~~
68		-
69		~~- if (storeFlatText && beginTask("ConceptImporter.finish", "finishImport")) {~~
70		~~- textStore.finalizeImport();~~
71		~~- endTask("ConceptImporter.finish", "finishImport");~~
72		~~- }~~
73		-
74	64	if (beginTask("ConceptImporter.finish", "finishBadLinks")) {
75	65	store.finishBadLinks();
76	66	endTask("ConceptImporter.finish", "finishBadLinks");
—	—	@@ -98,11 +88,6 @@
99	89	endTask("ConceptImporter.finish", "finishAliases");
100	90	}
101	91
102		~~- if (propertyStore!=null && beginTask("ConceptImporter.finish", "propertyStore#finishAliases")) {~~
103		~~- propertyStore.finishAliases();~~
104		~~- endTask("ConceptImporter.finish", "propertyStore#finishAliases");~~
105		~~- }~~
106		-
107	92	//TODO: finish aliases for textStore!
108	93
109	94	if (beginTask("ConceptImporter.finish", "finishRelations")) {
—	—	@@ -243,7 +228,7 @@
244	229
245	230	@Override
246	231	public int importPage(WikiPage analyzerPage, Date timestamp) throws PersistenceException {
247		~~- ResourceType ptype = analyzerPage.getResourceType();~~
	232	+ ResourceType rcType = analyzerPage.getResourceType();
248	233	String name = analyzerPage.getConceptName();
249	234	String rcName = analyzerPage.getResourceName();
250	235	String text = analyzerPage.getText().toString();
—	—	@@ -251,18 +236,18 @@
252	237	//String title = analyzerPage.getTitle().toString();
253	238
254	239	//TODO: check if page is stored. if up to date, skip. if older, update. if missing, create. optionally force update.
255		~~- int rcId = storeResource(rcName, ptype, timestamp);~~
	240	+ int rcId = storeResource(rcName, rcType, timestamp);
256	241
257	242	if (storeFlatText) {
258		~~- textStore.storeRawText(rcId, rcName, ptype, text);~~
	243	+ textStore.storeRawText(rcId, rcName, text);
259	244	}
260	245
261	246	if (storeFlatText) {
262	247	CharSequence plain = analyzerPage.getPlainText(false);
263		~~- textStore.storePlainText(rcId, rcName, ptype, plain.toString());~~
	248	+ textStore.storePlainText(rcId, rcName, plain.toString());
264	249	}
265	250
266		~~- if (ptype == ResourceType.CATEGORY) {~~
	251	+ if (rcType == ResourceType.CATEGORY) {
267	252	List<WikiTextAnalyzer.WikiLink> links = analyzerPage.getLinks();
268	253	linkTracker.step(links.size());
269	254
—	—	@@ -290,7 +275,7 @@
291	276	// need resolve-ids on langling, then!
292	277	// beware aliased categories!
293	278	}
294		~~- else if (ptype == ResourceType.ARTICLE \|\| ptype == ResourceType.SUPPLEMENT) {~~
	279	+ else if (rcType == ResourceType.ARTICLE \|\| rcType == ResourceType.SUPPLEMENT) {
295	280	conceptTracker.step();
296	281
297	282	//TODO: handle "other meanings" header (mini-disambig!)
—	—	@@ -392,7 +377,7 @@
393	378
394	379	//FIXME: store supplement links
395	380	}
396		~~- else if (ptype == ResourceType.DISAMBIG) {~~
	381	+ else if (rcType == ResourceType.DISAMBIG) {
397	382	//storeConcept(rcId, name, ConceptType.NONE);
398	383
399	384	Set<CharSequence> terms = analyzerPage.getTitleTerms();
—	—	@@ -416,7 +401,7 @@
417	402	}
418	403	}
419	404	}
420		~~- else if (ptype == ResourceType.LIST) {~~
	405	+ else if (rcType == ResourceType.LIST) {
421	406	//storeConcept(rcId, name, ConceptType.NONE);
422	407
423	408	//FIXME: extract fewer links... use disambig-logic?
—	—	@@ -428,37 +413,49 @@
429	414	//TODO: extract concept name from "List of..." ?
430	415	//FIXME: category-like interpretation!
431	416	}
432		~~- else if (ptype == ResourceType.REDIRECT) {~~
433		~~- WikiTextAnalyzer.WikiLink link = analyzerPage.getRedirect();~~
434		-
435		~~- if (link==null) {~~
436		~~- warn(rcId, "bad redirect (no link)", "Text: "+StringUtils.clipString(text, 256, "..."), null);~~
437		~~- }~~
438		~~- else if (link.getInterwiki()!=null \|\| link.getNamespace()!=0) {~~
439		~~- //redirects to other wikis or into another namespace are handeled as BAD page.~~
440		~~- out.info("skipped bad redirect "+rcName+" -> "+link);~~
441		~~- }~~
442		~~- else if (name.equals(link.getPage().toString())) {~~
443		~~- warn(rcId, "bad redirect (self-link)", "page "+name, null);~~
444		~~- }~~
445		~~- else {~~
446		~~- int conceptId = storeConcept(rcId, name, ConceptType.ALIAS);~~
447		~~- storePageTerms(rcId, analyzerPage.getTitleTerms(), -1, link.getPage().toString(), ExtractionRule.TERM_FROM_REDIRECT );~~
448		~~- storeConceptAlias(rcId, conceptId, name, -1, link.getPage().toString(), AliasScope.REDIRECT); //TODO: confidence?...~~
449		-
450		~~- //FIXME: redir to section!~~
451		~~- }~~
	417	+ else if (rcType == ResourceType.REDIRECT) {
	418	+ storeAlias(analyzerPage, rcId);
452	419	}
453		~~- else if (ptype == ResourceType.BAD) {~~
	420	+ else if (rcType == ResourceType.BAD) {
454	421	out.info("skipped BAD page "+rcName);
455	422	}
456	423	else {
457		~~- out.warn("skipped page "+rcName+" ["+ptype+"]");~~
	424	+ out.warn("skipped page "+rcName+" ["+rcType+"]");
458	425	}
459	426
460	427	return rcId;
461	428	}
462	429
	430	+ protected int storeAlias(WikiPage analyzerPage, int rcId) throws PersistenceException {
	431	+ String name = analyzerPage.getConceptName();
	432	+ String rcName = analyzerPage.getResourceName();
	433	+ String text = analyzerPage.getText().toString();
	434	+
	435	+ WikiTextAnalyzer.WikiLink link = analyzerPage.getRedirect();
	436	+
	437	+ int conceptId = 0;
	438	+
	439	+ if (link==null) {
	440	+ warn(rcId, "bad redirect (no link)", "Text: "+StringUtils.clipString(text, 256, "..."), null);
	441	+ }
	442	+ else if (link.getInterwiki()!=null \|\| link.getNamespace()!=0) {
	443	+ //redirects to other wikis or into another namespace are handeled as BAD page.
	444	+ out.info("skipped bad redirect "+rcName+" -> "+link);
	445	+ }
	446	+ else if (name.equals(link.getPage().toString())) {
	447	+ warn(rcId, "bad redirect (self-link)", "page "+name, null);
	448	+ }
	449	+ else {
	450	+ conceptId = storeConcept(rcId, name, ConceptType.ALIAS);
	451	+ storePageTerms(rcId, analyzerPage.getTitleTerms(), -1, link.getPage().toString(), ExtractionRule.TERM_FROM_REDIRECT );
	452	+ storeConceptAlias(rcId, conceptId, name, -1, link.getPage().toString(), AliasScope.REDIRECT); //TODO: confidence?...
	453	+
	454	+ //FIXME: redir to section!
	455	+ }
	456	+
	457	+ return conceptId;
	458	+ }
	459	+
463	460	public static void declareOptions(Arguments args) {
464	461	AbstractImporter.declareOptions(args);
465	462
—	—	@@ -551,7 +548,6 @@
552	549	for (CharSequence supp: supplementLinks) {
553	550	storeConceptAlias(rcId, -1, supp.toString(), cid, name, AliasScope.SUPPLEMENT);
554	551	}
555		-
556	552	}
557	553
558	554	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DebugLocalConceptStoreBuilder.java
—	—	@@ -32,14 +32,22 @@
33	33
34	34	public class DebugTextStoreBuilder implements TextStoreBuilder {
35	35
36		~~- public void storePlainText(int textId, String name, ResourceType ptype, String text) throws PersistenceException {~~
37		~~- log("* storePlainText("+textId+", "+name+", "+ptype+", "+ptype+") *");~~
	36	+ public void storePlainText(int rcId, String name, String text) throws PersistenceException {
	37	+ log("* storePlainText("+rcId+", "+name+") *");
38	38	}
39	39
40		~~- public void storeRawText(int textId, String name, ResourceType ptype, String text) throws PersistenceException {~~
41		~~- log("* storeRawText("+textId+", "+name+", "+ptype+", "+ptype+") *");~~
	40	+ public void storeRawText(int rcId, String name, String text) throws PersistenceException {
	41	+ log("* storeRawText("+rcId+", "+name+") *");
42	42	}
43	43
	44	+ public void finishAliases() throws PersistenceException {
	45	+ log("* finishAliases *");
	46	+ }
	47	+
	48	+ public void finishIdReferences() throws PersistenceException {
	49	+ log("* finishIdReferences *");
	50	+ }
	51	+
44	52	public void checkConsistency() throws PersistenceException {
45	53	log("* checkConsistency *");
46	54	}
—	—	@@ -125,11 +133,14 @@
126	134	log("* finishAliases *");
127	135	}
128	136
129		-
130	137	public void finalizeImport() throws PersistenceException {
131	138	log("* finalizeImport *");
132	139	}
133	140
	141	+ public void finishIdReferences() throws PersistenceException {
	142	+ log("* finishIdReferences *");
	143	+ }
	144	+
134	145	public void prepareImport() throws PersistenceException {
135	146	log("* prepareImport *");
136	147	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/TextStoreBuilder.java
—	—	@@ -1,13 +1,15 @@
2	2	package de.brightbyte.wikiword.store.builder;
3	3
4	4	import de.brightbyte.util.PersistenceException;
5		~~-import de.brightbyte.wikiword.ResourceType;~~
6	5
7		~~-public interface TextStoreBuilder extends WikiWordStoreBuilder {~~
8		~~- public abstract void storeRawText(int textId, String name, ResourceType ptype, String text)~~
	6	+public interface TextStoreBuilder extends WikiWordStoreBuilder, IncrementalStoreBuilder {
	7	+ public abstract void storeRawText(int rcId, String rcName, String text)
9	8	throws PersistenceException;
10	9
11		~~- public abstract void storePlainText(int textId, String name, ResourceType ptype, String text)~~
	10	+ public abstract void storePlainText(int rcId, String rcName, String text)
12	11	throws PersistenceException;
	12	+
	13	+ //public abstract void finishAliases() throws PersistenceException;
	14	+ //public abstract void finishIdReferences() throws PersistenceException;
13	15
14	16	}
\ No newline at end of file
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/PropertyStoreBuilder.java
—	—	@@ -3,21 +3,12 @@
4	4	import de.brightbyte.util.PersistenceException;
5	5	import de.brightbyte.wikiword.store.WikiWordLocalStore;
6	6
7		~~-public interface PropertyStoreBuilder extends WikiWordStoreBuilder, WikiWordLocalStore {~~
	7	+public interface PropertyStoreBuilder extends WikiWordStoreBuilder, WikiWordLocalStore, ConceptBasedStoreBuilder {
	8	+
8	9	public abstract void storeProperty(int resourceId, int conceptId, String concept, String property, String value)
9	10	throws PersistenceException;
10	11
11		- /*
12		~~- public abstract int storeConcept(int rcId, String name, ConceptType ctype)~~
13		~~- throws PersistenceException;~~
14		-
15		~~- public abstract int storeResource(String name, ResourceType ptype, Date time)~~
16		~~- throws PersistenceException;~~
17		-
18		~~- public abstract void storeConceptAlias(int rcId, int source, String sourceName, int target, String targetName, AliasScope scope)~~
19		~~- throws PersistenceException;~~
20		~~- */~~
21		-
22	12	public abstract void finishAliases() throws PersistenceException;
	13	+ public abstract void finishIdReferences() throws PersistenceException;
23	14
24	15	}
\ No newline at end of file
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
—	—	@@ -492,7 +492,35 @@
493	493
494	494	return executeChunkedUpdate("resolveRedirects", table.getName()+"."+relNameField+"+"+relIdField, sql, where, aliasTable, "source", chunkFactor);
495	495	}
	496	+
	497	+
	498	+ /**
	499	+ * Builds id-references from name-references
	500	+ */
	501	+ protected int buildIdLinks(DatabaseTable table, String relNameField, String relIdField, int chunkFactor) throws PersistenceException {
	502	+ DatabaseField nmField = table.getField(relNameField);
	503	+ DatabaseField idField = table.getField(relIdField);
	504	+
	505	+ if (!(nmField instanceof ReferenceField)) throw new IllegalArgumentException(relNameField+" is not a reference field in table "+table.getName());
	506	+ if (!(idField instanceof ReferenceField)) throw new IllegalArgumentException(relIdField+" is not a reference field in table "+table.getName());
	507	+
	508	+ String nmTable = ((ReferenceField)nmField).getTargetTable();
	509	+ String idTable = ((ReferenceField)idField).getTargetTable();
	510	+
	511	+ if (!nmTable.equals(idTable)) throw new IllegalArgumentException(relNameField+" and "+relIdField+" in table "+table.getName()+" do not reference the same table: "+nmTable+" != "+idTable);
	512	+ DatabaseTable target = getTable(nmTable);
496	513
	514	+ String targetNameField = ((ReferenceField)nmField).getTargetField();
	515	+ String targetIdField = ((ReferenceField)idField).getTargetField();
	516	+
	517	+ String sql = "UPDATE "+table.getSQLName()+" as R JOIN "+target.getSQLName()+" as E "
	518	+ + " ON R."+relNameField+" = E."+targetNameField+" "
	519	+ + " SET R."+relIdField+" = E."+targetIdField+" ";
	520	+ String where = " R."+relIdField+" IS NULL";
	521	+
	522	+ return executeChunkedUpdate("buildIdLinks", table.getName()+"."+relNameField, sql, where, target, targetIdField, chunkFactor);
	523	+ }
	524	+
497	525	public void finalizeImport() throws PersistenceException {
498	526	flush();
499	527	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/ConceptBasedStoreBuilder.java
—	—	@@ -0,0 +1,10 @@
	2	+package de.brightbyte.wikiword.store.builder;
	3	+
	4	+import de.brightbyte.util.PersistenceException;
	5	+
	6	+public interface ConceptBasedStoreBuilder extends WikiWordStoreBuilder {
	7	+
	8	+ public void finishIdReferences() throws PersistenceException;
	9	+ public void finishAliases() throws PersistenceException;
	10	+
	11	+}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
—	—	@@ -12,12 +12,11 @@
13	13
14	14	import javax.sql.DataSource;
15	15
16		~~-import org.ardverk.collection.PatriciaTrie;~~
17		~~-import org.ardverk.collection.StringKeyAnalyzer;~~
18		-
19	16	import de.brightbyte.application.Agenda;
	17	+import de.brightbyte.data.Pair;
20	18	import de.brightbyte.data.PersistentIdManager;
21	19	import de.brightbyte.data.cursor.CursorProcessor;
	20	+import de.brightbyte.data.cursor.DataCursor;
22	21	import de.brightbyte.data.cursor.DataSet;
23	22	import de.brightbyte.db.DatabaseAccess;
24	23	import de.brightbyte.db.DatabaseDataSet;
—	—	@@ -172,18 +171,39 @@
173	172	}
174	173	else {
175	174	//FIXME: should fail if we are continuing a previous import, but the file doesn't exist.
176		~~- //FIXME: should failon partial load~~
	175	+ //FIXME: should fail on partial load
177	176	//XXX: could probably be skipped if continuing at a stage after dump reading
178	177	if (idManager!=null) {
179		~~- log("loading persisted ID map..."+" memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");~~
180		~~- idManager.load();~~
181		~~- log("Max persisted ID: "+idManager.getMaxId()+"; memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");~~
	178	+ if (idManager.fileExists()) {
	179	+ log("loading persisted ID map..."+" memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");
	180	+ idManager.load();
	181	+ log("Max persisted ID: "+idManager.getMaxId()+"; memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");
	182	+ } else {
	183	+ log("building persisted ID map..."+" memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");
	184	+ DataCursor<Pair<String, Integer>> cursor = getConceptIdCursor();
	185	+ idManager.slurp(cursor);
	186	+ cursor.close();
	187	+ log("Max persisted ID: "+idManager.getMaxId()+"; memory used: "+(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())/1024+"KB");
	188	+ }
182	189	}
183	190	}
184	191
185	192	super.initialize(purge, dropAll);
186	193	}
187	194
	195	+ protected DataCursor<Pair<String, Integer>> getConceptIdCursor() throws PersistenceException {
	196	+ String sql = "SELECT name, id from " + conceptTable.getSQLName();
	197	+ ResultSet rs = executeQuery("getConceptIdCursor", sql);
	198	+
	199	+ DatabaseDataSet.Factory<Pair<String, Integer>> f = new DatabaseDataSet.Factory<Pair<String, Integer>>() {
	200	+ public Pair<String, Integer> newInstance(ResultSet row) throws Exception {
	201	+ return new Pair<String, Integer>(row.getString(1), row.getInt(2));
	202	+ }
	203	+ };
	204	+
	205	+ return new DatabaseDataSet.Cursor<Pair<String, Integer>>(rs, f);
	206	+ }
	207	+
188	208	public ConceptType getConceptType(int type) {
189	209	return corpus.getConceptTypes().getType(type);
190	210	}
—	—	@@ -258,7 +278,6 @@
259	279	throw new PersistenceException(e);
260	280	}
261	281	}
262		-
263	282
264	283	/**
265	284	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeResourceAbout(java.lang.String, de.brightbyte.wikiword.ResourceType, java.util.Date, int conceptId, String conceptName)
—	—	@@ -525,6 +544,10 @@
526	545	}
527	546
528	547	public void finalizeImport() throws PersistenceException {
	548	+ if (idManager!=null) { //delete temporary ID file
	549	+ idManager.deleteFile();
	550	+ }
	551	+
529	552	try {
530	553	flush();
531	554	if (beginTask("DatabaseLocalConceptStore.finishImport", "enableKeys")) {
—	—	@@ -534,6 +557,16 @@
535	558	} catch (SQLException e) {
536	559	throw new PersistenceException(e);
537	560	}
	561	+
	562	+ if (propertyStore!=null && beginTask("finishAliases", "propertyStore.finalizeImport")) {
	563	+ propertyStore.finalizeImport();
	564	+ endTask("finishAliases", "propertyStore.finalizeImport");
	565	+ }
	566	+
	567	+ if (textStore!=null && beginTask("finishAliases", "textStore.finalizeImport")) {
	568	+ textStore.finalizeImport();
	569	+ endTask("finishAliases", "textStore.finalizeImport");
	570	+ }
538	571	}
539	572
540	573	public void finishSections() throws PersistenceException {
—	—	@@ -744,6 +777,18 @@
745	778	endTask("finishIdReferences", "buildIdLinks:alias", n+" references");
746	779	}
747	780	//if (beginTask("finishIdReferences", "buildIdLinks:reference")) buildIdLinks(referenceTable, "target_name", "target");
	781	+
	782	+ if (idManager==null && propertyStore!=null && beginTask("finishIdReferences", "propertyStore.finishIdReferences")) {
	783	+ propertyStore.finishIdReferences();
	784	+ endTask("finishIdReferences", "propertyStore.finishIdReferences");
	785	+ }
	786	+
	787	+ /*
	788	+ if (idManager==null && textStore!=null && beginTask("finishIdReferences", "textStore.finishIdReferences")) {
	789	+ textStore.finishIdReferences();
	790	+ endTask("finishIdReferences", "textStore.finishIdReferences");
	791	+ }
	792	+ */
748	793	}
749	794
750	795	public void finishAliases() throws PersistenceException {
—	—	@@ -771,7 +816,19 @@
772	817	endTask("finishAliases", "resolveRedirects:broad", n+" entries");
773	818	}
774	819
	820	+ if (propertyStore!=null && beginTask("finishAliases", "propertyStore.finishAliases")) {
	821	+ propertyStore.finishAliases();
	822	+ endTask("finishAliases", "propertyStore.finishAliases");
	823	+ }
	824	+
775	825	/*
	826	+ if (textStore!=null && beginTask("finishAliases", "textStore.finishAliases")) {
	827	+ textStore.finishAliases();
	828	+ endTask("finishAliases", "textStore.finishAliases");
	829	+ }
	830	+ */
	831	+
	832	+ /*
776	833	//NOTE: way too late for that!
777	834	if (beginTask("finishAliases", "resolveRedirects:section")) {
778	835	int n = resolveRedirects(sectionTable, "concept_name", null);
—	—	@@ -1128,33 +1185,6 @@
1129	1186	return executeChunkedUpdate("buildMeanings", "buildMeanings", sql, group, linkTable, "target");
1130	1187	}
1131	1188
1132		- /**
1133		~~- * Builds id-references from name-references~~
1134		~~- */~~
1135		~~- protected int buildIdLinks(DatabaseTable table, String relNameField, String relIdField, int chunkFactor) throws PersistenceException {~~
1136		~~- DatabaseField nmField = table.getField(relNameField);~~
1137		~~- DatabaseField idField = table.getField(relIdField);~~
1138		-
1139		~~- if (!(nmField instanceof ReferenceField)) throw new IllegalArgumentException(relNameField+" is not a reference field in table "+table.getName());~~
1140		~~- if (!(idField instanceof ReferenceField)) throw new IllegalArgumentException(relIdField+" is not a reference field in table "+table.getName());~~
1141		-
1142		~~- String nmTable = ((ReferenceField)nmField).getTargetTable();~~
1143		~~- String idTable = ((ReferenceField)idField).getTargetTable();~~
1144		-
1145		~~- if (!nmTable.equals(idTable)) throw new IllegalArgumentException(relNameField+" and "+relIdField+" in table "+table.getName()+" do not reference the same table: "+nmTable+" != "+idTable);~~
1146		~~- DatabaseTable target = getTable(nmTable);~~
1147		-
1148		~~- String targetNameField = ((ReferenceField)nmField).getTargetField();~~
1149		~~- String targetIdField = ((ReferenceField)idField).getTargetField();~~
1150		-
1151		~~- String sql = "UPDATE "+table.getSQLName()+" as R JOIN "+target.getSQLName()+" as E "~~
1152		~~- + " ON R."+relNameField+" = E."+targetNameField+" "~~
1153		~~- + " SET R."+relIdField+" = E."+targetIdField+" ";~~
1154		~~- String where = " R."+relIdField+" IS NULL";~~
1155		-
1156		~~- return executeChunkedUpdate("buildIdLinks", table.getName()+"."+relNameField, sql, where, target, targetIdField, chunkFactor);~~
1157		~~- }~~
1158		-
1159	1189	/////////////////////////////////////////////////////////////////////////////////////////////
1160	1190	protected class DatabaseLocalStatisticsStoreBuilder extends DatabaseStatisticsStoreBuilder {
1161	1191	protected EntityTable termTable;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/IncrementalStoreBuilder.java
—	—	@@ -2,7 +2,7 @@
3	3
4	4	import de.brightbyte.util.PersistenceException;
5	5
6		~~-public interface IncrementalStoreBuilder {~~
	6	+public interface IncrementalStoreBuilder extends WikiWordStoreBuilder {
7	7
8	8	public void deleteDataAfter(int delAfter, boolean inclusive) throws PersistenceException;
9	9
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/LocalConceptStoreBuilder.java
—	—	@@ -18,7 +18,7 @@
19	19	* (generally by a WikiTextAnalyzer) may be written to. It may be backed by
20	20	* a RDBMS, or some other way of storing the data.
21	21	*/
22		~~-public interface LocalConceptStoreBuilder extends WikiWordConceptStoreBuilder<LocalConcept>, IncrementalStoreBuilder {~~
	22	+public interface LocalConceptStoreBuilder extends WikiWordConceptStoreBuilder<LocalConcept>, IncrementalStoreBuilder, ConceptBasedStoreBuilder {
23	23
24	24	public abstract void storeDefinition(int rcId, int conceptId, String definition)
25	25	throws PersistenceException;
—	—	@@ -90,8 +90,6 @@
91	91	public void finishSections() throws PersistenceException;
92	92	public void finishBadLinks() throws PersistenceException;
93	93	public void finishMissingConcepts() throws PersistenceException;
94		~~- public void finishIdReferences() throws PersistenceException;~~
95		~~- public void finishAliases() throws PersistenceException;~~
96	94	public void finishRelations() throws PersistenceException;
97	95	public void finishMeanings() throws PersistenceException;
98	96	//public void finishConceptInfo() throws PersistenceException;
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseTextStoreBuilder.java
—	—	@@ -4,13 +4,15 @@
5	5	import java.sql.SQLException;
6	6
7	7	import de.brightbyte.application.Agenda;
	8	+import de.brightbyte.data.PersistentIdManager;
8	9	import de.brightbyte.db.EntityTable;
9	10	import de.brightbyte.db.Inserter;
	11	+import de.brightbyte.db.RelationTable;
10	12	import de.brightbyte.util.PersistenceException;
11	13	import de.brightbyte.wikiword.ConceptType;
12	14	import de.brightbyte.wikiword.Corpus;
13		~~-import de.brightbyte.wikiword.ResourceType;~~
14	15	import de.brightbyte.wikiword.TweakSet;
	16	+import de.brightbyte.wikiword.schema.AliasScope;
15	17	import de.brightbyte.wikiword.schema.LocalConceptStoreSchema;
16	18	import de.brightbyte.wikiword.schema.TextStoreSchema;
17	19
—	—	@@ -59,6 +61,8 @@
60	62
61	63	plainTextTable = (EntityTable)plainTextInserter.getTable();
62	64	rawTextTable = (EntityTable)rawTextInserter.getTable();
	65	+
	66	+ //this.idManager = idManager;
63	67	}
64	68
65	69	@Override
—	—	@@ -67,21 +71,22 @@
68	72	deleteDataFrom(rcId, op, plainTextTable, "id");
69	73	}
70	74
	75	+ /*
71	76	protected int getResourceId(String title) throws SQLException {
72	77	String sql = "select id from "+localConceptDatabase.getSQLTableName("resource")
73	78	+" where name = "+localConceptDatabase.quoteString(title);
74	79	return (Integer) localConceptDatabase.executeSingleValueQuery("getResourceId", sql);
75		~~- }~~
76		-
	80	+ }*/
	81	+
77	82	/**
78	83	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeRawText(int, java.lang.String)
79	84	*/
80		~~- public void storeRawText(int textId, String title, String text) throws PersistenceException {~~
	85	+ public void storeRawText(int rcId, String title, String text) throws PersistenceException {
81	86	try {
82	87	if (rawTextInserter==null) rawTextInserter = rawTextTable.getInserter();
83		~~- int rcId = getResourceId(title); //TODO: use join?~~
	88	+ //if (rcId<=0) rcId = getResourceId(title); //TODO: use join?
84	89
85		~~- rawTextInserter.updateInt("id", textId);~~
	90	+ rawTextInserter.updateInt("id", rcId);
86	91	rawTextInserter.updateInt("resource", rcId);
87	92	rawTextInserter.updateString("text", text);
88	93	rawTextInserter.updateRow();
—	—	@@ -93,12 +98,12 @@
94	99	/**
95	100	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storePlainText(int, java.lang.String)
96	101	*/
97		~~- public void storePlainText(int textId, String title, String text) throws PersistenceException {~~
	102	+ public void storePlainText(int rcId, String title, String text) throws PersistenceException {
98	103	try {
99	104	if (plainTextInserter==null) plainTextInserter = plainTextTable.getInserter();
100		~~- int rcId = getResourceId(title); //TODO: use join?~~
	105	+// if (rcId<=0) rcId = getResourceId(title); //TODO: use join?
101	106
102		~~- plainTextInserter.updateInt("id", textId);~~
	107	+ plainTextInserter.updateInt("id", rcId);
103	108	plainTextInserter.updateInt("resource", rcId);
104	109	plainTextInserter.updateString("text", text);
105	110	plainTextInserter.updateRow();
—	—	@@ -107,6 +112,7 @@
108	113	}
109	114	}
110	115
	116	+ /*
111	117	public void storePlainText(int textId, String name, ResourceType ptype, String text) throws PersistenceException {
112	118	storePlainText(textId, name, text);
113	119	}
—	—	@@ -114,7 +120,7 @@
115	121	public void storeRawText(int textId, String name, ResourceType ptype, String text) throws PersistenceException {
116	122	storeRawText(textId, name, text);
117	123	}
118		-
	124	+ */
119	125	public ConceptType getConceptType(int type) {
120	126	return localConceptDatabase.getConceptType(type);
121	127	}
—	—	@@ -122,4 +128,21 @@
123	129	public Corpus getCorpus() {
124	130	return ((TextStoreSchema)database).getCorpus();
125	131	}
	132	+
	133	+ /*
	134	+ public void finishAliases() throws PersistenceException {
	135	+ if (beginTask("DatabaseTextStoreBuilder.finishAliases", "resolveRedirects:property")) {
	136	+ RelationTable aliasTable = (RelationTable)database.getTable("alias");
	137	+ int n = resolveRedirects(aliasTable, rawTextTable, "concept_name", idManager!=null ? "concept" : null, AliasScope.REDIRECT, 3, null, null);
	138	+ endTask("DatabaseTextStoreBuilder.finishAliases", "resolveRedirects:property", n+" entries");
	139	+ }
	140	+ }
	141	+
	142	+ public void finishIdReferences() throws PersistenceException {
	143	+ if (idManager==null && beginTask("DatabaseTextStoreBuilder.finishIdReferences", "buildIdLinks:property")) {
	144	+ int n = buildIdLinks(rawTextTable, "concept_name", "concept", 1);
	145	+ endTask("DatabaseTextStoreBuilder.finishIdReferences", "buildIdLinks:property", n+" references");
	146	+ }
	147	+ }
	148	+ */
126	149	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
—	—	@@ -4,6 +4,7 @@
5	5	import java.sql.SQLException;
6	6
7	7	import de.brightbyte.application.Agenda;
	8	+import de.brightbyte.data.PersistentIdManager;
8	9	import de.brightbyte.db.Inserter;
9	10	import de.brightbyte.db.RelationTable;
10	11	import de.brightbyte.util.PersistenceException;
—	—	@@ -17,12 +18,13 @@
18	19
19	20	protected RelationTable propertyTable;
20	21	protected Inserter propertyInserter;
21		~~- private LocalConceptStoreSchema conceptStoreSchema;~~
	22	+ protected LocalConceptStoreSchema conceptStoreSchema;
	23	+ protected PersistentIdManager idManager;
22	24
23	25	public DatabasePropertyStoreBuilder(Corpus corpus, Connection connection, TweakSet tweaks) throws SQLException, PersistenceException {
24	26	this(new LocalConceptStoreSchema(corpus, connection, tweaks, true),
25	27	new PropertyStoreSchema(corpus, connection, tweaks, true),
26		~~- tweaks, null);~~
	28	+ null, tweaks, null);
27	29	}
28	30
29	31	public DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException, PersistenceException {
—	—	@@ -32,11 +34,12 @@
33	35	new PropertyStoreSchema(conceptStore.getCorpus(),
34	36	conceptStore.getDatabaseAccess().getConnection(),
35	37	tweaks, true),
	38	+ conceptStore.idManager,
36	39	tweaks,
37	40	conceptStore.getAgenda());
38	41	}
39	42
40		~~- protected DatabasePropertyStoreBuilder(LocalConceptStoreSchema conceptStoreSchema, PropertyStoreSchema database, TweakSet tweaks, Agenda agenda) throws SQLException, PersistenceException {~~
	43	+ protected DatabasePropertyStoreBuilder(LocalConceptStoreSchema conceptStoreSchema, PropertyStoreSchema database, PersistentIdManager idManager, TweakSet tweaks, Agenda agenda) throws SQLException, PersistenceException {
41	44	super(database, tweaks, agenda);
42	45
43	46	//this.conceptStore = conceptStore;
—	—	@@ -45,6 +48,7 @@
46	49	this.propertyTable = (RelationTable)propertyInserter.getTable();
47	50
48	51	this.conceptStoreSchema = conceptStoreSchema;
	52	+ this.idManager = idManager;
49	53	}
50	54
51	55	@Override
—	—	@@ -62,16 +66,6 @@
63	67	super.flush();
64	68	}
65	69
66		~~- protected int getConceptId(String title) throws SQLException {~~
67		~~- //String sql = "select id from "+localConceptDatabase.getSQLTableName("resource")~~
68		~~- // +" where name = "+localConceptDatabase.quoteString(title);~~
69		~~- //return (Integer) localConceptDatabase.executeSingleValueQuery("getResourceId", sql);~~
70		-
71		~~- //TODO: get concept id~~
72		~~- throw new UnsupportedOperationException();~~
73		~~- //return -1;~~
74		~~- }~~
75		-
76	70	/**
77	71	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeRawText(int, java.lang.String)
78	72	*/
—	—	@@ -80,7 +74,8 @@
81	75	//int cId = getConceptId(name); //TODO: use join? //FIXME: when not provided
82	76
83	77	propertyInserter.updateInt("resource", resourceId);
84		~~- if (concept>0) propertyInserter.updateInt("concept", concept); //FIXME: id cache!~~
	78	+ if (concept>0) propertyInserter.updateInt("concept", concept);
	79	+ else if (idManager!=null) propertyInserter.updateInt("concept", idManager.aquireId(name));
85	80	propertyInserter.updateString("concept_name", name);
86	81	propertyInserter.updateString("property", property);
87	82	propertyInserter.updateString("value", value);
—	—	@@ -108,12 +103,17 @@
109	104	}
110	105
111	106	public void finishAliases() throws PersistenceException {
112		-
113		~~- if (beginTask("finishAliases", "resolveRedirects:property")) {~~
	107	+ if (beginTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property")) {
114	108	RelationTable aliasTable = (RelationTable)conceptStoreSchema.getTable("alias");
115		-
116		~~- int n = resolveRedirects(aliasTable, propertyTable, "concept_name", "concept", AliasScope.REDIRECT, 3, null, null);~~
117		~~- endTask("finishAliases", "resolveRedirects:property", n+" entries");~~
	109	+ int n = resolveRedirects(aliasTable, propertyTable, "concept_name", idManager!=null ? "concept" : null, AliasScope.REDIRECT, 3, null, null);
	110	+ endTask("DatabasePropertyStoreBuilder.finishAliases", "resolveRedirects:property", n+" entries");
118	111	}
119	112	}
	113	+
	114	+ public void finishIdReferences() throws PersistenceException {
	115	+ if (idManager==null && beginTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property")) {
	116	+ int n = buildIdLinks(propertyTable, "concept_name", "concept", 1);
	117	+ endTask("DatabasePropertyStoreBuilder.finishIdReferences", "buildIdLinks:property", n+" references");
	118	+ }
	119	+ }
120	120	}
\ No newline at end of file

Status & tagging log

16:00, 26 May 2009 Tim Starling (talk | contribs) changed the status of r50230 [removed: new added: deferred]
19:41, 5 May 2009 Aaron Schulz (talk | contribs) changed the status of r50230 [removed: new added: deferred]