r50285 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r50284‎ \| r50285 \| r50286 >
Date:	21:15, 6 May 2009
Author:	daniel
Status:	deferred
Tags:
Comment:	standalone property extraction
Modified paths:	/trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/AbstractImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportConcepts.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportProperties.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordConceptStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportProperties.java
—	—	@@ -16,7 +16,9 @@
17	17	private PropertyStoreBuilder propertyStore;
18	18
19	19	public ImportProperties() {
20		~~- super("ExtractProperties");~~
	20	+ super("ImportProperties");
	21	+
	22	+ baseTasks = new String[] {"ImportProperties", "ImportConcepts", "BuildStatistics", "BuildConceptInfo", };
21	23	}
22	24
23	25	@Override
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportConcepts.java
—	—	@@ -20,7 +20,7 @@
21	21	private TextStoreBuilder textStore;
22	22
23	23	public ImportConcepts() {
24		~~- super("ImportDump");~~
	24	+ super("ImportConcepts");
25	25	}
26	26
27	27	@Override
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ImportApp.java
—	—	@@ -25,7 +25,8 @@
26	26	protected static enum Operation {
27	27	FRESH,
28	28	CONTINUE,
29		~~- APPEND~~
	29	+ APPEND,
	30	+ ATTACH
30	31	}
31	32
32	33	private boolean useAgenda;
—	—	@@ -33,7 +34,8 @@
34	35	protected Agenda agenda;
35	36
36	37	protected Operation operation = null;
37		~~- private Monitor agendaMonitor;~~
	38	+ private Monitor agendaMonitor;
	39	+ protected String[] baseTasks = new String[] {};
38	40
39	41	public ImportApp(String agendaTask, boolean allowGlobal, boolean allowLocal) { //TODO: agenda-params!
40	42	super(allowGlobal, allowLocal);
—	—	@@ -73,7 +75,7 @@
74	76	this.agendaMonitor = monitor;
75	77	}
76	78
77		~~- protected void initAgenda(Agenda agenda) throws PersistenceException {~~
	79	+ protected void initAgenda(Agenda agenda, String... canAttachTo) throws PersistenceException {
78	80	agenda.setLogger(
79	81	logLevel<=LOG_INFO ? out : null,
80	82	logLevel<=LOG_FINE ? out : null
—	—	@@ -86,12 +88,16 @@
87	89	if (args.isSet("fresh")) operation = Operation.FRESH;
88	90	else if (args.isSet("continue")) operation = Operation.CONTINUE;
89	91	else if (args.isSet("append")) operation = Operation.APPEND;
	92	+ else if (args.isSet("attach")) operation = Operation.ATTACH;
90	93
91	94	//String phase = fresh ? null : args.getStringOption("phase", null);
92	95	//if (phase != null) ctinue = true;
93	96	Prompt p = new Prompt();
94	97
95		~~- if (agenda.wasFinished(agendaTask)) {~~
	98	+ if (operation==Operation.ATTACH && agenda.canRelyUpon(canAttachTo)) {
	99	+ p.println("### building upon previous task ("+agenda.getLastRootTask()+")");
	100	+ }
	101	+ else if (agenda.wasFinished(agendaTask)) {
96	102	p.println("### the last run FINISHED.");
97	103
98	104	if (operation==Operation.FRESH) {
—	—	@@ -100,12 +106,15 @@
101	107	else if (operation==Operation.APPEND) {
102	108	p.println("### performing APPENDING import!");
103	109	}
	110	+ else if (operation==Operation.ATTACH) {
	111	+ p.println("### performing ATTACHING import!");
	112	+ }
104	113	else if (operation==Operation.CONTINUE) {
105	114	p.println("### noting to CONTINUE.");
106	115	System.exit(0);
107	116	}
108	117	else {
109		~~- String s = p.prompt("### type \"fresh\" to start a fresh run, or \"append\" to append.", "");~~
	118	+ String s = p.prompt("### type \"fresh\" to start a fresh run", "");
110	119
111	120	if (s==null) {
112	121	p.println("### UNEXPECTED EOF.");
—	—	@@ -118,9 +127,9 @@
119	128	if (s.equals("fresh")) {
120	129	operation = Operation.FRESH;
121	130	}
122		~~- else if (s.equals("append")) {~~
	131	+ /*else if (s.equals("append")) {
123	132	operation = Operation.APPEND;
124		~~- }~~
	133	+ }*/
125	134	else {
126	135	p.println("### aborted.");
127	136	System.exit(0);
—	—	@@ -139,6 +148,8 @@
140	149	else {
141	150	if (operation==Operation.APPEND) {
142	151	p.println("### can not append to incomplete database!");
	152	+ } else if (operation==Operation.ATTACH) {
	153	+ p.println("### can not attach to incomplete database!");
143	154	}
144	155
145	156	String s = p.prompt("### type \"fresh\" to start a fresh run, or \"continue\" to continue the previous run\".", "");
—	—	@@ -175,6 +186,9 @@
176	187	if (operation==Operation.APPEND) {
177	188	p.println("### can not append to incomplete database!");
178	189	operation = null;
	190	+ } else if (operation==Operation.ATTACH) {
	191	+ p.println("### can not attach to incomplete database!");
	192	+ operation = null;
179	193	}
180	194
181	195	if (operation == null) {
—	—	@@ -268,7 +282,7 @@
269	283	if (!noimport) {
270	284	if (useAgenda) {
271	285	agenda = conceptStore.getAgenda();
272		~~- initAgenda(agenda);~~
	286	+ initAgenda(agenda, baseTasks = new String[] {});
273	287	}
274	288
275	289	if (operation == Operation.FRESH) {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
—	—	@@ -20,6 +20,7 @@
21	21	import de.brightbyte.wikiword.analyzer.WikiPage;
22	22	import de.brightbyte.wikiword.model.LocalConceptReference;
23	23	import de.brightbyte.wikiword.schema.AliasScope;
	24	+import de.brightbyte.wikiword.store.builder.IncrementalStoreBuilder;
24	25	import de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder;
25	26	import de.brightbyte.wikiword.store.builder.PropertyStoreBuilder;
26	27	import de.brightbyte.wikiword.store.builder.TextStoreBuilder;
—	—	@@ -120,6 +121,11 @@
121	122	store.flush();
122	123	}
123	124
	125	+ protected void deleteDataAfter(int delAfter) throws PersistenceException {
	126	+ ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!
	127	+ if (propertyStore!=null && storeProperties) ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!
	128	+ }
	129	+
124	130	protected void buildTermsForMissingConcepts() throws PersistenceException {
125	131	if (!analyzer.isInitialized()) { //XXX: ugly hack!
126	132	analyzer.initialize(Namespace.canonicalNamespaces, true);
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/AbstractImporter.java
—	—	@@ -125,7 +125,8 @@
126	126
127	127	int delAfter = (Integer)rec.parameters.get("lastRcId_");
128	128	out.info("=== DIRTY BLOCK FOR SAFEPOINT#"+safepointNumber+", Deleting entries starting after id: #"+delAfter+" ===");
129		~~- ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!~~
	129	+
	130	+ deleteDataAfter(delAfter);
130	131	}
131	132
132	133	out.info("=== BEGINNING BLOCK FOR SAFEPOINT#"+safepointNumber+": "+getAgenda().getCurrentRecord().parameters+" ===");
—	—	@@ -135,6 +136,10 @@
136	137	return doit;
137	138	}
138	139
	140	+ protected void deleteDataAfter(int delAfter) throws PersistenceException {
	141	+ ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!
	142	+ }
	143	+
139	144	protected void concludeStep() throws PersistenceException{
140	145	safepointTicks++;
141	146	if (safepointTicks>=safepointInterval) {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java
—	—	@@ -13,6 +13,7 @@
14	14	import de.brightbyte.wikiword.analyzer.WikiTextAnalyzer;
15	15	import de.brightbyte.wikiword.analyzer.WikiPage;
16	16	import de.brightbyte.wikiword.store.builder.ConceptBasedStoreBuilder;
	17	+import de.brightbyte.wikiword.store.builder.IncrementalStoreBuilder;
17	18	import de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder;
18	19
19	20	public class PropertyImporter extends ConceptImporter {
—	—	@@ -83,6 +84,11 @@
84	85	return cid;
85	86	}
86	87
	88	+ protected void deleteDataAfter(int delAfter) throws PersistenceException {
	89	+ if (buildConcepts) ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false);
	90	+ ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false);
	91	+ }
	92	+
87	93	@Override
88	94	protected boolean isRelevant(WikiPage analyzerPage) {
89	95	ResourceType t = analyzerPage.getResourceType();
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabasePropertyStoreBuilder.java
—	—	@@ -5,6 +5,7 @@
6	6
7	7	import de.brightbyte.application.Agenda;
8	8	import de.brightbyte.data.PersistentIdManager;
	9	+import de.brightbyte.db.DatabaseTable;
9	10	import de.brightbyte.db.Inserter;
10	11	import de.brightbyte.db.RelationTable;
11	12	import de.brightbyte.util.PersistenceException;
—	—	@@ -27,6 +28,13 @@
28	29	null, tweaks, null);
29	30	}
30	31
	32	+ protected DatabaseTable getTable(String name) {
	33	+ if (database.hasTable(name))
	34	+ return database.getTable(name);
	35	+ else
	36	+ return conceptStoreSchema.getTable(name);
	37	+ }
	38	+
31	39	public DatabasePropertyStoreBuilder(DatabaseLocalConceptStoreBuilder conceptStore, TweakSet tweaks) throws SQLException, PersistenceException {
32	40	this(new LocalConceptStoreSchema(conceptStore.getCorpus(),
33	41	conceptStore.getDatabaseAccess().getConnection(),
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
—	—	@@ -216,9 +216,14 @@
217	217	public void flush() throws PersistenceException{
218	218	if (idManager!=null) idManager.flush();
219	219	super.flush();
	220	+
	221	+ if (propertyStore!=null)
	222	+ propertyStore.flush();
	223	+
	224	+ if (textStore!=null)
	225	+ textStore.flush();
220	226	}
221	227
222		-
223	228	protected void deleteDataFrom(int rcId, String op) throws PersistenceException {
224	229	deleteDataFrom(rcId, op, definitionTable, "resource");
225	230
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordConceptStoreBuilder.java
—	—	@@ -408,6 +408,18 @@
409	409	return executeChunkedUpdate("finishGlobalConcepts", "similarities:bilink", sql, suffix, linkTable, "A.anchor", 1);
410	410	}
411	411
	412	+
	413	+ @Override
	414	+ public void flush() throws PersistenceException{
	415	+ super.flush();
	416	+
	417	+ if (statsStore!=null)
	418	+ statsStore.flush();
	419	+
	420	+ if (infoStore!=null)
	421	+ infoStore.flush();
	422	+ }
	423	+
412	424	//----------------------------------------------------------------------------------
413	425
414	426	protected abstract DatabaseStatisticsStoreBuilder newStatisticsStoreBuilder() throws SQLException, PersistenceException;

Status & tagging log

16:00, 26 May 2009 Tim Starling (talk | contribs) changed the status of r50285 [removed: new added: deferred]
22:33, 6 May 2009 Brion VIBBER (talk | contribs) changed the status of r50285 [removed: new added: deferred]