r51534 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r51533‎ \| r51534 \| r51535 >
Date:	12:58, 6 June 2009
Author:	daniel
Status:	deferred
Tags:
Comment:	LoadForeignProperties operational
Modified paths:	/trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/ForeignEntityStoreDescriptor.java (modified) (history) /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/LoadForeignProperties.java (modified) (history) /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/data/ForeignEntityCursor.java (modified) (history) /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/processor/AbstractProcessor.java (modified) (history) /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/DatabaseForeignPropertyStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/IntegratorSchema.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/processor/AbstractProcessor.java
—	—	@@ -40,7 +40,7 @@
41	41	}
42	42
43	43	public void reset() {
44		~~- itemTracker = new ImportProgressTracker("pages");~~
	44	+ itemTracker = new ImportProgressTracker("items");
45	45	progressTicks = 0;
46	46	}
47	47
—	—	@@ -58,6 +58,7 @@
59	59	}
60	60
61	61	public void tracerStep() {
	62	+ itemTracker.step();
62	63	progressTicks++;
63	64	if (progressTicks>progressInterval) {
64	65	trackerChunk();
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/ForeignEntityStoreDescriptor.java
—	—	@@ -1,5 +1,8 @@
2	2	package de.brightbyte.wikiword.integrator;
3	3
	4	+import java.io.File;
	5	+import java.net.MalformedURLException;
	6	+import java.net.URL;
4	7	import java.util.List;
5	8	import java.util.Map;
6	9
—	—	@@ -23,17 +26,17 @@
24	27	return getTweak("foreign.query", null);
25	28	}
26	29
27		~~- public String getSourceFileName() {~~
	30	+ public String getSourceFileName() { //FIXME
28	31	return getTweak("foreign.file", null);
29	32	}
30	33
31	34	public String[] getDataFields() {
32		~~- List<String> v = getTweak("foreign.field", (List<String>)null);~~
	35	+ List<String> v = getTweak("foreign.fields", (List<String>)null);
33	36	if (v==null) return null;
34	37	return (String[]) v.toArray(new String[v.size()]);
35	38	}
36	39
37		~~- public Map<String, String> getSplitExpressions() {~~
	40	+ public Map<String, String> getSplitExpressions() { //FIXME:!
38	41	return getTweak("split", (Map<String, String>)null);
39	42	}
40	43
—	—	@@ -42,11 +45,11 @@
43	46	}
44	47
45	48	public String getPropertyNameField() {
46		~~- return getTweak("foreign.property-name-field", "value");~~
	49	+ return getTweak("foreign.property-name-field", null);
47	50	}
48	51
49	52	public String getConceptIdField() {
50		~~- return getTweak("foreign.concept-id-field", "id");~~
	53	+ return getTweak("foreign.concept-id-field", null);
51	54	}
52	55
53	56	public String getConceptNameField() {
—	—	@@ -54,9 +57,23 @@
55	58	}
56	59
57	60	public String getAuthorityName() {
58		~~- String name = getTweak("foreign.authority-name", null);~~
	61	+ String name = getTweak("foreign.authority", null);
59	62	if (name==null) throw new RuntimeException("authority name not specified!");
60	63	return name;
61	64	}
	65	+
	66	+ public void setBaseURL(URL baseURL) {
	67	+ parameters.put(".baseURL", baseURL);
	68	+ }
62	69
	70	+ public URL getBaseURL() {
	71	+ try {
	72	+ URL u = getTweak(".baseURL", (URL)null);
	73	+ if (u==null) u = new File(".").toURI().toURL();
	74	+ return u;
	75	+ } catch (MalformedURLException e) {
	76	+ return null;
	77	+ }
	78	+ }
	79	+
63	80	}
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/LoadForeignProperties.java
—	—	@@ -25,6 +25,7 @@
26	26	import de.brightbyte.wikiword.integrator.data.MangelingFeatureSetCursor;
27	27	import de.brightbyte.wikiword.integrator.data.ResultSetFeatureSetCursor;
28	28	import de.brightbyte.wikiword.integrator.data.TsvFeatureSetCursor;
	29	+import de.brightbyte.wikiword.integrator.processor.ForeignPropertyPassThrough;
29	30	import de.brightbyte.wikiword.integrator.processor.ForeignPropertyProcessor;
30	31	import de.brightbyte.wikiword.integrator.store.DatabaseForeignPropertyStoreBuilder;
31	32	import de.brightbyte.wikiword.integrator.store.ForeignPropertyStoreBuilder;
—	—	@@ -37,24 +38,38 @@
38	39	*/
39	40	public class LoadForeignProperties extends StoreBackedApp<ForeignPropertyStoreBuilder> {
40	41
41		~~- protected ForeignPropertyStoreBuilder propertyStore;~~
	42	+ //protected ForeignPropertyStoreBuilder propertyStore;
42	43	protected ForeignPropertyProcessor propertyProcessor;
43	44	protected InputFileHelper inputHelper;
	45	+ private ForeignEntityStoreDescriptor sourceDescriptor;
44	46
45	47	public LoadForeignProperties() {
46	48	super(true, true);
47	49	}
48	50
	51	+ protected InputFileHelper getInputHelper() {
	52	+ if (inputHelper==null) {
	53	+ inputHelper = new InputFileHelper(tweaks);
	54	+ }
	55	+ return inputHelper;
	56	+ }
	57	+
49	58	@Override
50	59	protected WikiWordStoreFactory<? extends ForeignPropertyStoreBuilder> createConceptStoreFactory() throws IOException, PersistenceException {
51	60	return new DatabaseForeignPropertyStoreBuilder.Factory(getTargetTableName(), getConfiguredDataset(), getConfiguredDataSource(), tweaks);
52	61	}
53	62
54		~~- protected String getTargetTableName() {~~
55		~~- return args.getParameterCount() > 2 ? args.getParameter(2) : "foreign_property";~~
	63	+ protected String getTargetTableName() throws IOException {
	64	+ if (args.getParameterCount() > 2) return args.getParameter(2);
	65	+
	66	+ String authority = getSourceDescriptor().getAuthorityName();
	67	+ authority = authority.replaceAll("[^\\w\\d]", "_").toLowerCase();
	68	+
	69	+ return authority+"_property";
56	70	}
57	71
58	72	protected String getSourceDescriptionFileName() {
	73	+ if (args.getParameterCount() < 2) throw new IllegalArgumentException("missing second parameter (descripion file name)");
59	74	return args.getParameter(1);
60	75	}
61	76
—	—	@@ -73,13 +88,17 @@
74	89	DataCursor<ForeignEntity> cursor = openPropertySource();
75	90
76	91	section("-- process properties --------------------------------------------------");
	92	+ this.conceptStore.prepareImport();
	93	+
	94	+ this.propertyProcessor = new ForeignPropertyPassThrough(conceptStore); //FIXME
77	95	this.propertyProcessor.processProperties(cursor);
78		-
79	96	cursor.close();
	97	+
	98	+ this.conceptStore.finalizeImport();
80	99	}
81	100
82	101	protected DataCursor<ForeignEntity> openPropertySource() throws IOException, SQLException, PersistenceException {
83		~~- ForeignEntityStoreDescriptor sourceDescriptor = loadSourceDescriptor();~~
	102	+ ForeignEntityStoreDescriptor sourceDescriptor = getSourceDescriptor();
84	103
85	104	String enc = sourceDescriptor.getDataEncoding();
86	105	String sql = sourceDescriptor.getSqlQuery();
—	—	@@ -87,8 +106,8 @@
88	107
89	108	if (sql==null) {
90	109	String n = sourceDescriptor.getSourceFileName();
91		~~- String format = inputHelper.getFormat(n);~~
92		~~- in = inputHelper.open(n);~~
	110	+ String format = getInputHelper().getFormat(n); //FIXME: explicit format!
	111	+ in = getInputHelper().open(sourceDescriptor.getBaseURL(), n);
93	112
94	113	if (format!=null && format.equals("sql")) {
95	114	sql = IOUtil.slurp(in, enc);
—	—	@@ -129,15 +148,21 @@
130	149	return new ForeignEntityCursor(fsc, sourceDescriptor.getAuthorityName(), sourceDescriptor.getConceptIdField(), sourceDescriptor.getConceptNameField());
131	150	}
132	151
133		~~- protected ForeignEntityStoreDescriptor loadSourceDescriptor() throws IOException {~~
134		~~- ForeignEntityStoreDescriptor descriptor = new ForeignEntityStoreDescriptor();~~
	152	+ protected ForeignEntityStoreDescriptor getSourceDescriptor() throws IOException {
	153	+ if (sourceDescriptor!=null) return sourceDescriptor;
135	154
	155	+ sourceDescriptor = new ForeignEntityStoreDescriptor(tweaks);
	156	+
136	157	String n = getSourceDescriptionFileName();
137		~~- InputStream in = inputHelper.open(n);~~
138		~~- descriptor.loadTweaks(in);~~
	158	+ InputStream in = getInputHelper().open(n);
	159	+ sourceDescriptor.setBaseURL(getInputHelper().getBaseURL(n));
	160	+ sourceDescriptor.loadTweaks(in);
139	161	in.close();
140	162
141		~~- return descriptor;~~
	163	+ sourceDescriptor.setTweaks(System.getProperties(), "wikiword.source."); //XXX: doc
	164	+ sourceDescriptor.setTweaks(args, "source."); //XXX: doc
	165	+
	166	+ return sourceDescriptor;
142	167	}
143	168
144	169	@SuppressWarnings("unchecked")
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/data/ForeignEntityCursor.java
—	—	@@ -12,9 +12,12 @@
13	13	protected String nameField;
14	14
15	15	public ForeignEntityCursor(DataCursor<FeatureSet> source, String authorityId, String idField, String nameField) {
	16	+ if (idField==null) throw new NullPointerException();
16	17	if (authorityId==null) throw new NullPointerException();
17		~~- if (idField==null) throw new NullPointerException();~~
	18	+ if (source==null) throw new NullPointerException();
	19	+ if (nameField==null) nameField = idField;
18	20
	21	+ this.source = source;
19	22	this.authorityId = authorityId;
20	23	this.idField = idField;
21	24	this.nameField = nameField;
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/IntegratorSchema.java
—	—	@@ -32,7 +32,7 @@
33	33
34	34	table.addField( new DatabaseField(this, "property", getTextType(255), null, true, KeyType.INDEX) );
35	35	table.addField( new DatabaseField(this, "value", getTextType(255), null, true, null) );
36		~~- table.addField( new DatabaseField(this, "qualifier", getTextType(64), null, true, null) );~~
	36	+ table.addField( new DatabaseField(this, "qualifier", getTextType(64), null, false, null) );
37	37
38	38	table.addKey( new DatabaseKey(this, KeyType.INDEX, "property_value", new String[] {"property", "value"}) );
39	39	table.addKey( new DatabaseKey(this, KeyType.INDEX, "external_property", new String[] {"external_authority", "external_id", "property"}) );
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/DatabaseForeignPropertyStoreBuilder.java
—	—	@@ -52,6 +52,8 @@
53	53	protected DatabaseForeignPropertyStoreBuilder(String table, IntegratorSchema integratorSchema, TweakSet tweaks, Agenda agenda) throws SQLException, PersistenceException {
54	54	super(integratorSchema, tweaks, agenda);
55	55
	56	+ integratorSchema.newForeignPropertyTable(table);
	57	+
56	58	this.propertyInserter = configureTable(table, 128, 4*32);
57	59	this.propertyTable = (RelationTable)propertyInserter.getTable();
58	60	}
—	—	@@ -99,4 +101,9 @@
100	102	}
101	103	}
102	104	*/
	105	+
	106	+ public void prepareImport() throws PersistenceException {
	107	+ createTables(true);
	108	+ }
	109	+
103	110	}
\ No newline at end of file

Status & tagging log

14:50, 6 June 2009 Tim Starling (talk | contribs) changed the status of r51534 [removed: new added: deferred]