r52260 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:r52259‎ | r52260 | r52261 >
Date:21:44, 21 June 2009
Author:daniel
Status:deferred
Tags:
Comment:
db unit testing improvements
Modified paths:
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/AbstractIntegratorApp.java (modified) (history)
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/FeatureSetSourceDescriptor.java (modified) (history)
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/LoadForeignProperties.java (modified) (history)
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/data/TsvFeatureSetCursor.java (modified) (history)
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/processor/ForeignPropertyPassThrough.java (modified) (history)
  • /trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/IntegratorSchema.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/processor/ForeignPropertyPassThrough.java
@@ -10,6 +10,7 @@
1111
1212 public class ForeignPropertyPassThrough extends AbstractForeignPropertyProcessor {
1313 protected ForeignPropertyStoreBuilder store;
 14+ protected String qualifier;
1415
1516 public ForeignPropertyPassThrough(ForeignPropertyStoreBuilder store) {
1617 if (store==null) throw new NullPointerException();
@@ -24,9 +25,17 @@
2526 List<Object> vv = p.getValue();
2627
2728 for (Object v: vv) {
28 - store.storeProperty(e.getAuthority(), e.getID(), prop, String.valueOf(v), null);
 29+ store.storeProperty(e.getAuthority(), e.getID(), prop, String.valueOf(v), qualifier);
2930 }
3031 }
3132 }
3233
 34+ public String getQualifier() {
 35+ return qualifier;
 36+ }
 37+
 38+ public void setQualifier(String qualifier) {
 39+ this.qualifier = qualifier;
 40+ }
 41+
3342 }
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/AbstractIntegratorApp.java
@@ -18,6 +18,7 @@
1919 import de.brightbyte.data.cursor.DataCursor;
2020 import de.brightbyte.db.SqlScriptRunner;
2121 import de.brightbyte.io.IOUtil;
 22+import de.brightbyte.io.LineCursor;
2223 import de.brightbyte.text.Chunker;
2324 import de.brightbyte.util.BeanUtils;
2425 import de.brightbyte.util.PersistenceException;
@@ -185,10 +186,19 @@
186187
187188 fsc = new ResultSetFeatureSetCursor(rs, fields);
188189 } else {
189 - fsc = new TsvFeatureSetCursor(in, enc);
 190+ LineCursor lines = new LineCursor(in, enc);
190191
191 - if (fields!=null) ((TsvFeatureSetCursor)fsc).setFields(fields);
192 - else ((TsvFeatureSetCursor)fsc).readFields();
 192+ Chunker chunker = sourceDescriptor.getCsvLineChunker();
 193+
 194+ fsc = new TsvFeatureSetCursor(lines, chunker);
 195+
 196+ if (fields!=null) {
 197+ if (sourceDescriptor.getSkipHeader()) ((TsvFeatureSetCursor)fsc).readFields();
 198+ ((TsvFeatureSetCursor)fsc).setFields(fields);
 199+ } else {
 200+ ((TsvFeatureSetCursor)fsc).readFields();
 201+ fields = ((TsvFeatureSetCursor)fsc).getFields();
 202+ }
193203 }
194204
195205 String propField = sourceDescriptor.getPropertyNameField();
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/LoadForeignProperties.java
@@ -45,7 +45,14 @@
4646 @Override
4747 protected ForeignPropertyProcessor createProcessor(ForeignPropertyStoreBuilder conceptStore) throws InstantiationException {
4848 // FIXME: parameter list is restrictive, pass descriptor
49 - return instantiate(sourceDescriptor, "foreignPropertyProcessorClass", ForeignPropertyPassThrough.class, conceptStore);
 49+ ForeignPropertyProcessor processor = instantiate(sourceDescriptor, "foreignPropertyProcessorClass", ForeignPropertyPassThrough.class, conceptStore);
 50+
 51+ if (processor instanceof ForeignPropertyPassThrough) {
 52+ String qualifier = sourceDescriptor.getTweak("property-qualifier", null);
 53+ if (qualifier!=null) ((ForeignPropertyPassThrough)processor).setQualifier(qualifier);
 54+ }
 55+
 56+ return processor;
5057 }
5158
5259 public static void main(String[] argv) throws Exception {
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/FeatureSetSourceDescriptor.java
@@ -8,6 +8,7 @@
99 import java.util.Map;
1010
1111 import de.brightbyte.text.Chunker;
 12+import de.brightbyte.text.CsvLineChunker;
1213 import de.brightbyte.wikiword.TweakSet;
1314
1415 public class FeatureSetSourceDescriptor extends TweakSet {
@@ -77,8 +78,8 @@
7879 }
7980
8081
81 - public Map<String, Chunker> getDataFieldChunkers() { //FIXME: factory/parser!
82 - return getTweak("foreign.chunkers", (Map<String, Chunker>)null);
 82+ public Map<String, Chunker> getDataFieldChunkers() {
 83+ return getTweak("field-chunkers", (Map<String, Chunker>)null);
8384 }
8485
8586 public String getPropertyValueField() {
@@ -97,5 +98,26 @@
9899 return getTweak("property-subject-name-field", null);
99100 }
100101
 102+ public boolean getSkipHeader() {
 103+ return getTweak("skip-header", false);
 104+ }
 105+
 106+ public Chunker getCsvLineChunker() {
 107+ Chunker chunker = getTweak("csv-chunker", null);
 108+
 109+ if (chunker==null) {
 110+ char ch = getTweak("csv-separator", '\u008F');
 111+ if (ch!='\u008F') chunker = new CsvLineChunker(ch, getTweak("csv-backslash-escape", false));
 112+ }
 113+
 114+ if (chunker==null) {
 115+ if (getTweak("file-format", "tsv").equals("csv"))
 116+ chunker = CsvLineChunker.csv;
 117+ }
 118+
 119+ if (chunker==null) chunker = CsvLineChunker.tsv;
 120+ return chunker;
 121+ }
 122+
101123
102124 }
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/data/TsvFeatureSetCursor.java
@@ -8,7 +8,8 @@
99
1010 import de.brightbyte.data.cursor.DataCursor;
1111 import de.brightbyte.io.LineCursor;
12 -import de.brightbyte.io.TsvCursor;
 12+import de.brightbyte.io.ChunkingCursor;
 13+import de.brightbyte.text.Chunker;
1314 import de.brightbyte.text.CsvLineChunker;
1415 import de.brightbyte.util.PersistenceException;
1516
@@ -17,25 +18,29 @@
1819 private String[] fields;
1920
2021 public TsvFeatureSetCursor(InputStream in, String enc) throws UnsupportedEncodingException {
21 - this( new TsvCursor(in, enc) );
 22+ this( new ChunkingCursor(in, enc) );
2223 }
2324
2425 public TsvFeatureSetCursor(Reader rd) {
25 - this(new TsvCursor(rd));
 26+ this(new ChunkingCursor(rd));
2627 }
2728
2829 public TsvFeatureSetCursor(BufferedReader reader) {
29 - this(new TsvCursor(reader));
 30+ this(new ChunkingCursor(reader));
3031 }
3132
3233 public TsvFeatureSetCursor(LineCursor lines) {
33 - this(new TsvCursor(lines));
 34+ this(new ChunkingCursor(lines));
3435 }
3536
36 - public TsvFeatureSetCursor(LineCursor lines, CsvLineChunker chunker) {
37 - this(new TsvCursor(lines, chunker));
 37+ public TsvFeatureSetCursor(LineCursor lines, char separator, boolean esc) {
 38+ this(new ChunkingCursor(lines, new CsvLineChunker(separator, esc)));
3839 }
3940
 41+ public TsvFeatureSetCursor(LineCursor lines, Chunker chunker) {
 42+ this(new ChunkingCursor(lines, chunker));
 43+ }
 44+
4045 public TsvFeatureSetCursor(DataCursor<List<String>> source) {
4146 if (source==null) throw new NullPointerException();
4247 this.source = source;
Index: trunk/WikiWord/WikiWordIntegrator/src/main/java/de/brightbyte/wikiword/integrator/store/IntegratorSchema.java
@@ -42,7 +42,7 @@
4343 return table;
4444 }
4545
46 - public RelationTable newConceptMappingTable(String name, boolean unique) {
 46+ public RelationTable newConceptAssociationTable(String name, boolean unique) {
4747 RelationTable table = new RelationTable(this, name, getDefaultTableAttributes());
4848
4949 table.addField( new DatabaseField(this, "external_authority", getTextType(64), null, true, null) );

Status & tagging log