r67395 MediaWiki - Code Review archive

Repository:MediaWiki
Revision:< r67394‎ | r67395 | r67396 >
Date:09:42, 5 June 2010
Author:daniel
Status:deferred
Tags:
Comment:
introduced PropertyStore for querying concept properties
Modified paths:
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/ConceptFeatures.java (modified) (history)
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/ConceptProperties.java (added) (history)
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/DatabaseFeatureStore.java (modified) (history)
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/DatabasePropertyStore.java (added) (history)
  • /trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/PropertyStore.java (added) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/PropertyStore.java
——@@ -0,0 +1,18 @@
 2+package de.brightbyte.wikiword.store;
 3+
 4+import java.util.Collection;
 5+import java.util.Map;
 6+
 7+import de.brightbyte.util.PersistenceException;
 8+import de.brightbyte.wikiword.model.ConceptProperties;
 9+import de.brightbyte.wikiword.model.WikiWordConcept;
 10+
 11+public interface PropertyStore<T extends WikiWordConcept> {
 12+
 13+ public ConceptProperties<T> getConceptProperties(int concept) throws PersistenceException;
 14+ public ConceptProperties<T> getConceptProperties(int concept, Collection<String> props) throws PersistenceException;
 15+
 16+ public Map<Integer, ConceptProperties<T>> getConceptsProperties(int[] concepts) throws PersistenceException;
 17+ public Map<Integer, ConceptProperties<T>> getConceptsProperties(int[] concepts, Collection<String> props) throws PersistenceException;
 18+
 19+}
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/DatabasePropertyStore.java
——@@ -0,0 +1,306 @@
 2+package de.brightbyte.wikiword.store;
 3+
 4+import java.sql.ResultSet;
 5+import java.sql.SQLException;
 6+import java.util.ArrayList;
 7+import java.util.Collection;
 8+import java.util.Collections;
 9+import java.util.HashMap;
 10+import java.util.List;
 11+import java.util.Map;
 12+
 13+import de.brightbyte.data.MultiMap;
 14+import de.brightbyte.data.ValueSetMultiMap;
 15+import de.brightbyte.data.cursor.DataSet;
 16+import de.brightbyte.db.DatabaseDataSet;
 17+import de.brightbyte.db.DatabaseUtil;
 18+import de.brightbyte.db.EntityTable;
 19+import de.brightbyte.db.QueryDataSet;
 20+import de.brightbyte.db.RelationTable;
 21+import de.brightbyte.db.TemporaryTableDataSet;
 22+import de.brightbyte.util.PersistenceException;
 23+import de.brightbyte.wikiword.ConceptType;
 24+import de.brightbyte.wikiword.TweakSet;
 25+import de.brightbyte.wikiword.model.ConceptProperties;
 26+import de.brightbyte.wikiword.model.WikiWordConcept;
 27+import de.brightbyte.wikiword.schema.PropertyStoreSchema;
 28+import de.brightbyte.wikiword.store.DatabaseFeatureStore.NormalizationMode;
 29+
 30+public class DatabasePropertyStore<T extends WikiWordConcept>
 31+ extends DatabaseWikiWordStore
 32+ implements PropertyStore<T> {
 33+
 34+ protected WikiWordConcept.Factory<T> conceptFactory;
 35+ protected DatabaseWikiWordConceptStore<T> conceptStore;
 36+ protected RelationTable propertyTable;
 37+ protected EntityTable conceptTable;
 38+
 39+ private DatabaseDataSet.Factory<ConceptProperties<T>> conceptPropertiesFactory = new DatabaseDataSet.Factory<ConceptProperties<T>>() {
 40+
 41+ public ConceptProperties<T> newInstance(ResultSet row) throws Exception {
 42+ int concept = -1;
 43+ String name = null;
 44+ ValueSetMultiMap<String, String> properties = new ValueSetMultiMap<String, String>();
 45+
 46+ do {
 47+ int c = row.getInt("concept");
 48+ if (concept < 0) {
 49+ concept = c;
 50+ name = row.getString("concept_name");
 51+ }
 52+ else if (concept != c) break;
 53+
 54+ String property = row.getString("property");
 55+ String value = row.getString("value");
 56+
 57+ properties.put(property, value);
 58+ } while (row.next());
 59+
 60+ if (concept<0) return null;
 61+
 62+ T r = newConcept(concept, name, null, 1, -1); //TODO: global vs. local
 63+ return new ConceptProperties<T>(r, properties);
 64+ }
 65+
 66+ };
 67+
 68+ protected DatabasePropertyStore(DatabaseWikiWordConceptStore<T> conceptStore, PropertyStoreSchema database, TweakSet tweaks) throws SQLException {
 69+ super(database, tweaks);
 70+
 71+ this.conceptStore = conceptStore;
 72+ this.conceptFactory = conceptStore.getConceptFactory();
 73+
 74+ this.conceptTable = (EntityTable)conceptStore.getDatabaseAccess().getTable("concept");
 75+ this.propertyTable = (RelationTable)database.getTable("property");
 76+ }
 77+
 78+ protected T newConcept(int id, String name, ConceptType type, int cardinality, double relevance) throws PersistenceException {
 79+ return conceptFactory.newInstance(id, name, type, cardinality, relevance);
 80+ }
 81+
 82+ public ConceptProperties<T> getConceptProperties(int concept) throws PersistenceException {
 83+ return getConceptProperties(concept, null);
 84+ }
 85+
 86+ public ConceptProperties<T> getConceptProperties(int concept, Collection<String> properties) throws PersistenceException {
 87+ try {
 88+ String sql = "SELECT concept, concept_name, property, value FROM " +propertyTable.getSQLName()+" as F ";
 89+ sql += " WHERE concept = "+concept;
 90+ if ( properties != null ) sql += " AND property IN " + database.encodeSet(properties) + " ";
 91+
 92+ return readConceptProperties("getConceptProperties", sql, "concept", null, null, null, "property", "normal_weight"); //FIXME: name, card, relevance
 93+ } catch (SQLException e) {
 94+ throw new PersistenceException(e);
 95+ }
 96+ }
 97+
 98+ public Map<Integer, ConceptProperties<T>> getConceptsProperties(int[] concepts) throws PersistenceException {
 99+ return getConceptsProperties(concepts, null);
 100+ }
 101+
 102+ public Map<Integer, ConceptProperties<T>> getConceptsProperties(int[] concepts, Collection<String> properties) throws PersistenceException {
 103+ if (concepts.length==0) return Collections.emptyMap();
 104+
 105+ try {
 106+ String sql = "SELECT concept, concept_name, property, value FROM " +propertyTable.getSQLName()+" as F ";
 107+ sql += " WHERE concept IN "+database.encodeSet(concepts);
 108+ if ( properties != null ) sql += " AND property IN " + database.encodeSet(properties) + " ";
 109+
 110+ return readConceptsProperties("getConceptsProperties", sql, "concept", null, null, null, "property", "normal_weight"); //FIXME: name, card, relevance
 111+ } catch (SQLException e) {
 112+ throw new PersistenceException(e);
 113+ }
 114+ }
 115+
 116+ protected MultiMap<String, String, ? extends Collection<String>> readMultiMap(String name, String sql, String keyField, String valueField, NormalizationMode normalize) throws PersistenceException {
 117+ try {
 118+ ResultSet rs = database.executeQuery(name, sql);
 119+ try {
 120+ MultiMap<String, String, ? extends Collection<String>> v = readMultiMap(rs, null, keyField, valueField, new ValueSetMultiMap<String, String>(), normalize);
 121+ return v;
 122+ } finally {
 123+ rs.close();
 124+ }
 125+ } catch (SQLException e) {
 126+ throw new PersistenceException(e);
 127+ }
 128+ }
 129+
 130+ protected ConceptProperties<T> readConceptProperties(String name, String sql,
 131+ String conceptField, String nameField, String cardinalityField, String relevanceField,
 132+ String keyField, String valueField) throws PersistenceException {
 133+ try {
 134+ ResultSet rs = database.executeQuery(name, sql);
 135+ try {
 136+ return readConceptProperties(rs, conceptField, nameField, cardinalityField, relevanceField, keyField, valueField);
 137+ } finally {
 138+ rs.close();
 139+ }
 140+ } catch (SQLException e) {
 141+ throw new PersistenceException(e);
 142+ }
 143+ }
 144+
 145+ protected Map<Integer, ConceptProperties<T>> readConceptsProperties(String name, String sql,
 146+ String conceptField, String nameField, String cardinalityField, String relevanceField,
 147+ String keyField, String valueField) throws PersistenceException {
 148+ try {
 149+ ResultSet rs = database.executeQuery(name, sql);
 150+ Map<Integer, ConceptProperties<T>> propertys = new HashMap<Integer, ConceptProperties<T>>();
 151+
 152+ try {
 153+ while (!rs.isAfterLast()) {
 154+ ConceptProperties<T> f = readConceptProperties(rs, conceptField, nameField, cardinalityField, relevanceField, keyField, valueField);
 155+ propertys.put(f.getId(), f);
 156+ }
 157+
 158+ return propertys;
 159+ } finally {
 160+ rs.close();
 161+ }
 162+ } catch (SQLException e) {
 163+ throw new PersistenceException(e);
 164+ }
 165+ }
 166+
 167+ public ConceptProperties<T> readConceptProperties(ResultSet rs,
 168+ String conceptField, String nameField, String cardinalityField, String relevanceField,
 169+ String keyField, String valueField) throws PersistenceException {
 170+ try {
 171+ rs.next(); //TODO: return what iof this fails??
 172+ int id = DatabaseUtil.asInt(rs.getObject(conceptField));
 173+ String n = nameField == null ? null : DatabaseUtil.asString(rs.getObject(nameField));
 174+ int c = cardinalityField == null ? 1 : DatabaseUtil.asInt(rs.getObject(cardinalityField));
 175+ double r = relevanceField == null ? 1 : DatabaseUtil.asDouble(rs.getObject(relevanceField));
 176+ rs.previous();
 177+
 178+ MultiMap<String, String, ? extends Collection<String>> props = readMultiMap(rs, conceptField, keyField, valueField, new ValueSetMultiMap<String, String>(), NormalizationMode.AUTO);
 179+
 180+ T ref = conceptFactory.newInstance(id, n, null, c, r); //XXX: type??
 181+ return new ConceptProperties<T>(ref, props);
 182+ } catch (SQLException e) {
 183+ throw new PersistenceException(e);
 184+ }
 185+ }
 186+
 187+ protected MultiMap<String, String, ? extends Collection<String>> readMultiMap(ResultSet rs, String conceptField, String propertyField, String valueField, MultiMap<String, String, ? extends Collection<String>> v, NormalizationMode normalize) throws SQLException {
 188+ if (v==null) v = new ValueSetMultiMap<String, String>();
 189+
 190+ int concept = -1;
 191+ int count = 0;
 192+
 193+ while (rs.next()) {
 194+ String p = rs.getString(propertyField);
 195+
 196+ if (conceptField!=null) {
 197+ Object c = rs.getObject(conceptField);
 198+ if (concept<0) concept = DatabaseUtil.asInt(c);
 199+ else if (concept!=DatabaseUtil.asInt(c)) {
 200+ if (!rs.previous()) throw new RuntimeException ("push-back failed on result set! "+rs.getClass()); //push back
 201+ break;
 202+ }
 203+ }
 204+
 205+ count++;
 206+
 207+ String n = rs.getString(valueField);
 208+ v.put(p, n);
 209+ }
 210+
 211+ return v;
 212+ }
 213+
 214+ public DataSet<ConceptProperties<T>> getNeighbourhoodProperties(final int concept) throws PersistenceException {
 215+ /*
 216+ String sql = "SELECT C.id as concept, C.name as name, X.property as property, X.normal_weight as value ";
 217+ sql += " FROM " + propertyTable.getSQLName() + " as X force key (PRIMARY) ";
 218+ sql += " JOIN "+propertyTable.getSQLName()+" as N force key (PRIMARY) ON N.property = X.concept ";
 219+ sql += " JOIN "+propertyTable.getSQLName()+" as F force key (property) ON F.property = N.concept ";
 220+ sql += " JOIN "+conceptTable.getSQLName()+" as C force key (PRIMARY) ON C.id = X.concept ";
 221+ sql += " WHERE F.concept = "+concept;
 222+ sql += " ORDER BY X.concept";
 223+
 224+ return new QueryDataSet<WikiWordConceptProperties>(database, getConceptPropertiesFactory(), "getNeighbourhoodProperties", sql, false);
 225+ */
 226+ return new TemporaryTableDataSet<ConceptProperties<T>>(database, getConceptPropertiesFactory()) {
 227+
 228+ @Override
 229+ public Cursor<ConceptProperties<T>> cursor() throws PersistenceException {
 230+ try {
 231+ String n = database.createTemporaryTable("id integer not null, name varchar(255) default null, primary key (id)");
 232+ String sql = "INSERT IGNORE INTO "+n+" (id) ";
 233+ sql += " SELECT N.property as id ";
 234+ sql += " FROM "+propertyTable.getSQLName()+" AS F ";
 235+ sql += " JOIN "+propertyTable.getSQLName()+" AS N ON F.property = N.concept ";
 236+ sql += " WHERE F.concept = "+concept;
 237+
 238+ database.executeUpdate("getNeighbourhoodProperties#neighbours", sql);
 239+
 240+ sql = "UPDATE "+n+" as N ";
 241+ sql+= " JOIN "+conceptTable.getSQLName()+" AS C ON C.id = N.id ";
 242+ sql+= " SET N.name = C.name ";
 243+
 244+ database.executeUpdate("getNeighbourhoodProperties#names", sql);
 245+
 246+ sql = "SELECT N.id as concept, N.name as name, X.property as property, X.normal_weight as value ";
 247+ sql+= " FROM "+n+" AS N ";
 248+ sql+= " JOIN "+propertyTable.getSQLName()+" as X ON X.concept = N.id ";
 249+
 250+ ResultSet rs = database.executeQuery("getNeighbourhoodProperties#propertys", sql);
 251+ return new TemporaryTableDataSet.Cursor<ConceptProperties<T>>(rs, factory, database, new String[] { n }, database.getLogOutput() );
 252+ } catch (SQLException e) {
 253+ throw new PersistenceException(e);
 254+ }
 255+ }
 256+ };
 257+ }
 258+
 259+ private DatabaseDataSet.Factory<ConceptProperties<T>> getConceptPropertiesFactory() {
 260+ return conceptPropertiesFactory;
 261+ }
 262+
 263+ public DataSet<? extends T> getNeighbours(int concept) throws PersistenceException {
 264+ String sql = "SELECT DISTINCT C.id as cId, C.name as cName ";
 265+ sql += " FROM " + conceptTable.getSQLName() + " as C ";
 266+ sql += " JOIN "+propertyTable.getSQLName()+" as N ON N.property = C.id ";
 267+ sql += " JOIN "+propertyTable.getSQLName()+" as F ON F.property = N.concept ";
 268+ sql += " WHERE F.concept = "+concept+" ";
 269+
 270+ return new QueryDataSet<T>(database, conceptStore.getRowConceptFactory(), "getNeighbours", sql, false);
 271+ }
 272+
 273+ public List<Integer> getNeighbourList(int concept) throws PersistenceException {
 274+ String sql = "SELECT DISTINCT N.property as concept ";
 275+ sql += " FROM "+propertyTable.getSQLName()+" as N ";
 276+ sql += " JOIN "+propertyTable.getSQLName()+" as F ON F.property = N.concept ";
 277+ sql += " WHERE F.concept = "+concept;
 278+
 279+ return readIdList("getNeighbourList", sql, "concept");
 280+ }
 281+
 282+ protected List<Integer> readIdList(String name, String sql, String valueField) throws PersistenceException {
 283+ try {
 284+ ResultSet rs = database.executeQuery(name, sql);
 285+ try {
 286+ return readIdList(rs, valueField, new ArrayList<Integer>());
 287+ } finally {
 288+ rs.close();
 289+ }
 290+ } catch (SQLException e) {
 291+ throw new PersistenceException(e);
 292+ }
 293+ }
 294+
 295+ protected List<Integer> readIdList(ResultSet rs, String valueField, List<Integer> v) throws SQLException {
 296+ if (v==null) v = new ArrayList<Integer>();
 297+
 298+ while (rs.next()) {
 299+ Number n = (Number)rs.getObject(valueField);
 300+
 301+ v.add(n instanceof Integer ? (Integer)n : n.intValue());
 302+ }
 303+
 304+ return v;
 305+ }
 306+
 307+}
\ No newline at end of file
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/store/DatabaseFeatureStore.java
——@@ -89,7 +89,7 @@
9090 sql += " WHERE concept = "+concept;
9191 if (maxConceptFeatures>0) sql += " ORDER BY normal_weight DESC LIMIT "+maxConceptFeatures;
9292
93 - return readConceptFeatures("getFeatureVector", sql, "concept", null, null, null, "feature", "normal_weight"); //FIXME: name, card, relevance
 93+ return readConceptFeatures("getConceptFeatures", sql, "concept", null, null, null, "feature", "normal_weight"); //FIXME: name, card, relevance
9494 }
9595
9696 public Map<Integer, ConceptFeatures<T, Integer>> getConceptsFeatures(int[] concepts) throws PersistenceException {
——@@ -100,7 +100,7 @@
101101 sql += " WHERE concept IN "+database.encodeSet(concepts);
102102 if (maxConceptFeatures>0) sql += " ORDER BY concept, normal_weight DESC";
103103
104 - return readConceptsFeatures("getFeatureVectors", sql, "concept", null, null, null, "feature", "normal_weight"); //FIXME: name, card, relevance
 104+ return readConceptsFeatures("getConceptsFeatures", sql, "concept", null, null, null, "feature", "normal_weight"); //FIXME: name, card, relevance
105105 } catch (SQLException e) {
106106 throw new PersistenceException(e);
107107 }
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/ConceptProperties.java
——@@ -0,0 +1,66 @@
 2+package de.brightbyte.wikiword.model;
 3+
 4+import gnu.trove.impl.Constants;
 5+
 6+import java.util.Collection;
 7+
 8+import de.brightbyte.data.IntLabeledVector;
 9+import de.brightbyte.data.LabeledVector;
 10+import de.brightbyte.data.MultiMap;
 11+
 12+public class ConceptProperties<C extends WikiWordConcept> {
 13+ protected MultiMap<String, String, ? extends Collection<String>> properties;
 14+ protected C concept;
 15+
 16+ public ConceptProperties(C concept, MultiMap<String, String, ? extends Collection<String>> properties) {
 17+ if (properties==null) throw new NullPointerException();
 18+ if (concept==null) throw new NullPointerException();
 19+ this.properties = properties;
 20+ this.concept = concept;
 21+ }
 22+
 23+ public String toString() {
 24+ return concept+ ":"+properties;
 25+ }
 26+
 27+ public MultiMap<String, String, ? extends Collection<String>> getProperties() {
 28+ return properties;
 29+ }
 30+
 31+ public WikiWordConcept getConcept() {
 32+ return concept;
 33+ }
 34+
 35+ public int getId() {
 36+ return concept.getId();
 37+ }
 38+
 39+
 40+ @Override
 41+ public int hashCode() {
 42+ return concept.hashCode();
 43+ }
 44+
 45+ @Override
 46+ public boolean equals(Object obj) {
 47+ if (this == obj)
 48+ return true;
 49+ if (obj == null)
 50+ return false;
 51+ if (getClass() != obj.getClass())
 52+ return false;
 53+ final ConceptProperties other = (ConceptProperties) obj;
 54+
 55+ return concept.equals(other.concept);
 56+ }
 57+
 58+ public static LabeledVector<Integer>newIntFeaturVector() {
 59+ return newIntFeaturVector( -1 );
 60+ }
 61+
 62+ public static LabeledVector<Integer>newIntFeaturVector(int capacity) {
 63+ if ( capacity <= 0 ) capacity = Constants.DEFAULT_CAPACITY;
 64+ return new IntLabeledVector(capacity);
 65+ }
 66+
 67+}
Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/ConceptFeatures.java
——@@ -6,9 +6,9 @@
77
88 public class ConceptFeatures<C extends WikiWordConcept, K> {
99 protected LabeledVector<K> features;
10 - protected WikiWordConcept concept;
 10+ protected C concept;
1111
12 - public ConceptFeatures(WikiWordConcept concept, LabeledVector<K> features) {
 12+ public ConceptFeatures(C concept, LabeledVector<K> features) {
1313 if (features==null) throw new NullPointerException();
1414 if (concept==null) throw new NullPointerException();
1515 this.features = features;

Status & tagging log