r57716 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r57715‎ \| r57716 \| r57717 >
Date:	20:07, 14 October 2009
Author:	daniel
Status:	deferred
Tags:
Comment:	fixed index issues
Modified paths:	/trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/PropertyImporter.java
—	—	@@ -89,8 +89,12 @@
90	90	}
91	91
92	92	protected void deleteDataAfter(int delAfter) throws PersistenceException {
93		~~- if (buildConcepts) ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false);~~
94		~~- ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false);~~
	93	+ if (buildConcepts) super.deleteDataAfter(delAfter);
	94	+ else {
	95	+ ((IncrementalStoreBuilder)propertyStore).prepareMassProcessing();
	96	+ ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false);
	97	+ ((IncrementalStoreBuilder)propertyStore).prepareMassInsert();
	98	+ }
95	99	}
96	100
97	101	@Override
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
—	—	@@ -116,8 +116,13 @@
117	117	}
118	118
119	119	protected void deleteDataAfter(int delAfter) throws PersistenceException {
120		~~- ((IncrementalStoreBuilder)store).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!~~
121		~~- if (propertyStore!=null && storeProperties) ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false); //FIXME: make sure we are not off by one!~~
	120	+ super.deleteDataAfter(delAfter);
	121	+
	122	+ if (propertyStore!=null && storeProperties) {
	123	+ ((IncrementalStoreBuilder)propertyStore).prepareMassProcessing();
	124	+ ((IncrementalStoreBuilder)propertyStore).deleteDataAfter(delAfter, false);
	125	+ ((IncrementalStoreBuilder)propertyStore).prepareMassInsert();
	126	+ }
122	127	}
123	128
124	129	protected void storeSuffixInfo(WikiPage analyzerPage, int rcId, int conceptId, String conceptName) throws PersistenceException {
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseWikiWordStoreBuilder.java
—	—	@@ -211,6 +211,14 @@
212	212	database.error("failed to store warning!", e);
213	213	}
214	214	}
	215	+
	216	+ protected void enableKeys() throws PersistenceException {
	217	+ try {
	218	+ database.enableKeys();
	219	+ } catch (SQLException e) {
	220	+ throw new PersistenceException(e);
	221	+ }
	222	+ }
215	223
216	224	/**
217	225	* @see de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder#storeWarning(int, java.lang.String, java.lang.String)
—	—	@@ -266,11 +274,19 @@
267	275	String sql;
268	276
269	277	if (via!=null) {
	278	+ DatabaseKey k = via.findKeyForField(viaField);
	279	+ String forceCIndex = k==null ? null : k.getSQLName();
	280	+
	281	+ k = rel.findKeyForField(field);
	282	+ String forceTIndex = k==null ? null : k.getSQLName();
	283	+
270	284	sql = "DELETE ";
271	285	if (deleteVia) sql += " T, C FROM ";
272	286	else sql += " FROM T USING ";
273	287	sql += rel.getSQLName()+" AS T ";
	288	+ if (forceTIndex!=null) sql += " FORCE INDEX(" + forceTIndex + ") ";
274	289	sql += " LEFT JOIN "+via.getSQLName()+" AS C";
	290	+ if (forceCIndex!=null) sql += " FORCE INDEX(" + forceCIndex + ") ";
275	291	sql += " ON T."+field+" = C."+viaJoinField+" ";
276	292	sql += " WHERE C."+viaField+" "+op+" "+rcId;
277	293	}
—	—	@@ -298,14 +314,22 @@
299	315	sql += " )";
300	316	*/
301	317
	318	+ DatabaseKey k = table.findKeyForField(field);
	319	+ String forceEIndex = k==null ? null : k.getSQLName();
	320	+
	321	+ k = ref.findKeyForField(refField);
	322	+ String forceRIndex = k==null ? null : k.getSQLName();
	323	+
302	324	sql = "DELETE FROM E ";
303	325	sql += " USING " + table.getSQLName() + " AS E ";
	326	+ if (forceEIndex!=null) sql += " FORCE INDEX(" + forceEIndex + ") ";
304	327	sql += " LEFT JOIN " + ref.getSQLName() + " AS R ";
	328	+ if (forceRIndex!=null) sql += " FORCE INDEX(" + forceRIndex + ") ";
305	329	sql += " ON R." + refField + " = E." + field;
306	330	sql += " WHERE R." + refField + " IS NULL;";
307	331
308	332	long t = System.currentTimeMillis();
309		~~- int c = executeUpdate("deleteDataFrom", sql);~~
	333	+ int c = executeUpdate("deleteOrphansFrom", sql);
310	334	trace("deleted "+c+" orphan rows from "+table.getName()+" where no reference exists from "+ref.getSQLName()+"."+refField+", took "+(System.currentTimeMillis()-t)/1000+" sec");
311	335	}
312	336
—	—	@@ -498,12 +522,12 @@
499	523
500	524	if (forceRIndex==null) {
501	525	DatabaseKey k = table.findKeyForField(relIdField);
502		~~- forceRIndex = k==null ? null : k.getName();~~
	526	+ forceRIndex = k==null ? null : k.getSQLName();
503	527	}
504	528
505	529	if (forceRIndex==null && relIdField==null) {
506	530	DatabaseKey k = table.findKeyForField(relNameField);
507		~~- forceRIndex = k==null ? null : k.getName();~~
	531	+ forceRIndex = k==null ? null : k.getSQLName();
508	532	}
509	533
510	534	if (forceRIndex!=null && table.getKey(forceRIndex)==null) throw new IllegalArgumentException("unknown key: "+forceRIndex);
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseLocalConceptStoreBuilder.java
—	—	@@ -626,11 +626,7 @@
627	627	}
628	628
629	629	public void prepareMassProcessing() throws PersistenceException {
630		~~- try {~~
631		~~- database.enableKeys();~~
632		~~- } catch (SQLException e) {~~
633		~~- throw new PersistenceException(e);~~
634		~~- }~~
	630	+ this.enableKeys();
635	631
636	632	if (propertyStore!=null) {
637	633	propertyStore.prepareMassProcessing();
—	—	@@ -673,15 +669,8 @@
674	670	}
675	671
676	672	public void preparePostProcessing() throws PersistenceException {
677		~~- try {~~
678	673	flush();
679		~~- if (beginTask("DatabaseLocalConceptStore.preparePostProcessing", "enableKeys")) {~~
680		~~- database.enableKeys();~~
681		~~- endTask("DatabaseLocalConceptStore.preparePostProcessing", "enableKeys");~~
682		~~- }~~
683		~~- } catch (SQLException e) {~~
684		~~- throw new PersistenceException(e);~~
685		~~- }~~
	674	+ enableKeys();
686	675
687	676	if (propertyStore!=null && beginTask("preparePostProcessing", "propertyStore.preparePostProcessing")) {
688	677	propertyStore.preparePostProcessing();
—	—	@@ -877,6 +866,8 @@
878	867	}
879	868
880	869	public void finishIdReferences() throws PersistenceException {
	870	+ enableKeys();
	871	+
881	872	if (idManager==null && beginTask("finishIdReferences", "buildIdLinks:about")) {
882	873	int n = buildIdLinks(aboutTable, "concept_name", "concept", 1);
883	874	endTask("finishIdReferences", "buildIdLinks:about", n+" references");
—	—	@@ -927,6 +918,8 @@
928	919	}
929	920
930	921	public void finishAliases() throws PersistenceException {
	922	+ enableKeys();
	923	+
931	924	if (beginTask("finishAliases", "resolveRedirects:link")) {
932	925	//XXX: SLOW!
933	926	//TODO: smaller chunks? chunk on target table, not alias table? force index?

Status & tagging log

00:49, 31 October 2009 😂 (talk | contribs) changed the status of r57716 [removed: new added: deferred]