r64506 MediaWiki - Code Review archive

Repository:	MediaWiki
Revision:	< r64505‎ \| r64506 \| r64507 >
Date:	22:30, 1 April 2010
Author:	daniel
Status:	deferred
Tags:
Comment:	tweaks
Modified paths:	/trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/WikiWordConcept.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/StreamProcessorApp.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/AbstractPageProcessor.java (modified) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/ChunkedProgressRateTracker.java (added) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/ImportProgressTracker.java (deleted) (history) /trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseProximityStoreBuilder.java (modified) (history)

Diff [purge]

Index: trunk/WikiWord/WikiWord/src/main/java/de/brightbyte/wikiword/model/WikiWordConcept.java
—	—	@@ -124,7 +124,7 @@
125	125	}
126	126
127	127	public void setType(ConceptType type) {
128		~~- if (this.type!=null) throw new IllegalStateException("property already initialized");~~
	128	+ if (this.type!=null && !this.type.equals(ConceptType.UNKNOWN)) throw new IllegalStateException("property already initialized");
129	129	this.type = type;
130	130	}
131	131
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/ImportProgressTracker.java
—	—	@@ -1,36 +0,0 @@
2		-/**
3		- *
4		~~- */~~
5		~~-package de.brightbyte.wikiword.processor;~~
6		-
7		~~-import java.text.MessageFormat;~~
8		-
9		~~-import de.brightbyte.job.Progress;~~
10		~~-import de.brightbyte.job.ProgressRateTracker;~~
11		-
12		~~-public class ImportProgressTracker extends ProgressRateTracker {~~
13		~~- protected long counter = 0;~~
14		~~- protected String name;~~
15		-
16		~~- public ImportProgressTracker(String name) {~~
17		~~- this.name = name;~~
18		~~- }~~
19		-
20		~~- public void step() {~~
21		~~- step(1);~~
22		~~- }~~
23		-
24		~~- public void step(int c) {~~
25		~~- counter+= c;~~
26		~~- }~~
27		-
28		~~- public void chunk() {~~
29		~~- super.progress(new Progress.Event(Progress.PROGRESS, null, name, 1, position+counter, null));~~
30		~~- counter = 0;~~
31		~~- }~~
32		-
33		~~- @Override~~
34		~~- public String toString() {~~
35		~~- return MessageFormat.format("{0}: {1,number,0} ({2,number,0.0}/sec, currently {3,number,0.0}/sec)", name, position, getAverageRate(), getCurrentRate());~~
36		~~- }~~
37		-}
\ No newline at end of file
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/AbstractPageProcessor.java
—	—	@@ -29,8 +29,8 @@
30	30
31	31	protected WikiTextAnalyzer analyzer;
32	32
33		~~- private ImportProgressTracker pageTracker;~~
34		~~- private ImportProgressTracker bulkTracker;~~
	33	+ private ChunkedProgressRateTracker pageTracker;
	34	+ private ChunkedProgressRateTracker bulkTracker;
35	35	private MemoryTracker memoryTracker;
36	36
37	37	private int progressTicks = 0;
—	—	@@ -95,8 +95,8 @@
96	96	}
97	97
98	98	public void reset() {
99		~~- pageTracker = new ImportProgressTracker("pages");~~
100		~~- bulkTracker = new ImportProgressTracker("chars");~~
	99	+ pageTracker = new ChunkedProgressRateTracker("pages");
	100	+ bulkTracker = new ChunkedProgressRateTracker("chars");
101	101	memoryTracker = new MemoryTracker();
102	102	progressTicks = 0;
103	103	}
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/ChunkedProgressRateTracker.java
—	—	@@ -0,0 +1,56 @@
	2	+/**
	3	+ *
	4	+ */
	5	+package de.brightbyte.wikiword.processor;
	6	+
	7	+import java.text.MessageFormat;
	8	+
	9	+import de.brightbyte.job.Progress;
	10	+import de.brightbyte.job.ProgressRateTracker;
	11	+
	12	+public class ChunkedProgressRateTracker extends ProgressRateTracker {
	13	+ protected long counter = 0;
	14	+ protected long timestamp = 0;
	15	+ protected String name;
	16	+
	17	+ public ChunkedProgressRateTracker(String name) {
	18	+ this.name = name;
	19	+ }
	20	+
	21	+ public void step() {
	22	+ step(1);
	23	+ }
	24	+
	25	+ public void step(int c) {
	26	+ counter+= c;
	27	+ }
	28	+
	29	+ public long getCurrentChunkSize() {
	30	+ return counter;
	31	+ }
	32	+
	33	+ public long getLastChunkTime() {
	34	+ return timestamp;
	35	+ }
	36	+
	37	+ public boolean chunkIf(long counter, int sec) {
	38	+ if ((counter>0 && this.counter>=0 && this.counter >= counter)
	39	+ \|\| (sec>0 && this.timestamp>0 && (System.currentTimeMillis() - this.timestamp)>sec*1000)) {
	40	+ chunk();
	41	+ return true;
	42	+ } else {
	43	+ return false;
	44	+ }
	45	+ }
	46	+
	47	+ public void chunk() {
	48	+ super.progress(new Progress.Event(Progress.PROGRESS, null, name, 1, position+counter, null));
	49	+ counter = 0;
	50	+ timestamp = System.currentTimeMillis();
	51	+ }
	52	+
	53	+ @Override
	54	+ public String toString() {
	55	+ return MessageFormat.format("{0}: {1,number,0} ({2,number,0.0}/sec, currently {3,number,0.0}/sec)", name, position, getAverageRate(), getCurrentRate());
	56	+ }
	57	+}
\ No newline at end of file
Property changes on: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/processor/ChunkedProgressRateTracker.java
___________________________________________________________________
Name: svn:mergeinfo
1	58	+
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/builder/ConceptImporter.java
—	—	@@ -16,7 +16,7 @@
17	17	import de.brightbyte.wikiword.TweakSet;
18	18	import de.brightbyte.wikiword.analyzer.WikiPage;
19	19	import de.brightbyte.wikiword.analyzer.WikiTextAnalyzer;
20		~~-import de.brightbyte.wikiword.processor.ImportProgressTracker;~~
	20	+import de.brightbyte.wikiword.processor.ChunkedProgressRateTracker;
21	21	import de.brightbyte.wikiword.schema.AliasScope;
22	22	import de.brightbyte.wikiword.store.builder.IncrementalStoreBuilder;
23	23	import de.brightbyte.wikiword.store.builder.LocalConceptStoreBuilder;
—	—	@@ -29,9 +29,9 @@
30	30	private boolean storeFlatText = true;
31	31	private boolean storeRawText = true;
32	32
33		~~- protected ImportProgressTracker conceptTracker;~~
34		~~- protected ImportProgressTracker linkTracker;~~
35		~~- protected ImportProgressTracker propertyTracker;~~
	33	+ protected ChunkedProgressRateTracker conceptTracker;
	34	+ protected ChunkedProgressRateTracker linkTracker;
	35	+ protected ChunkedProgressRateTracker propertyTracker;
36	36
37	37	protected LocalConceptStoreBuilder store;
38	38	protected PropertyStoreBuilder propertyStore;
—	—	@@ -178,9 +178,9 @@
179	179	@Override
180	180	public void reset() {
181	181	super.reset();
182		~~- conceptTracker = new ImportProgressTracker("concepts");~~
183		~~- linkTracker = new ImportProgressTracker("links");~~
184		~~- propertyTracker = new ImportProgressTracker("properties");~~
	182	+ conceptTracker = new ChunkedProgressRateTracker("concepts");
	183	+ linkTracker = new ChunkedProgressRateTracker("links");
	184	+ propertyTracker = new ChunkedProgressRateTracker("properties");
185	185	}
186	186
187	187	@Override
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/StreamProcessorApp.java
—	—	@@ -26,6 +26,9 @@
27	27	protected DataCursor<? extends I> cursor;
28	28	protected DataSink<? super O> sink;
29	29
	30	+ protected boolean usingStdin;
	31	+ protected boolean usingStdout;
	32	+
30	33	protected InputFileHelper inputHelper;
31	34
32	35	public StreamProcessorApp(boolean allowGlobal, boolean allowLocal) {
—	—	@@ -55,8 +58,14 @@
56	59	protected Writer getOutputWriter() throws FileNotFoundException, UnsupportedEncodingException {
57	60	if (outputWriter==null) {
58	61	File f = getOutputFile();
59		~~- if (f==null) outputWriter = ConsoleIO.writer;~~
60		~~- else outputWriter = new OutputStreamWriter(getOutputStream(), getOutputFileEncoding());~~
	62	+ if (f==null) {
	63	+ outputWriter = ConsoleIO.writer;
	64	+ usingStdout = true;
	65	+ } else {
	66	+ OutputStream out = getOutputStream();
	67	+ outputWriter = new OutputStreamWriter(out, getOutputFileEncoding());
	68	+ usingStdout = out == System.out;
	69	+ }
61	70	}
62	71
63	72	return outputWriter;
—	—	@@ -65,9 +74,12 @@
66	75	protected OutputStream getOutputStream() throws FileNotFoundException {
67	76	if (outputStream==null) {
68	77	File f = getOutputFile();
69		~~- if (f==null) outputStream = System.out;~~
70		~~- else {~~
	78	+ if (f==null) {
	79	+ outputStream = System.out;
	80	+ usingStdout = true;
	81	+ } else {
71	82	outputStream = new BufferedOutputStream(new FileOutputStream(f, args.isSet("append")));
	83	+ usingStdout = false;
72	84	info("Writing output to "+f);
73	85	}
74	86	}
—	—	@@ -78,8 +90,14 @@
79	91	protected Reader getInputReader() throws IOException {
80	92	if (inputReader==null) {
81	93	File f = getOutputFile();
82		~~- if (f==null) inputReader = ConsoleIO.newReader();~~
83		~~- else inputReader = new InputStreamReader(getInputStream(), getOutputFileEncoding());~~
	94	+ if (f==null) {
	95	+ inputReader = ConsoleIO.newReader();
	96	+ usingStdin = true;
	97	+ } else {
	98	+ InputStream in = getInputStream();
	99	+ inputReader = new InputStreamReader(in, getOutputFileEncoding());
	100	+ usingStdin = (in == System.in);
	101	+ }
84	102	}
85	103
86	104	return inputReader;
—	—	@@ -88,10 +106,13 @@
89	107	protected InputStream getInputStream() throws IOException {
90	108	if (inputStream==null) {
91	109	File f = getOutputFile();
92		~~- if (f==null) inputStream = System.in;~~
93		~~- else {~~
	110	+ if (f==null) {
	111	+ inputStream = System.in;
	112	+ usingStdin = true;
	113	+ } else {
94	114	inputStream = inputHelper.openFile(f);
95	115	info("Reading input from "+f);
	116	+ usingStdin = false;
96	117	}
97	118	}
98	119
Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/store/builder/DatabaseProximityStoreBuilder.java
—	—	@@ -12,7 +12,7 @@
13	13	import de.brightbyte.db.RelationTable;
14	14	import de.brightbyte.util.PersistenceException;
15	15	import de.brightbyte.wikiword.TweakSet;
16		~~-import de.brightbyte.wikiword.processor.ImportProgressTracker;~~
	16	+import de.brightbyte.wikiword.processor.ChunkedProgressRateTracker;
17	17	import de.brightbyte.wikiword.schema.ProximityStoreSchema;
18	18
19	19	public class DatabaseProximityStoreBuilder
—	—	@@ -416,8 +416,8 @@
417	417	protected int lastId ;
418	418	protected int level;
419	419
420		~~- protected ImportProgressTracker conceptTracker;~~
421		~~- protected ImportProgressTracker featureTracker;~~
	420	+ protected ChunkedProgressRateTracker conceptTracker;
	421	+ protected ChunkedProgressRateTracker featureTracker;
422	422
423	423	protected FeatureBuilder builder;
424	424
—	—	@@ -427,8 +427,8 @@
428	428	this.name = name;
429	429	this.level = level;
430	430	this.conceptTable = conceptStore.getDatabaseAccess().getTable("concept");
431		~~- this.conceptTracker = new ImportProgressTracker("concepts");~~
432		~~- this.featureTracker = new ImportProgressTracker("features");~~
	431	+ this.conceptTracker = new ChunkedProgressRateTracker("concepts");
	432	+ this.featureTracker = new ChunkedProgressRateTracker("features");
433	433	this.builder = builder;
434	434	}
435	435

Status & tagging log

17:57, 5 April 2010 😂 (talk | contribs) changed the status of r64506 [removed: new added: deferred]