| Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/WordSenseIndexer.java |
| — | — | @@ -1,8 +1,6 @@ |
| 2 | 2 | package de.brightbyte.wikiword.extract; |
| 3 | 3 | |
| 4 | | -import java.io.FileNotFoundException; |
| 5 | 4 | import java.io.IOException; |
| 6 | | -import java.io.UnsupportedEncodingException; |
| 7 | 5 | import java.text.ParseException; |
| 8 | 6 | import java.util.Collections; |
| 9 | 7 | import java.util.List; |
| — | — | @@ -11,10 +9,8 @@ |
| 12 | 10 | import de.brightbyte.data.cursor.DataCursor; |
| 13 | 11 | import de.brightbyte.data.cursor.DataSink; |
| 14 | 12 | import de.brightbyte.data.measure.Measure; |
| 15 | | -import de.brightbyte.io.ConsoleIO; |
| 16 | 13 | import de.brightbyte.io.LineCursor; |
| 17 | 14 | import de.brightbyte.io.LineSink; |
| 18 | | -import de.brightbyte.io.OutputSink; |
| 19 | 15 | import de.brightbyte.text.Chunker; |
| 20 | 16 | import de.brightbyte.text.RegularExpressionChunker; |
| 21 | 17 | import de.brightbyte.util.PersistenceException; |
| — | — | @@ -46,18 +42,18 @@ |
| 47 | 43 | } |
| 48 | 44 | |
| 49 | 45 | @Override |
| 50 | | - protected DataSink<String> openSink() throws PersistenceException { |
| | 46 | + protected DataSink<String> openSink(int paramIndex) throws PersistenceException { |
| 51 | 47 | try { |
| 52 | | - return new LineSink(getOutputWriter()); |
| | 48 | + return new LineSink(getOutputWriter(paramIndex)); |
| 53 | 49 | } catch (IOException e) { |
| 54 | 50 | throw new PersistenceException(e); |
| 55 | 51 | } |
| 56 | 52 | } |
| 57 | 53 | |
| 58 | 54 | @Override |
| 59 | | - protected DataCursor<String> openCursor() throws PersistenceException { |
| | 55 | + protected DataCursor<String> openCursor(int paramIndex) throws PersistenceException { |
| 60 | 56 | try { |
| 61 | | - return new LineCursor(getInputReader()); |
| | 57 | + return new LineCursor(getInputReader(paramIndex)); |
| 62 | 58 | } catch (IOException e) { |
| 63 | 59 | throw new PersistenceException(e); |
| 64 | 60 | } |
| Index: trunk/WikiWord/WikiWordBuilder/src/main/java/de/brightbyte/wikiword/extract/StreamProcessorApp.java |
| — | — | @@ -16,7 +16,6 @@ |
| 17 | 17 | import de.brightbyte.data.cursor.DataCursor; |
| 18 | 18 | import de.brightbyte.data.cursor.DataSink; |
| 19 | 19 | import de.brightbyte.io.ConsoleIO; |
| 20 | | -import de.brightbyte.io.LogOutput; |
| 21 | 20 | import de.brightbyte.util.PersistenceException; |
| 22 | 21 | import de.brightbyte.wikiword.StoreBackedApp; |
| 23 | 22 | import de.brightbyte.wikiword.builder.InputFileHelper; |
| — | — | @@ -37,33 +36,44 @@ |
| 38 | 37 | } |
| 39 | 38 | |
| 40 | 39 | |
| 41 | | - protected File getOutputFile() { |
| | 40 | + protected File getOutputFile(int paramIndex) { |
| 42 | 41 | if (outputFile==null) { |
| 43 | | - if (args.getParameterCount()>2) { |
| 44 | | - outputFile = new File(args.getParameter(2)); |
| | 42 | + if (args.getParameterCount()>=paramIndex) { |
| | 43 | + String f = args.getParameter(paramIndex); |
| | 44 | + if (!f.equals("-")) outputFile = new File(f); |
| 45 | 45 | } |
| 46 | 46 | } |
| 47 | 47 | return outputFile; |
| 48 | 48 | } |
| 49 | 49 | |
| | 50 | + protected String getInputPath(int paramIndex) { |
| | 51 | + if (inputPath==null) { |
| | 52 | + if (args.getParameterCount()>=paramIndex) { |
| | 53 | + inputPath = args.getParameter(paramIndex); |
| | 54 | + } |
| | 55 | + } |
| | 56 | + return inputPath; |
| | 57 | + } |
| | 58 | + |
| 50 | 59 | protected String getOutputFileEncoding() { |
| 51 | 60 | return args.getStringOption("outputencoding", "UTF-8"); |
| 52 | 61 | } |
| 53 | 62 | |
| | 63 | + protected String inputPath; |
| 54 | 64 | protected File outputFile; |
| 55 | 65 | protected Writer outputWriter; |
| 56 | 66 | protected OutputStream outputStream; |
| 57 | 67 | private InputStream inputStream; |
| 58 | 68 | private Reader inputReader; |
| 59 | 69 | |
| 60 | | - protected Writer getOutputWriter() throws FileNotFoundException, UnsupportedEncodingException { |
| | 70 | + protected Writer getOutputWriter(int paramIndex) throws FileNotFoundException, UnsupportedEncodingException { |
| 61 | 71 | if (outputWriter==null) { |
| 62 | | - File f = getOutputFile(); |
| | 72 | + File f = getOutputFile(paramIndex); |
| 63 | 73 | if (f==null) { |
| 64 | 74 | outputWriter = ConsoleIO.writer; |
| 65 | 75 | usingStdout = true; |
| 66 | 76 | } else { |
| 67 | | - OutputStream out = getOutputStream(); |
| | 77 | + OutputStream out = getOutputStream(paramIndex); |
| 68 | 78 | outputWriter = new OutputStreamWriter(out, getOutputFileEncoding()); |
| 69 | 79 | usingStdout = out == System.out; |
| 70 | 80 | } |
| — | — | @@ -76,9 +86,9 @@ |
| 77 | 87 | return outputWriter; |
| 78 | 88 | } |
| 79 | 89 | |
| 80 | | - protected OutputStream getOutputStream() throws FileNotFoundException { |
| | 90 | + protected OutputStream getOutputStream(int paramIndex) throws FileNotFoundException { |
| 81 | 91 | if (outputStream==null) { |
| 82 | | - File f = getOutputFile(); |
| | 92 | + File f = getOutputFile(paramIndex); |
| 83 | 93 | if (f==null) { |
| 84 | 94 | outputStream = System.out; |
| 85 | 95 | usingStdout = true; |
| — | — | @@ -92,14 +102,14 @@ |
| 93 | 103 | return outputStream; |
| 94 | 104 | } |
| 95 | 105 | |
| 96 | | - protected Reader getInputReader() throws IOException { |
| | 106 | + protected Reader getInputReader(int paramIndex) throws IOException { |
| 97 | 107 | if (inputReader==null) { |
| 98 | | - File f = getOutputFile(); |
| 99 | | - if (f==null) { |
| | 108 | + String path = getInputPath(paramIndex); |
| | 109 | + if (path==null || path.equals("-")) { |
| 100 | 110 | inputReader = ConsoleIO.newReader(); |
| 101 | 111 | usingStdin = true; |
| 102 | 112 | } else { |
| 103 | | - InputStream in = getInputStream(); |
| | 113 | + InputStream in = getInputStream(paramIndex); |
| 104 | 114 | inputReader = new InputStreamReader(in, getOutputFileEncoding()); |
| 105 | 115 | usingStdin = (in == System.in); |
| 106 | 116 | } |
| — | — | @@ -108,15 +118,15 @@ |
| 109 | 119 | return inputReader; |
| 110 | 120 | } |
| 111 | 121 | |
| 112 | | - protected InputStream getInputStream() throws IOException { |
| | 122 | + protected InputStream getInputStream(int paramIndex) throws IOException { |
| 113 | 123 | if (inputStream==null) { |
| 114 | | - File f = getOutputFile(); |
| 115 | | - if (f==null) { |
| | 124 | + String path = getInputPath(paramIndex); |
| | 125 | + if (path==null || path.equals("-")) { |
| 116 | 126 | inputStream = System.in; |
| 117 | 127 | usingStdin = true; |
| 118 | 128 | } else { |
| 119 | | - inputStream = inputHelper.openFile(f); |
| 120 | | - info("Reading input from "+f); |
| | 129 | + inputStream = inputHelper.open(path); |
| | 130 | + info("Reading input from "+path); |
| 121 | 131 | usingStdin = false; |
| 122 | 132 | } |
| 123 | 133 | } |
| — | — | @@ -127,20 +137,20 @@ |
| 128 | 138 | @Override |
| 129 | 139 | public void run() throws Exception { |
| 130 | 140 | init(); |
| 131 | | - open(); |
| | 141 | + open(1); // 0 = corpus, 1 = input, 2 = output |
| 132 | 142 | |
| 133 | 143 | runTransfer(cursor); |
| 134 | 144 | |
| 135 | 145 | close(); |
| 136 | 146 | } |
| 137 | 147 | |
| 138 | | - protected void open() throws PersistenceException { |
| 139 | | - cursor = openCursor(); |
| 140 | | - sink = openSink(); |
| | 148 | + protected void open(int paramOffset) throws PersistenceException { |
| | 149 | + cursor = openCursor(paramOffset); |
| | 150 | + sink = openSink(paramOffset+1); |
| 141 | 151 | } |
| 142 | 152 | |
| 143 | | - protected abstract DataCursor<? extends I> openCursor() throws PersistenceException; |
| 144 | | - protected abstract DataSink<? super O> openSink() throws PersistenceException; |
| | 153 | + protected abstract DataCursor<? extends I> openCursor(int paramIndex) throws PersistenceException; |
| | 154 | + protected abstract DataSink<? super O> openSink(int paramIndex) throws PersistenceException; |
| 145 | 155 | |
| 146 | 156 | protected void init() throws Exception { |
| 147 | 157 | // noop |