| Index: trunk/lucene-search-3/src/main/java/org/wikimedia/lsearch/analyzers/Aggregate.java |
| — | — | @@ -8,6 +8,8 @@ |
| 9 | 9 | import org.apache.lucene.analysis.Analyzer; |
| 10 | 10 | import org.apache.lucene.analysis.Token; |
| 11 | 11 | import org.apache.lucene.analysis.TokenStream; |
| | 12 | +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; |
| | 13 | +import org.apache.lucene.analysis.tokenattributes.TermAttribute; |
| 12 | 14 | import org.wikimedia.lsearch.config.IndexId; |
| 13 | 15 | |
| 14 | 16 | /** |
| — | — | @@ -67,14 +69,40 @@ |
| 68 | 70 | this.flags = flags; |
| 69 | 71 | } |
| 70 | 72 | |
| 71 | | - private ArrayList<Token> toTokenArray(TokenStream stream) throws IOException { |
| | 73 | + private ArrayList<Token> toTokenArray(TokenStream tokenStream) throws IOException { |
| 72 | 74 | ArrayList<Token> tt = new ArrayList<Token>(); |
| 73 | | - Token t = null; |
| 74 | | - while( (t = stream.next()) != null && tt.size() < 0xff-1){ |
| 75 | | - tt.add(t); |
| 76 | | - } |
| | 75 | + |
| | 76 | + // TODO: remove 2.9.x api |
| | 77 | + |
| | 78 | + /** |
| | 79 | + Token reusableToken = new Token(); |
| | 80 | + while ((reusableToken = tokenStream.next(reusableToken)) != null |
| | 81 | + && tt.size() < 0xff - 1) { |
| | 82 | + tt.add(reusableToken); |
| | 83 | + } |
| | 84 | + |
| | 85 | + */ |
| | 86 | + OffsetAttribute offsetAttribute = (OffsetAttribute) tokenStream.getAttribute(OffsetAttribute.class); |
| | 87 | + |
| | 88 | + TermAttribute termAttribute = (TermAttribute) tokenStream.getAttribute(TermAttribute.class); |
| | 89 | + //TODO: update above to 3.5 api by replacing with |
| | 90 | + //CharTermAttribute charTermAttribute = (CharTermAttribute) tokenStream.getAttribute(CharTermAttribute.class); |
| | 91 | + |
| | 92 | + |
| | 93 | + while (tokenStream.incrementToken() && tt.size() < 0xff - 1) { |
| | 94 | + |
| | 95 | + tt.add(new Token(termAttribute.term(),offsetAttribute.startOffset(),offsetAttribute.endOffset())); |
| | 96 | + //TODO: update above to 3.5 api replacing with |
| | 97 | + //tt.add(new Token(charTermAttribute.toString(),offsetAttribute.startOffset(),offsetAttribute.endOffset())); |
| | 98 | + |
| | 99 | + |
| | 100 | + } |
| | 101 | + //} |
| 77 | 102 | return tt; |
| 78 | 103 | } |
| | 104 | + |
| | 105 | + |
| | 106 | + |
| 79 | 107 | |
| 80 | 108 | /** Number of tokens */ |
| 81 | 109 | public int length(){ |