Apache Solr open-source search software
SOLR-16059: FieldType.DefaultAnalyzer: correct reader.read() loop and… (#712)
Dawid Weissbb85382c3bef
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.javaindex bf4461ac809..5154562c262 100644--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java@@ -502,15 +502,36 @@ public abstract class FieldType extends FieldProperties {@Overridepublic TokenStreamComponents createComponents(String fieldName) {Tokenizer ts = new Tokenizer() {+ private boolean done = false;+ private int finalOffset;+final char[] cbuf = new char[maxChars];final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);final BytesTermAttribute bytesAtt = isPointField() ? addAttribute(BytesTermAttribute.class) : null;final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);+@Overridepublic boolean incrementToken() throws IOException {+ if (done) {+ return false;+ }+clearAttributes();- int n = input.read(cbuf,0,maxChars);- if (n<=0) return false;+ done = true;++ int max = maxChars;+ int n = 0;+ while (max > 0) {+ int len = input.read(cbuf, n, max);+ if (len < 0) break;+ max -= len;+ n += len;+ }++ if (n == 0) {+ return false;+ }+if (isPointField()) {BytesRef b = ((PointField)FieldType.this).toInternalByteRef(new String(cbuf, 0, n));bytesAtt.setBytesRef(b);@@ -518,9 +539,23 @@ public abstract class FieldType extends FieldProperties {String s = toInternal(new String(cbuf, 0, n));termAtt.setEmpty().append(s);}- offsetAtt.setOffset(correctOffset(0),correctOffset(n));++ finalOffset = correctOffset(n);+ offsetAtt.setOffset(correctOffset(0), finalOffset);return true;}++ @Override+ public void end() throws IOException {+ super.end();+ offsetAtt.setOffset(finalOffset, finalOffset);+ }++ @Override+ public void reset() throws IOException {+ super.reset();+ this.done = false;+ }};return new TokenStreamComponents(ts);diff --git a/solr/core/src/test/org/apache/solr/schema/StrFieldAnalyzerTest.java b/solr/core/src/test/org/apache/solr/schema/StrFieldAnalyzerTest.javanew file mode 100644index 00000000000..b893cfbdd5e--- /dev/null+++ b/solr/core/src/test/org/apache/solr/schema/StrFieldAnalyzerTest.java@@ -0,0 +1,36 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.schema;++import org.apache.lucene.analysis.Analyzer;+import org.apache.lucene.analysis.BaseTokenStreamTestCase;++import java.io.IOException;++public class StrFieldAnalyzerTest extends BaseTokenStreamTestCase {+ public void testOffsetSanity() throws IOException {+ Analyzer analyzer = new StrField().getIndexAnalyzer();+ assertTokenStreamContents(+ analyzer.tokenStream("fieldName", "abc"),+ new String[] {"abc"},+ new int[] {0},+ new int[] {3},+ new int[] {1},+ new int[] {1},+ 3);+ }+}