Skip to content

Apache Lucene open-source search software

Add TokenStreamColumn for columnar batch indexing (#16224)

Tim Brooks70d043335bcf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 991bca34148..1db7fb6613b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -313,6 +313,8 @@ API Changes
* GITHUB#16227: Add dense indexing path for BinaryColumn while using experimental column api (Tim Brooks)
+* GITHUB#16224 Add TokenStreamColumn for experimental columnar batch indexing. (Tim Brooks)
+
New Features
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java
index 09be21ce819..4fc0ef728dd 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java
@@ -60,6 +60,8 @@ public abstract sealed class ColumnFieldAdapter extends Field
dc.fieldType(),
dc.fieldType().stored() ? dc.storedType() : null,
dictionaryCursor(dc.tuples(), dc.dictionary()));
+ case TokenStreamColumn tsc ->
+ new BinaryColumnAdapter(tsc.name(), tsc.fieldType(), tsc.tuples());
default ->
throw new IllegalArgumentException("Unknown column type: " + column.getClass().getName());
};
@@ -147,6 +149,9 @@ final class LongColumnAdapter extends ColumnFieldAdapter {
final class BinaryColumnAdapter extends ColumnFieldAdapter {
private final ObjectTupleCursor<BytesRef> cursor;
+ // Non-null only with a TokenStreamColumn; mutually exclusive with cursor. When set, this
+ // adapter feeds the inverter the caller's TokenStream directly instead of decoding bytes
+ private final ObjectTupleCursor<TokenStream> tokenStreamCursor;
private final StoredValue reusableStoredValue;
private final StoredValue.Type storedType;
private final boolean tokenized;
@@ -161,12 +166,29 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {
ObjectTupleCursor<BytesRef> cursor) {
super(name, fieldType);
this.cursor = cursor;
+ this.tokenStreamCursor = null;
this.tokenized = fieldType.tokenized();
this.indexed = fieldType.indexOptions() != IndexOptions.NONE;
this.storedType = storedType;
this.reusableStoredValue = storedType != null ? newReusableStoredValue(storedType) : null;
}
+ /**
+ * Token-stream mode: the column yields a {@link TokenStream} per doc for inversion. A {@link
+ * TokenStreamColumn} is validated to be inverted-only, so {@code tokenized}/{@code indexed} are
+ * always true.
+ */
+ BinaryColumnAdapter(
+ String name, IndexableFieldType fieldType, ObjectTupleCursor<TokenStream> tokenStreamCursor) {
+ super(name, fieldType);
+ this.cursor = null;
+ this.tokenStreamCursor = tokenStreamCursor;
+ this.tokenized = true;
+ this.indexed = true;
+ this.storedType = null;
+ this.reusableStoredValue = null;
+ }
+
private static StoredValue newReusableStoredValue(StoredValue.Type type) {
return switch (type) {
case STRING -> new StoredValue("");
@@ -177,6 +199,7 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {
}
private String decodedString() {
+ assert cursor != null : "decodedString() unreachable in token-stream mode";
if (cachedString == null) {
BytesRef ref = cursor.value();
cachedString = new String(ref.bytes, ref.offset, ref.length, StandardCharsets.UTF_8);
@@ -187,16 +210,19 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {
@Override
public int nextDoc() {
cachedString = null;
- return cursor.nextDoc();
+ return tokenStreamCursor != null ? tokenStreamCursor.nextDoc() : cursor.nextDoc();
}
@Override
public BytesRef binaryValue() {
- return cursor.value();
+ return tokenStreamCursor != null ? null : cursor.value();
}
@Override
public String stringValue() {
+ if (tokenStreamCursor != null) {
+ return null;
+ }
return tokenized ? decodedString() : null;
}
@@ -205,6 +231,7 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {
if (reusableStoredValue == null) {
return null;
}
+ assert cursor != null : "storedValue() unreachable in token-stream mode";
switch (storedType) {
case STRING -> reusableStoredValue.setStringValue(decodedString());
case BINARY -> reusableStoredValue.setBinaryValue(cursor.value());
@@ -224,6 +251,10 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {
@Override
public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {
+ if (tokenStreamCursor != null) {
+ // Caller-supplied token stream: hand it straight to the inverter, bypassing the analyzer.
+ return tokenStreamCursor.value();
+ }
if (tokenized) {
return analyzer.tokenStream(name(), stringValue());
}
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
index d0e9d262be1..2d62e81a311 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java
@@ -246,6 +246,30 @@ public final class ColumnValidation {
}
}
+ /** Validates a {@link TokenStreamColumn} against the field type it will feed. */
+ public static void validateTokenStreamColumn(
+ TokenStreamColumn column, IndexableFieldType fieldType) {
+ if (fieldType.indexOptions() == IndexOptions.NONE || fieldType.tokenized() == false) {
+ throw new IllegalArgumentException(
+ "TokenStreamColumn \""
+ + column.name()
+ + "\" requires indexOptions != NONE and tokenized == true; got indexOptions="
+ + fieldType.indexOptions()
+ + ", tokenized="
+ + fieldType.tokenized());
+ }
+ if (fieldType.stored()
+ || fieldType.docValuesType() != DocValuesType.NONE
+ || fieldType.pointDimensionCount() != 0
+ || fieldType.vectorDimension() != 0) {
+ throw new IllegalArgumentException(
+ "TokenStreamColumn \""
+ + column.name()
+ + "\" must be inverted-only: stored=false, docValuesType=NONE,"
+ + " pointDimensionCount=0, vectorDimension=0");
+ }
+ }
+
/** Validates a {@link VectorColumn} against the field type it will feed. */
public static void validateVectorColumn(VectorColumn<?> column, IndexableFieldType fieldType) {
if (fieldType.vectorDimension() <= 0) {
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.java b/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.java
new file mode 100644
index 00000000000..da46e8bf5d9
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document.column;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableFieldType;
+
+/**
+ * A {@link Column} that provides caller-supplied {@link TokenStream}s for term inversion. This is
+ * the columnar analogue of setting a custom {@link TokenStream} on a {@link
+ * org.apache.lucene.document.Field}: the per-doc tokens are fed straight to the inverter without
+ * going through an {@link org.apache.lucene.analysis.Analyzer}.
+ *
+ * <p>Token-stream columns are inverted-index-only: the field type must declare {@code
+ * indexOptions() != NONE} and {@code tokenized() == true}, and must not also set doc values,
+ * points, be stored, or declare vectors. To both store a value and invert a custom token stream
+ * under one field name, add a separate stored {@link BinaryColumn} (or {@link LongColumn}) with the
+ * same name in the same batch — each column carries a distinct indexing aspect.
+ *
+ * @lucene.experimental
+ */
+public abstract class TokenStreamColumn extends Column {
+
+ /**
+ * Creates a TokenStreamColumn with the given field name, type, and density.
+ *
+ * @throws IllegalArgumentException if {@code fieldType} is not indexed or not tokenized
+ */
+ protected TokenStreamColumn(String name, IndexableFieldType fieldType, Density density) {
+ super(name, fieldType, density);
+ if (fieldType.indexOptions() == IndexOptions.NONE || fieldType.tokenized() == false) {
+ throw new IllegalArgumentException(
+ "TokenStreamColumn \""
+ + name
+ + "\" requires fieldType.indexOptions() != NONE and fieldType.tokenized() == true; got"
+ + " indexOptions="
+ + fieldType.indexOptions()
+ + ", tokenized="
+ + fieldType.tokenized());
+ }
+ }
+
+ /** Returns a fresh tuple cursor starting at the beginning of the batch. */
+ public abstract ObjectTupleCursor<TokenStream> tuples();
+}
diff --git a/lucene/core/src/java/org/apache/lucene/document/column/package-info.java b/lucene/core/src/java/org/apache/lucene/document/column/package-info.java
index 31d2677dca9..f498b6147b7 100644
--- a/lucene/core/src/java/org/apache/lucene/document/column/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/document/column/package-info.java
@@ -43,6 +43,10 @@
* inversion, and stored binary or string fields.
* <li>{@link org.apache.lucene.document.column.VectorColumn} — KNN vectors (FLOAT32 or BYTE
* encoding); vector-only field type.
+ * <li>{@link org.apache.lucene.document.column.TokenStreamColumn} — caller-supplied {@link
+ * org.apache.lucene.analysis.TokenStream}s for term inversion (the columnar analogue of a
+ * custom token stream on a {@link org.apache.lucene.document.Field}); inverted-index-only
+ * field type.
* </ul>
*
* <h2>Cursors</h2>
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
index f7008fe339b..9f99330fc2f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java
@@ -56,6 +56,7 @@ import org.apache.lucene.document.column.LongValuesCursor;
import org.apache.lucene.document.column.ObjectTupleCursor;
import org.apache.lucene.document.column.OrdinalsCursor;
import org.apache.lucene.document.column.OrdinalsTupleCursor;
+import org.apache.lucene.document.column.TokenStreamColumn;
import org.apache.lucene.document.column.VectorColumn;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
@@ -727,6 +728,7 @@ final class IndexingChain implements Accountable {
case LongColumn lc -> ColumnValidation.validateLongColumn(lc, fieldType);
case DictionaryColumn dc -> ColumnValidation.validateDictionaryColumn(dc, fieldType);
case VectorColumn<?> vc -> ColumnValidation.validateVectorColumn(vc, fieldType);
+ case TokenStreamColumn tsc -> ColumnValidation.validateTokenStreamColumn(tsc, fieldType);
default ->
throw new IllegalArgumentException(
"Unknown column type: " + column.getClass().getName());
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
index ab3fa496a1f..aa2ff860b98 100644
--- a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java
@@ -17,6 +17,7 @@
package org.apache.lucene.document.column;
import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.index.IndexableFieldType;
@@ -147,6 +148,38 @@ public class ColumnBatchTestUtil {
}
}
+ /** Sparse {@link TokenStreamColumn} backed by parallel docId/stream arrays. */
+ public static class ArrayTokenStreamColumn extends TokenStreamColumn {
+ private final int[] docIds;
+ private final TokenStream[] streams;
+
+ public ArrayTokenStreamColumn(
+ String name, IndexableFieldType fieldType, int[] docIds, TokenStream[] streams) {
+ super(name, fieldType, Density.SPARSE);
+ assert docIds.length == streams.length;
+ this.docIds = docIds;
+ this.streams = streams;
+ }
+
+ @Override
+ public ObjectTupleCursor<TokenStream> tuples() {
+ return new ObjectTupleCursor<>() {
+ int pos = -1;
+
+ @Override
+ public int nextDoc() {
+ pos++;
+ return pos < docIds.length ? docIds[pos] : DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public TokenStream value() {
+ return streams[pos];
+ }
+ };
+ }
+ }
+
/** Dense {@link BinaryColumn} backed by a fixed {@link BytesRef} array, one value per doc. */
public static class ArrayDenseBinaryColumn extends BinaryColumn {
private final BytesRef[] values;
diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.java
new file mode 100644
index 00000000000..a582bf6b885
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document.column;
+
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayLongColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayTokenStreamColumn;
+import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.analysis.CannedTokenStream;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
+import org.apache.lucene.tests.analysis.Token;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
+
+/** Tests for {@link TokenStreamColumn} batch indexing. */
+public class TestColumnBatchTokenStreamColumn extends LuceneTestCase {
+
+ private static FieldType invertedType() {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ type.setTokenized(true);
+ type.freeze();
+ return type;
+ }
+
+ private static TokenStream canned(String... terms) {
+ Token[] tokens = new Token[terms.length];
+ int offset = 0;
+ for (int i = 0; i < terms.length; i++) {
+ tokens[i] = new Token(terms[i], offset, offset + terms[i].length());
+ offset += terms[i].length() + 1;
+ }
+ int finalOffset = terms.length == 0 ? 0 : tokens[terms.length - 1].endOffset();
+ return new CannedTokenStream(0, finalOffset, tokens);
+ }
+
+ private static List<Integer> positions(LeafReader leaf, String field, String term)
+ throws IOException {
+ List<Integer> all = new ArrayList<>();
+ Terms terms = leaf.terms(field);
+ if (terms == null) {
+ return all;
+ }
+ TermsEnum te = terms.iterator();
+ if (te.seekExact(new BytesRef(term)) == false) {
+ return all;
+ }
+ PostingsEnum pe = te.postings(null, PostingsEnum.POSITIONS);
+ int doc;
+ while ((doc = pe.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ int freq = pe.freq();
+ for (int i = 0; i < freq; i++) {
+ all.add(doc * 1000 + pe.nextPosition());
+ }
+ }
+ return all;
+ }
+
+ public void testTokenStreamInversion() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ int[] docIds = {0, 1};
+ TokenStream[] streams = {canned("quick", "brown", "fox"), canned("lazy", "brown", "dog")};
+ w.addBatch(simpleBatch(2, new ArrayTokenStreamColumn("body", invertedType(), docIds, streams)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ IndexSearcher s = new IndexSearcher(r);
+
+ assertEquals(2, s.count(new TermQuery(new Term("body", "brown"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "quick"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "dog"))));
+ assertEquals(0, s.count(new TermQuery(new Term("body", "missing"))));
+
+ // Positions are honored: "quick brown" matches doc 0, "brown quick" does not.
+ assertEquals(1, s.count(new PhraseQuery("body", "quick", "brown")));
+ assertEquals(0, s.count(new PhraseQuery("body", "brown", "quick")));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ /** The inverted postings must be identical to adding the same token stream via addDocument. */
+ public void testParityWithAddDocument() throws IOException {
+ String[][] docs = {{"the", "quick", "brown", "fox"}, {"a", "lazy", "brown", "dog"}};
+
+ Directory batchDir = newDirectory();
+ IndexWriter bw = new IndexWriter(batchDir, newIndexWriterConfig(new MockAnalyzer(random())));
+ TokenStream[] streams = {canned(docs[0]), canned(docs[1])};
+ bw.addBatch(
+ simpleBatch(
+ 2, new ArrayTokenStreamColumn("body", invertedType(), new int[] {0, 1}, streams)));
+ DirectoryReader br = DirectoryReader.open(bw);
+ LeafReader batchLeaf = getOnlyLeafReader(br);
+
+ Directory docDir = newDirectory();
+ IndexWriter dw = new IndexWriter(docDir, newIndexWriterConfig(new MockAnalyzer(random())));
+ for (String[] doc : docs) {
+ Document d = new Document();
+ d.add(new Field("body", canned(doc), invertedType()));
+ dw.addDocument(d);
+ }
+ DirectoryReader dr = DirectoryReader.open(dw);
+ LeafReader docLeaf = getOnlyLeafReader(dr);
+
+ for (String term : new String[] {"the", "quick", "brown", "fox", "a", "lazy", "dog"}) {
+ assertEquals(
+ "positions mismatch for term " + term,
+ positions(docLeaf, "body", term),
+ positions(batchLeaf, "body", term));
+ }
+
+ br.close();
+ bw.close();
+ batchDir.close();
+ dr.close();
+ dw.close();
+ docDir.close();
+ }
+
+ public void testMultiValuedAccumulatesFreq() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ // Doc 0 has two values (repeated batch doc-id), doc 1 has one value.
+ int[] docIds = {0, 0, 1};
+ TokenStream[] streams = {canned("brown", "fox"), canned("brown", "dog"), canned("red", "cat")};
+ w.addBatch(simpleBatch(2, new ArrayTokenStreamColumn("body", invertedType(), docIds, streams)));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+
+ // "brown" appears once in each of doc 0's two values → freq 2 in doc 0.
+ TermsEnum te = leaf.terms("body").iterator();
+ assertTrue(te.seekExact(new BytesRef("brown")));
+ PostingsEnum pe = te.postings(null, PostingsEnum.FREQS);
+ assertEquals(0, pe.nextDoc());
+ assertEquals(2, pe.freq());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, pe.nextDoc());
+
+ IndexSearcher s = new IndexSearcher(r);
+ assertEquals(1, s.count(new TermQuery(new Term("body", "fox"))));
+ assertEquals(1, s.count(new TermQuery(new Term("body", "cat"))));
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMixedBatchKeepsOtherColumnsColumnar() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+
+ TokenStream[] streams = {canned("hello", "world"), canned("goodbye", "world")};
+ w.addBatch(
+ simpleBatch(
+ 2,
+ new ArrayTokenStreamColumn("body", invertedType(), new int[] {0, 1}, streams),
+ new ArrayLongColumn(
+ "num", NumericDocValuesField.TYPE, new int[] {0, 1}, new long[] {10, 20})));
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ IndexSearcher s = new IndexSearcher(r);
+
+ assertEquals(2, s.count(new TermQuery(new Term("body", "world"))));
+
+ NumericDocValues dv = leaf.getNumericDocValues("num");
+ assertEquals(0, dv.nextDoc());
+ assertEquals(10, dv.longValue());
+ assertEquals(1, dv.nextDoc());
+ assertEquals(20, dv.longValue());
+
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testConstructorRejectsNonIndexed() {
+ FieldType type = new FieldType();
+ type.setTokenized(true);
+ type.freeze();
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new ArrayTokenStreamColumn(
+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}));
+ }
+
+ public void testConstructorRejectsNonTokenized() {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ type.setTokenized(false);
+ type.freeze();
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ new ArrayTokenStreamColumn(
+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}));
+ }
+
+ public void testRejectsStored() throws IOException {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ type.setTokenized(true);
+ type.setStored(true);
+ type.freeze();
+ assertAddBatchRejected(type);
+ }
+
+ public void testRejectsDocValues() throws IOException {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ type.setTokenized(true);
+ type.setDocValuesType(DocValuesType.SORTED);
+ type.freeze();
+ assertAddBatchRejected(type);
+ }
+
+ public void testRejectsPoints() throws IOException {
+ FieldType type = new FieldType();
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ type.setTokenized(true);
+ type.setDimensions(1, Integer.BYTES);
+ type.freeze();
+ assertAddBatchRejected(type);
+ }
+
+ private void assertAddBatchRejected(FieldType type) throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+ expectThrows(
+ IllegalArgumentException.class,
+ () ->
+ w.addBatch(
+ simpleBatch(
+ 1,
+ new ArrayTokenStreamColumn(
+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}))));
+ w.close();
+ dir.close();
+ }
+}