Apache Lucene open-source search software
Add TokenStreamColumn for columnar batch indexing (#16224)
Tim Brooks70d043335bcf
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 991bca34148..1db7fb6613b 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -313,6 +313,8 @@ API Changes* GITHUB#16227: Add dense indexing path for BinaryColumn while using experimental column api (Tim Brooks)+* GITHUB#16224 Add TokenStreamColumn for experimental columnar batch indexing. (Tim Brooks)+New Features---------------------diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.javaindex 09be21ce819..4fc0ef728dd 100644--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnFieldAdapter.java@@ -60,6 +60,8 @@ public abstract sealed class ColumnFieldAdapter extends Fielddc.fieldType(),dc.fieldType().stored() ? dc.storedType() : null,dictionaryCursor(dc.tuples(), dc.dictionary()));+ case TokenStreamColumn tsc ->+ new BinaryColumnAdapter(tsc.name(), tsc.fieldType(), tsc.tuples());default ->throw new IllegalArgumentException("Unknown column type: " + column.getClass().getName());};@@ -147,6 +149,9 @@ final class LongColumnAdapter extends ColumnFieldAdapter {final class BinaryColumnAdapter extends ColumnFieldAdapter {private final ObjectTupleCursor<BytesRef> cursor;+ // Non-null only with a TokenStreamColumn; mutually exclusive with cursor. When set, this+ // adapter feeds the inverter the caller's TokenStream directly instead of decoding bytes+ private final ObjectTupleCursor<TokenStream> tokenStreamCursor;private final StoredValue reusableStoredValue;private final StoredValue.Type storedType;private final boolean tokenized;@@ -161,12 +166,29 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {ObjectTupleCursor<BytesRef> cursor) {super(name, fieldType);this.cursor = cursor;+ this.tokenStreamCursor = null;this.tokenized = fieldType.tokenized();this.indexed = fieldType.indexOptions() != IndexOptions.NONE;this.storedType = storedType;this.reusableStoredValue = storedType != null ? newReusableStoredValue(storedType) : null;}+ /**+ * Token-stream mode: the column yields a {@link TokenStream} per doc for inversion. A {@link+ * TokenStreamColumn} is validated to be inverted-only, so {@code tokenized}/{@code indexed} are+ * always true.+ */+ BinaryColumnAdapter(+ String name, IndexableFieldType fieldType, ObjectTupleCursor<TokenStream> tokenStreamCursor) {+ super(name, fieldType);+ this.cursor = null;+ this.tokenStreamCursor = tokenStreamCursor;+ this.tokenized = true;+ this.indexed = true;+ this.storedType = null;+ this.reusableStoredValue = null;+ }+private static StoredValue newReusableStoredValue(StoredValue.Type type) {return switch (type) {case STRING -> new StoredValue("");@@ -177,6 +199,7 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {}private String decodedString() {+ assert cursor != null : "decodedString() unreachable in token-stream mode";if (cachedString == null) {BytesRef ref = cursor.value();cachedString = new String(ref.bytes, ref.offset, ref.length, StandardCharsets.UTF_8);@@ -187,16 +210,19 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {@Overridepublic int nextDoc() {cachedString = null;- return cursor.nextDoc();+ return tokenStreamCursor != null ? tokenStreamCursor.nextDoc() : cursor.nextDoc();}@Overridepublic BytesRef binaryValue() {- return cursor.value();+ return tokenStreamCursor != null ? null : cursor.value();}@Overridepublic String stringValue() {+ if (tokenStreamCursor != null) {+ return null;+ }return tokenized ? decodedString() : null;}@@ -205,6 +231,7 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {if (reusableStoredValue == null) {return null;}+ assert cursor != null : "storedValue() unreachable in token-stream mode";switch (storedType) {case STRING -> reusableStoredValue.setStringValue(decodedString());case BINARY -> reusableStoredValue.setBinaryValue(cursor.value());@@ -224,6 +251,10 @@ final class BinaryColumnAdapter extends ColumnFieldAdapter {@Overridepublic TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) {+ if (tokenStreamCursor != null) {+ // Caller-supplied token stream: hand it straight to the inverter, bypassing the analyzer.+ return tokenStreamCursor.value();+ }if (tokenized) {return analyzer.tokenStream(name(), stringValue());}diff --git a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.javaindex d0e9d262be1..2d62e81a311 100644--- a/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java+++ b/lucene/core/src/java/org/apache/lucene/document/column/ColumnValidation.java@@ -246,6 +246,30 @@ public final class ColumnValidation {}}+ /** Validates a {@link TokenStreamColumn} against the field type it will feed. */+ public static void validateTokenStreamColumn(+ TokenStreamColumn column, IndexableFieldType fieldType) {+ if (fieldType.indexOptions() == IndexOptions.NONE || fieldType.tokenized() == false) {+ throw new IllegalArgumentException(+ "TokenStreamColumn \""+ + column.name()+ + "\" requires indexOptions != NONE and tokenized == true; got indexOptions="+ + fieldType.indexOptions()+ + ", tokenized="+ + fieldType.tokenized());+ }+ if (fieldType.stored()+ || fieldType.docValuesType() != DocValuesType.NONE+ || fieldType.pointDimensionCount() != 0+ || fieldType.vectorDimension() != 0) {+ throw new IllegalArgumentException(+ "TokenStreamColumn \""+ + column.name()+ + "\" must be inverted-only: stored=false, docValuesType=NONE,"+ + " pointDimensionCount=0, vectorDimension=0");+ }+ }+/** Validates a {@link VectorColumn} against the field type it will feed. */public static void validateVectorColumn(VectorColumn<?> column, IndexableFieldType fieldType) {if (fieldType.vectorDimension() <= 0) {diff --git a/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.java b/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.javanew file mode 100644index 00000000000..da46e8bf5d9--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/document/column/TokenStreamColumn.java@@ -0,0 +1,60 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document.column;++import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.IndexableFieldType;++/**+ * A {@link Column} that provides caller-supplied {@link TokenStream}s for term inversion. This is+ * the columnar analogue of setting a custom {@link TokenStream} on a {@link+ * org.apache.lucene.document.Field}: the per-doc tokens are fed straight to the inverter without+ * going through an {@link org.apache.lucene.analysis.Analyzer}.+ *+ * <p>Token-stream columns are inverted-index-only: the field type must declare {@code+ * indexOptions() != NONE} and {@code tokenized() == true}, and must not also set doc values,+ * points, be stored, or declare vectors. To both store a value and invert a custom token stream+ * under one field name, add a separate stored {@link BinaryColumn} (or {@link LongColumn}) with the+ * same name in the same batch — each column carries a distinct indexing aspect.+ *+ * @lucene.experimental+ */+public abstract class TokenStreamColumn extends Column {++ /**+ * Creates a TokenStreamColumn with the given field name, type, and density.+ *+ * @throws IllegalArgumentException if {@code fieldType} is not indexed or not tokenized+ */+ protected TokenStreamColumn(String name, IndexableFieldType fieldType, Density density) {+ super(name, fieldType, density);+ if (fieldType.indexOptions() == IndexOptions.NONE || fieldType.tokenized() == false) {+ throw new IllegalArgumentException(+ "TokenStreamColumn \""+ + name+ + "\" requires fieldType.indexOptions() != NONE and fieldType.tokenized() == true; got"+ + " indexOptions="+ + fieldType.indexOptions()+ + ", tokenized="+ + fieldType.tokenized());+ }+ }++ /** Returns a fresh tuple cursor starting at the beginning of the batch. */+ public abstract ObjectTupleCursor<TokenStream> tuples();+}diff --git a/lucene/core/src/java/org/apache/lucene/document/column/package-info.java b/lucene/core/src/java/org/apache/lucene/document/column/package-info.javaindex 31d2677dca9..f498b6147b7 100644--- a/lucene/core/src/java/org/apache/lucene/document/column/package-info.java+++ b/lucene/core/src/java/org/apache/lucene/document/column/package-info.java@@ -43,6 +43,10 @@* inversion, and stored binary or string fields.* <li>{@link org.apache.lucene.document.column.VectorColumn} — KNN vectors (FLOAT32 or BYTE* encoding); vector-only field type.+ * <li>{@link org.apache.lucene.document.column.TokenStreamColumn} — caller-supplied {@link+ * org.apache.lucene.analysis.TokenStream}s for term inversion (the columnar analogue of a+ * custom token stream on a {@link org.apache.lucene.document.Field}); inverted-index-only+ * field type.* </ul>** <h2>Cursors</h2>diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.javaindex f7008fe339b..9f99330fc2f 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexingChain.java@@ -56,6 +56,7 @@ import org.apache.lucene.document.column.LongValuesCursor;import org.apache.lucene.document.column.ObjectTupleCursor;import org.apache.lucene.document.column.OrdinalsCursor;import org.apache.lucene.document.column.OrdinalsTupleCursor;+import org.apache.lucene.document.column.TokenStreamColumn;import org.apache.lucene.document.column.VectorColumn;import org.apache.lucene.search.DocIdSetIterator;import org.apache.lucene.search.Sort;@@ -727,6 +728,7 @@ final class IndexingChain implements Accountable {case LongColumn lc -> ColumnValidation.validateLongColumn(lc, fieldType);case DictionaryColumn dc -> ColumnValidation.validateDictionaryColumn(dc, fieldType);case VectorColumn<?> vc -> ColumnValidation.validateVectorColumn(vc, fieldType);+ case TokenStreamColumn tsc -> ColumnValidation.validateTokenStreamColumn(tsc, fieldType);default ->throw new IllegalArgumentException("Unknown column type: " + column.getClass().getName());diff --git a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.javaindex ab3fa496a1f..aa2ff860b98 100644--- a/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java+++ b/lucene/core/src/test/org/apache/lucene/document/column/ColumnBatchTestUtil.java@@ -17,6 +17,7 @@package org.apache.lucene.document.column;import java.util.List;+import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.document.FieldType;import org.apache.lucene.document.StoredValue;import org.apache.lucene.index.IndexableFieldType;@@ -147,6 +148,38 @@ public class ColumnBatchTestUtil {}}+ /** Sparse {@link TokenStreamColumn} backed by parallel docId/stream arrays. */+ public static class ArrayTokenStreamColumn extends TokenStreamColumn {+ private final int[] docIds;+ private final TokenStream[] streams;++ public ArrayTokenStreamColumn(+ String name, IndexableFieldType fieldType, int[] docIds, TokenStream[] streams) {+ super(name, fieldType, Density.SPARSE);+ assert docIds.length == streams.length;+ this.docIds = docIds;+ this.streams = streams;+ }++ @Override+ public ObjectTupleCursor<TokenStream> tuples() {+ return new ObjectTupleCursor<>() {+ int pos = -1;++ @Override+ public int nextDoc() {+ pos++;+ return pos < docIds.length ? docIds[pos] : DocIdSetIterator.NO_MORE_DOCS;+ }++ @Override+ public TokenStream value() {+ return streams[pos];+ }+ };+ }+ }+/** Dense {@link BinaryColumn} backed by a fixed {@link BytesRef} array, one value per doc. */public static class ArrayDenseBinaryColumn extends BinaryColumn {private final BytesRef[] values;diff --git a/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.java b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.javanew file mode 100644index 00000000000..a582bf6b885--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/document/column/TestColumnBatchTokenStreamColumn.java@@ -0,0 +1,281 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.document.column;++import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayLongColumn;+import static org.apache.lucene.document.column.ColumnBatchTestUtil.ArrayTokenStreamColumn;+import static org.apache.lucene.document.column.ColumnBatchTestUtil.simpleBatch;++import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.analysis.TokenStream;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.FieldType;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.DocValuesType;+import org.apache.lucene.index.IndexOptions;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.LeafReader;+import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.PostingsEnum;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.Terms;+import org.apache.lucene.index.TermsEnum;+import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.PhraseQuery;+import org.apache.lucene.search.TermQuery;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.analysis.CannedTokenStream;+import org.apache.lucene.tests.analysis.MockAnalyzer;+import org.apache.lucene.tests.analysis.Token;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.util.BytesRef;++/** Tests for {@link TokenStreamColumn} batch indexing. */+public class TestColumnBatchTokenStreamColumn extends LuceneTestCase {++ private static FieldType invertedType() {+ FieldType type = new FieldType();+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);+ type.setTokenized(true);+ type.freeze();+ return type;+ }++ private static TokenStream canned(String... terms) {+ Token[] tokens = new Token[terms.length];+ int offset = 0;+ for (int i = 0; i < terms.length; i++) {+ tokens[i] = new Token(terms[i], offset, offset + terms[i].length());+ offset += terms[i].length() + 1;+ }+ int finalOffset = terms.length == 0 ? 0 : tokens[terms.length - 1].endOffset();+ return new CannedTokenStream(0, finalOffset, tokens);+ }++ private static List<Integer> positions(LeafReader leaf, String field, String term)+ throws IOException {+ List<Integer> all = new ArrayList<>();+ Terms terms = leaf.terms(field);+ if (terms == null) {+ return all;+ }+ TermsEnum te = terms.iterator();+ if (te.seekExact(new BytesRef(term)) == false) {+ return all;+ }+ PostingsEnum pe = te.postings(null, PostingsEnum.POSITIONS);+ int doc;+ while ((doc = pe.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {+ int freq = pe.freq();+ for (int i = 0; i < freq; i++) {+ all.add(doc * 1000 + pe.nextPosition());+ }+ }+ return all;+ }++ public void testTokenStreamInversion() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));++ int[] docIds = {0, 1};+ TokenStream[] streams = {canned("quick", "brown", "fox"), canned("lazy", "brown", "dog")};+ w.addBatch(simpleBatch(2, new ArrayTokenStreamColumn("body", invertedType(), docIds, streams)));++ DirectoryReader r = DirectoryReader.open(w);+ IndexSearcher s = new IndexSearcher(r);++ assertEquals(2, s.count(new TermQuery(new Term("body", "brown"))));+ assertEquals(1, s.count(new TermQuery(new Term("body", "quick"))));+ assertEquals(1, s.count(new TermQuery(new Term("body", "dog"))));+ assertEquals(0, s.count(new TermQuery(new Term("body", "missing"))));++ // Positions are honored: "quick brown" matches doc 0, "brown quick" does not.+ assertEquals(1, s.count(new PhraseQuery("body", "quick", "brown")));+ assertEquals(0, s.count(new PhraseQuery("body", "brown", "quick")));++ r.close();+ w.close();+ dir.close();+ }++ /** The inverted postings must be identical to adding the same token stream via addDocument. */+ public void testParityWithAddDocument() throws IOException {+ String[][] docs = {{"the", "quick", "brown", "fox"}, {"a", "lazy", "brown", "dog"}};++ Directory batchDir = newDirectory();+ IndexWriter bw = new IndexWriter(batchDir, newIndexWriterConfig(new MockAnalyzer(random())));+ TokenStream[] streams = {canned(docs[0]), canned(docs[1])};+ bw.addBatch(+ simpleBatch(+ 2, new ArrayTokenStreamColumn("body", invertedType(), new int[] {0, 1}, streams)));+ DirectoryReader br = DirectoryReader.open(bw);+ LeafReader batchLeaf = getOnlyLeafReader(br);++ Directory docDir = newDirectory();+ IndexWriter dw = new IndexWriter(docDir, newIndexWriterConfig(new MockAnalyzer(random())));+ for (String[] doc : docs) {+ Document d = new Document();+ d.add(new Field("body", canned(doc), invertedType()));+ dw.addDocument(d);+ }+ DirectoryReader dr = DirectoryReader.open(dw);+ LeafReader docLeaf = getOnlyLeafReader(dr);++ for (String term : new String[] {"the", "quick", "brown", "fox", "a", "lazy", "dog"}) {+ assertEquals(+ "positions mismatch for term " + term,+ positions(docLeaf, "body", term),+ positions(batchLeaf, "body", term));+ }++ br.close();+ bw.close();+ batchDir.close();+ dr.close();+ dw.close();+ docDir.close();+ }++ public void testMultiValuedAccumulatesFreq() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));++ // Doc 0 has two values (repeated batch doc-id), doc 1 has one value.+ int[] docIds = {0, 0, 1};+ TokenStream[] streams = {canned("brown", "fox"), canned("brown", "dog"), canned("red", "cat")};+ w.addBatch(simpleBatch(2, new ArrayTokenStreamColumn("body", invertedType(), docIds, streams)));++ DirectoryReader r = DirectoryReader.open(w);+ LeafReader leaf = getOnlyLeafReader(r);++ // "brown" appears once in each of doc 0's two values → freq 2 in doc 0.+ TermsEnum te = leaf.terms("body").iterator();+ assertTrue(te.seekExact(new BytesRef("brown")));+ PostingsEnum pe = te.postings(null, PostingsEnum.FREQS);+ assertEquals(0, pe.nextDoc());+ assertEquals(2, pe.freq());+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, pe.nextDoc());++ IndexSearcher s = new IndexSearcher(r);+ assertEquals(1, s.count(new TermQuery(new Term("body", "fox"))));+ assertEquals(1, s.count(new TermQuery(new Term("body", "cat"))));++ r.close();+ w.close();+ dir.close();+ }++ public void testMixedBatchKeepsOtherColumnsColumnar() throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));++ TokenStream[] streams = {canned("hello", "world"), canned("goodbye", "world")};+ w.addBatch(+ simpleBatch(+ 2,+ new ArrayTokenStreamColumn("body", invertedType(), new int[] {0, 1}, streams),+ new ArrayLongColumn(+ "num", NumericDocValuesField.TYPE, new int[] {0, 1}, new long[] {10, 20})));++ DirectoryReader r = DirectoryReader.open(w);+ LeafReader leaf = getOnlyLeafReader(r);+ IndexSearcher s = new IndexSearcher(r);++ assertEquals(2, s.count(new TermQuery(new Term("body", "world"))));++ NumericDocValues dv = leaf.getNumericDocValues("num");+ assertEquals(0, dv.nextDoc());+ assertEquals(10, dv.longValue());+ assertEquals(1, dv.nextDoc());+ assertEquals(20, dv.longValue());++ r.close();+ w.close();+ dir.close();+ }++ public void testConstructorRejectsNonIndexed() {+ FieldType type = new FieldType();+ type.setTokenized(true);+ type.freeze();+ expectThrows(+ IllegalArgumentException.class,+ () ->+ new ArrayTokenStreamColumn(+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}));+ }++ public void testConstructorRejectsNonTokenized() {+ FieldType type = new FieldType();+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);+ type.setTokenized(false);+ type.freeze();+ expectThrows(+ IllegalArgumentException.class,+ () ->+ new ArrayTokenStreamColumn(+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}));+ }++ public void testRejectsStored() throws IOException {+ FieldType type = new FieldType();+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);+ type.setTokenized(true);+ type.setStored(true);+ type.freeze();+ assertAddBatchRejected(type);+ }++ public void testRejectsDocValues() throws IOException {+ FieldType type = new FieldType();+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);+ type.setTokenized(true);+ type.setDocValuesType(DocValuesType.SORTED);+ type.freeze();+ assertAddBatchRejected(type);+ }++ public void testRejectsPoints() throws IOException {+ FieldType type = new FieldType();+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);+ type.setTokenized(true);+ type.setDimensions(1, Integer.BYTES);+ type.freeze();+ assertAddBatchRejected(type);+ }++ private void assertAddBatchRejected(FieldType type) throws IOException {+ Directory dir = newDirectory();+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));+ expectThrows(+ IllegalArgumentException.class,+ () ->+ w.addBatch(+ simpleBatch(+ 1,+ new ArrayTokenStreamColumn(+ "body", type, new int[] {0}, new TokenStream[] {canned("x")}))));+ w.close();+ dir.close();+ }+}