Apache Lucene open-source search software
Support DataInput as source for StoredField (#14213)
Tim Brooksf12e44b6515a
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 199822b658a..cd39edc3227 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -91,6 +91,8 @@ Improvements* GITHUB#14239: Hunspell's option to tolerate affix rule count mismatches wasimproved to tolerate more instances of this problem. (Robert Muir)+* GITHUB#14213: Allowing indexing stored-only StoredField directly from DataInput. (Tim Brooks)+Optimizations---------------------diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.javaindex 8c6ea3264ab..edab3b0d8c2 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene50/compressing/Lucene50CompressingStoredFieldsReader.java@@ -30,6 +30,7 @@ import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.index.StoredFieldVisitor;import org.apache.lucene.store.AlreadyClosedException;import org.apache.lucene.store.ByteArrayDataInput;@@ -290,7 +291,7 @@ public final class Lucene50CompressingStoredFieldsReader extends StoredFieldsReaswitch (bits & TYPE_MASK) {case BYTE_ARR:int length = in.readVInt();- visitor.binaryField(info, in, length);+ visitor.binaryField(info, new StoredFieldDataInput(in, length));break;case STRING:visitor.stringField(info, in.readString());diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.javaindex 6824b227f68..a9f9ce464a3 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java@@ -28,8 +28,8 @@ import org.apache.lucene.index.DocIDMerger;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.MergeState;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.index.StoredFieldVisitor;-import org.apache.lucene.store.DataInput;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.BytesRef;@@ -73,10 +73,11 @@ public abstract class StoredFieldsWriter implements Closeable, Accountable {/** Writes a stored double value. */public abstract void writeField(FieldInfo info, double value) throws IOException;- /** Writes a stored binary value from a {@link DataInput} and a {@code length}. */- public void writeField(FieldInfo info, DataInput value, int length) throws IOException {+ /** Writes a stored binary value from a {@link StoredFieldDataInput}. */+ public void writeField(FieldInfo info, StoredFieldDataInput value) throws IOException {+ int length = value.length();final byte[] bytes = new byte[length];- value.readBytes(bytes, 0, length);+ value.getDataInput().readBytes(bytes, 0, length);writeField(info, new BytesRef(bytes, 0, length));}@@ -191,8 +192,8 @@ public abstract class StoredFieldsWriter implements Closeable, Accountable {}@Override- public void binaryField(FieldInfo fieldInfo, DataInput value, int length) throws IOException {- writeField(remap(fieldInfo), value, length);+ public void binaryField(FieldInfo fieldInfo, StoredFieldDataInput value) throws IOException {+ writeField(remap(fieldInfo), value);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.javaindex 5025e97f8b3..315adff1473 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java@@ -51,6 +51,7 @@ import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.index.StoredFieldVisitor;import org.apache.lucene.store.AlreadyClosedException;import org.apache.lucene.store.ByteArrayDataInput;@@ -261,7 +262,7 @@ public final class Lucene90CompressingStoredFieldsReader extends StoredFieldsReaswitch (bits & TYPE_MASK) {case BYTE_ARR:int length = in.readVInt();- visitor.binaryField(info, in, length);+ visitor.binaryField(info, new StoredFieldDataInput(in, length));break;case STRING:visitor.stringField(info, in.readString());diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.javaindex a7afdbf30ab..68805763e78 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java@@ -34,9 +34,9 @@ import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.IndexFileNames;import org.apache.lucene.index.MergeState;import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.store.ByteBuffersDataInput;import org.apache.lucene.store.ByteBuffersDataOutput;-import org.apache.lucene.store.DataInput;import org.apache.lucene.store.DataOutput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;@@ -311,12 +311,13 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri}@Override- public void writeField(FieldInfo info, DataInput value, int length) throws IOException {+ public void writeField(FieldInfo info, StoredFieldDataInput value) throws IOException {+ int length = value.getLength();++numStoredFieldsInDoc;final long infoAndBits = (((long) info.number) << TYPE_BITS) | BYTE_ARR;bufferedDocs.writeVLong(infoAndBits);bufferedDocs.writeVInt(length);- bufferedDocs.copyBytes(value, length);+ bufferedDocs.copyBytes(value.getDataInput(), length);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.javaindex 9945fdeae17..9d89abd1ce4 100644--- a/lucene/core/src/java/org/apache/lucene/document/Field.java+++ b/lucene/core/src/java/org/apache/lucene/document/Field.java@@ -27,6 +27,7 @@ import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.IndexOptions;import org.apache.lucene.index.IndexableField;import org.apache.lucene.index.IndexableFieldType;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.util.BytesRef;/**@@ -619,6 +620,8 @@ public class Field implements IndexableField {return new StoredValue((double) fieldsData);} else if (fieldsData instanceof BytesRef) {return new StoredValue((BytesRef) fieldsData);+ } else if (fieldsData instanceof StoredFieldDataInput) {+ return new StoredValue((StoredFieldDataInput) fieldsData);} else if (fieldsData instanceof String) {return new StoredValue((String) fieldsData);} else {diff --git a/lucene/core/src/java/org/apache/lucene/document/StoredField.java b/lucene/core/src/java/org/apache/lucene/document/StoredField.javaindex c93b9b5d717..e1190df8980 100644--- a/lucene/core/src/java/org/apache/lucene/document/StoredField.java+++ b/lucene/core/src/java/org/apache/lucene/document/StoredField.java@@ -17,6 +17,7 @@package org.apache.lucene.document;import org.apache.lucene.index.IndexReader; // javadocs+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.search.IndexSearcher; // javadocsimport org.apache.lucene.util.BytesRef;@@ -105,6 +106,21 @@ public class StoredField extends Field {super(name, value, TYPE);}+ /**+ * Create a stored-only field with the given data input value.+ *+ * @param name field name+ * @param value BytesRef pointing to binary content (not copied)+ * @throws IllegalArgumentException if the field name or value is null.+ */+ public StoredField(String name, StoredFieldDataInput value) {+ super(name, TYPE);+ if (value == null) {+ throw new IllegalArgumentException("store field data input must not be null");+ }+ fieldsData = value;+ }+/*** Create a stored-only field with the given string value.*diff --git a/lucene/core/src/java/org/apache/lucene/document/StoredValue.java b/lucene/core/src/java/org/apache/lucene/document/StoredValue.javaindex 61401c7b2fc..413664de620 100644--- a/lucene/core/src/java/org/apache/lucene/document/StoredValue.java+++ b/lucene/core/src/java/org/apache/lucene/document/StoredValue.java@@ -18,6 +18,7 @@ package org.apache.lucene.document;import java.util.Objects;import org.apache.lucene.index.IndexableField;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.util.BytesRef;/**@@ -39,6 +40,8 @@ public final class StoredValue {DOUBLE,/** Type of binary values. */BINARY,+ /** Type of data input values. */+ DATA_INPUT,/** Type of string values. */STRING;}@@ -48,6 +51,7 @@ public final class StoredValue {private long longValue;private float floatValue;private double doubleValue;+ private StoredFieldDataInput dataInput;private BytesRef binaryValue;private String stringValue;@@ -81,7 +85,13 @@ public final class StoredValue {binaryValue = Objects.requireNonNull(value);}- /** Ctor for binary values. */+ /** Ctor for data input values. */+ public StoredValue(StoredFieldDataInput value) {+ type = Type.DATA_INPUT;+ dataInput = Objects.requireNonNull(value);+ }++ /** Ctor for string values. */public StoredValue(String value) {type = Type.STRING;stringValue = Objects.requireNonNull(value);@@ -132,6 +142,14 @@ public final class StoredValue {binaryValue = Objects.requireNonNull(value);}+ /** Set a data input value. */+ public void setDataInputValue(StoredFieldDataInput value) {+ if (type != Type.DATA_INPUT) {+ throw new IllegalArgumentException("Cannot set a data input value on a " + type + " value");+ }+ dataInput = Objects.requireNonNull(value);+ }+/** Set a string value. */public void setStringValue(String value) {if (type != Type.STRING) {@@ -180,6 +198,14 @@ public final class StoredValue {return binaryValue;}+ /** Retrieve a data input value. */+ public StoredFieldDataInput getDataInputValue() {+ if (type != Type.DATA_INPUT) {+ throw new IllegalArgumentException("Cannot get a data input value on a " + type + " value");+ }+ return dataInput;+ }+/** Retrieve a string value. */public String getStringValue() {if (type != Type.STRING) {diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.javaindex 8044a4cc387..693806da4fb 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortingStoredFieldsConsumer.java@@ -138,8 +138,8 @@ final class SortingStoredFieldsConsumer extends StoredFieldsConsumer {}@Override- public void binaryField(FieldInfo fieldInfo, DataInput value, int length) throws IOException {- writer.writeField(fieldInfo, value, length);+ public void binaryField(FieldInfo fieldInfo, StoredFieldDataInput value) throws IOException {+ writer.writeField(fieldInfo, value);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/index/StoredFieldDataInput.java b/lucene/core/src/java/org/apache/lucene/index/StoredFieldDataInput.javanew file mode 100644index 00000000000..f55a3b9ef04--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/StoredFieldDataInput.java@@ -0,0 +1,45 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import org.apache.lucene.store.ByteArrayDataInput;+import org.apache.lucene.store.DataInput;++/**+ * A fixed size DataInput which includes the length of the input. For use as a StoredField.+ *+ * @param in the data input+ * @param length the length of the data input+ * @lucene.experimental+ */+public record StoredFieldDataInput(DataInput in, int length) {++ /** Creates a StoredFieldDataInput from a ByteArrayDataInput */+ public StoredFieldDataInput(ByteArrayDataInput byteArrayDataInput) {+ this(byteArrayDataInput, byteArrayDataInput.length());+ }++ /** Returns the data input */+ public DataInput getDataInput() {+ return in;+ }++ /** Returns the length of the data input */+ public int getLength() {+ return length;+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/index/StoredFieldVisitor.java b/lucene/core/src/java/org/apache/lucene/index/StoredFieldVisitor.javaindex 95dbf11b720..35a75a6ab6a 100644--- a/lucene/core/src/java/org/apache/lucene/index/StoredFieldVisitor.java+++ b/lucene/core/src/java/org/apache/lucene/index/StoredFieldVisitor.java@@ -19,7 +19,6 @@ package org.apache.lucene.index;import java.io.IOException;import org.apache.lucene.document.Document;import org.apache.lucene.document.DocumentStoredFieldVisitor;-import org.apache.lucene.store.DataInput;/*** Expert: provides a low-level means of accessing the stored field values in an index. See {@link@@ -41,15 +40,17 @@ public abstract class StoredFieldVisitor {protected StoredFieldVisitor() {}/**- * Expert: Process a binary field directly from the {@link DataInput}. Implementors of this method- * must read {@code length} bytes from the given {@link DataInput}. The default implementation- * reads all byes in a newly created byte array and calls {@link #binaryField(FieldInfo, byte[])}.+ * Expert: Process a binary field directly from the {@link StoredFieldDataInput}. Implementors of+ * this method must read {@code StoredFieldDataInput#length} bytes from the given {@link+ * StoredFieldDataInput}. The default implementation reads all bytes in a newly created byte array+ * and calls {@link #binaryField(FieldInfo, byte[])}.*- * @param value newly allocated byte array with the binary contents.+ * @param value the stored field data input.*/- public void binaryField(FieldInfo fieldInfo, DataInput value, int length) throws IOException {+ public void binaryField(FieldInfo fieldInfo, StoredFieldDataInput value) throws IOException {+ int length = value.length();final byte[] data = new byte[length];- value.readBytes(data, 0, length);+ value.getDataInput().readBytes(data, 0, value.getLength());binaryField(fieldInfo, data);}diff --git a/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.javaindex 78f2f726a2f..24c5359d29a 100644--- a/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java@@ -79,6 +79,9 @@ class StoredFieldsConsumer {case BINARY:writer.writeField(info, value.getBinaryValue());break;+ case DATA_INPUT:+ writer.writeField(info, value.getDataInputValue());+ break;case STRING:writer.writeField(info, value.getStringValue());break;diff --git a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.javaindex 689206e92b5..c9572c2b593 100644--- a/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java+++ b/lucene/core/src/test/org/apache/lucene/document/TestBinaryDocument.java@@ -18,6 +18,8 @@ package org.apache.lucene.document;import java.nio.charset.StandardCharsets;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.StoredFieldDataInput;+import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.index.RandomIndexWriter;import org.apache.lucene.tests.util.LuceneTestCase;@@ -71,4 +73,48 @@ public class TestBinaryDocument extends LuceneTestCase {reader.close();dir.close();}++ public void testBinaryFieldFromDataInputInIndex() throws Exception {+ FieldType ft = new FieldType();+ ft.setStored(true);+ byte[] byteArray = binaryValStored.getBytes(StandardCharsets.UTF_8);+ StoredFieldDataInput storedFieldDataInput =+ new StoredFieldDataInput(new ByteArrayDataInput(byteArray));+ StoredField binaryFldStored = new StoredField("binaryStored", storedFieldDataInput);+ Field stringFldStored = new Field("stringStored", binaryValStored, ft);++ Document doc = new Document();++ doc.add(binaryFldStored);++ doc.add(stringFldStored);++ /* test for field count */+ assertEquals(2, doc.getFields().size());++ /* add the doc to a ram index */+ Directory dir = newDirectory();+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);+ writer.addDocument(doc);++ /* open a reader and fetch the document */+ IndexReader reader = writer.getReader();+ Document docFromReader = reader.storedFields().document(0);+ assertTrue(docFromReader != null);++ /* fetch the binary stored field and compare its content with the original one */+ BytesRef bytes = docFromReader.getBinaryValue("binaryStored");+ assertNotNull(bytes);+ String binaryFldStoredTest =+ new String(bytes.bytes, bytes.offset, bytes.length, StandardCharsets.UTF_8);+ assertTrue(binaryFldStoredTest.equals(binaryValStored));++ /* fetch the string field and compare its content with the original one */+ String stringFldStoredTest = docFromReader.get("stringStored");+ assertTrue(stringFldStoredTest.equals(binaryValStored));++ writer.close();+ reader.close();+ dir.close();+ }}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.javaindex 1ea5da23d4b..878f00ac551 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java@@ -38,6 +38,7 @@ import org.apache.lucene.document.StringField;import org.apache.lucene.document.TextField;import org.apache.lucene.index.IndexWriterConfig.OpenMode;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.analysis.MockAnalyzer;import org.apache.lucene.tests.index.DocHelper;@@ -366,7 +367,8 @@ public class TestDirectoryReader extends LuceneTestCase {public void testBinaryFields() throws IOException {Directory dir = newDirectory();- byte[] bin = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};+ byte[] bin1 = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};+ byte[] bin2 = new byte[] {10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};IndexWriter writer =new IndexWriter(@@ -387,7 +389,8 @@ public class TestDirectoryReader extends LuceneTestCase {.setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));Document doc = new Document();- doc.add(new StoredField("bin1", bin));+ doc.add(new StoredField("bin1", bin1));+ doc.add(new StoredField("bin2", new StoredFieldDataInput(new ByteArrayDataInput(bin2))));doc.add(new TextField("junk", "junk text", Field.Store.NO));writer.addDocument(doc);writer.close();@@ -398,11 +401,22 @@ public class TestDirectoryReader extends LuceneTestCase {assertEquals(1, fields.length);IndexableField b1 = fields[0];assertTrue(b1.binaryValue() != null);- BytesRef bytesRef = b1.binaryValue();- assertEquals(bin.length, bytesRef.length);- for (int i = 0; i < bin.length; i++) {- assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);+ BytesRef bytesRef1 = b1.binaryValue();+ assertEquals(bin1.length, bytesRef1.length);+ for (int i = 0; i < bin1.length; i++) {+ assertEquals(bin1[i], bytesRef1.bytes[i + bytesRef1.offset]);}+ fields = doc2.getFields("bin2");+ assertNotNull(fields);+ assertEquals(1, fields.length);+ IndexableField b2 = fields[0];+ assertTrue(b2.binaryValue() != null);+ BytesRef bytesRef2 = b2.binaryValue();+ assertEquals(bin2.length, bytesRef2.length);+ for (int i = 0; i < bin2.length; i++) {+ assertEquals(bin2[i], bytesRef2.bytes[i + bytesRef2.offset]);+ }+reader.close();// force merge@@ -421,10 +435,19 @@ public class TestDirectoryReader extends LuceneTestCase {assertEquals(1, fields.length);b1 = fields[0];assertTrue(b1.binaryValue() != null);- bytesRef = b1.binaryValue();- assertEquals(bin.length, bytesRef.length);- for (int i = 0; i < bin.length; i++) {- assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);+ bytesRef1 = b1.binaryValue();+ assertEquals(bin1.length, bytesRef1.length);+ for (int i = 0; i < bin1.length; i++) {+ assertEquals(bin1[i], bytesRef1.bytes[i + bytesRef1.offset]);+ }+ fields = doc2.getFields("bin2");+ assertNotNull(fields);+ assertEquals(1, fields.length);+ b2 = fields[0];+ bytesRef2 = b2.binaryValue();+ assertEquals(bin2.length, bytesRef2.length);+ for (int i = 0; i < bin2.length; i++) {+ assertEquals(bin2[i], bytesRef2.bytes[i + bytesRef2.offset]);}reader.close();dir.close();diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.javaindex c7e910c2536..c39c41340d1 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java@@ -1772,6 +1772,35 @@ public class TestIndexWriterExceptions extends LuceneTestCase {dir.close();}+ /** test a null data input value doesn't abort the entire segment */+ public void testNullStoredDataInputField() throws Exception {+ Directory dir = newDirectory();+ Analyzer analyzer = new MockAnalyzer(random());+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer));+ // add good document+ Document doc = new Document();+ iw.addDocument(doc);++ expectThrows(+ IllegalArgumentException.class,+ () -> {+ // set to null value+ StoredFieldDataInput v = null;+ Field theField = new StoredField("foo", v);+ doc.add(theField);+ iw.addDocument(doc);+ fail("didn't get expected exception");+ });++ assertNull(iw.getTragicException());+ iw.close();+ // make sure we see our good doc+ DirectoryReader r = DirectoryReader.open(dir);+ assertEquals(1, r.numDocs());+ r.close();+ dir.close();+ }+public void testCrazyPositionIncrementGap() throws Exception {Directory dir = newDirectory();Analyzer analyzer =diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingStoredFieldsFormat.javaindex d8d37718c11..2ea955c3c01 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingStoredFieldsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingStoredFieldsFormat.java@@ -24,8 +24,8 @@ import org.apache.lucene.codecs.StoredFieldsWriter;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.index.StoredFieldVisitor;-import org.apache.lucene.store.DataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.tests.util.TestUtil;@@ -161,9 +161,9 @@ public class AssertingStoredFieldsFormat extends StoredFieldsFormat {}@Override- public void writeField(FieldInfo info, DataInput value, int length) throws IOException {+ public void writeField(FieldInfo info, StoredFieldDataInput value) throws IOException {assert docStatus == Status.STARTED;- in.writeField(info, value, length);+ in.writeField(info, value);}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/cranky/CrankyStoredFieldsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/cranky/CrankyStoredFieldsFormat.javaindex 25d1695f534..6e8acca3ff9 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/cranky/CrankyStoredFieldsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/cranky/CrankyStoredFieldsFormat.java@@ -26,7 +26,7 @@ import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.MergeState;import org.apache.lucene.index.SegmentInfo;-import org.apache.lucene.store.DataInput;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.util.Accountable;@@ -149,11 +149,11 @@ class CrankyStoredFieldsFormat extends StoredFieldsFormat {}@Override- public void writeField(FieldInfo info, DataInput value, int length) throws IOException {+ public void writeField(FieldInfo info, StoredFieldDataInput value) throws IOException {if (random.nextInt(10000) == 0) {throw new IOException("Fake IOException from StoredFieldsWriter.writeField()");}- delegate.writeField(info, value, length);+ delegate.writeField(info, value);}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.javaindex c2aa7ff0e4d..6daccb35578 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseIndexFileFormatTestCase.java@@ -595,6 +595,9 @@ public abstract class BaseIndexFileFormatTestCase extends LuceneTestCase {case BINARY:consumer.writeField(field, value.getBinaryValue());break;+ case DATA_INPUT:+ consumer.writeField(field, value.getDataInputValue());+ break;case STRING:consumer.writeField(field, value.getStringValue());break;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.javaindex 84fdcde8173..f879cff8d59 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseStoredFieldsFormatTestCase.java@@ -57,6 +57,7 @@ import org.apache.lucene.index.LeafReader;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.NoMergePolicy;import org.apache.lucene.index.NumericDocValues;+import org.apache.lucene.index.StoredFieldDataInput;import org.apache.lucene.index.StoredFieldVisitor;import org.apache.lucene.index.StoredFields;import org.apache.lucene.index.Term;@@ -66,6 +67,7 @@ import org.apache.lucene.search.Sort;import org.apache.lucene.search.SortField;import org.apache.lucene.search.TermQuery;import org.apache.lucene.search.TopDocs;+import org.apache.lucene.store.ByteArrayDataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.MMapDirectory;import org.apache.lucene.tests.analysis.MockAnalyzer;@@ -682,7 +684,13 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormatdoc.add(new StoredField("d", random().nextDouble()));doc.add(new StoredField("f", random().nextFloat()));doc.add(new StoredField("s", RandomPicks.randomFrom(random(), stringValues)));- doc.add(new StoredField("b", new BytesRef(RandomPicks.randomFrom(random(), stringValues))));+ BytesRef value = new BytesRef(RandomPicks.randomFrom(random(), stringValues));+ doc.add(new StoredField("b", value));+ doc.add(+ new StoredField(+ "b2",+ new StoredFieldDataInput(+ new ByteArrayDataInput(value.bytes, value.offset, value.length))));docs[i] = doc;w.addDocument(doc);}@@ -713,6 +721,9 @@ public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormatassertEquals(expected.getField("d").numericValue(), doc.getField("d").numericValue());assertEquals(expected.getField("f").numericValue(), doc.getField("f").numericValue());assertEquals(expected.getField("b").binaryValue(), doc.getField("b").binaryValue());+ // The value is the same for fields "b" and "b2". Read the expected value from "b" as "b2" was+ // consumed during indexing+ assertEquals(expected.getField("b").binaryValue(), doc.getField("b2").binaryValue());}reader.close();