Apache Lucene open-source search software
LUCENE-10354: Clarify contract of codec APIs with missing/disabled fields. (#583)
Adrien Grandc8651afde70c
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.javaindex 96d381ac4a0..2731bccd067 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;import java.io.Closeable;import java.io.IOException;import org.apache.lucene.index.BinaryDocValues;+import org.apache.lucene.index.DocValuesType;import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.NumericDocValues;import org.apache.lucene.index.SortedDocValues;@@ -37,31 +38,38 @@ public abstract class DocValuesProducer implements Closeable {/*** Returns {@link NumericDocValues} for this field. The returned instance need not be thread-safe:- * it will only be used by a single thread.+ * it will only be used by a single thread. The behavior is undefined if the doc values type of+ * the given field is not {@link DocValuesType#NUMERIC}. The return value is never {@code null}.*/public abstract NumericDocValues getNumeric(FieldInfo field) throws IOException;/*** Returns {@link BinaryDocValues} for this field. The returned instance need not be thread-safe:- * it will only be used by a single thread.+ * it will only be used by a single thread. The behavior is undefined if the doc values type of+ * the given field is not {@link DocValuesType#BINARY}. The return value is never {@code null}.*/public abstract BinaryDocValues getBinary(FieldInfo field) throws IOException;/*** Returns {@link SortedDocValues} for this field. The returned instance need not be thread-safe:- * it will only be used by a single thread.+ * it will only be used by a single thread. The behavior is undefined if the doc values type of+ * the given field is not {@link DocValuesType#SORTED}. The return value is never {@code null}.*/public abstract SortedDocValues getSorted(FieldInfo field) throws IOException;/*** Returns {@link SortedNumericDocValues} for this field. The returned instance need not be- * thread-safe: it will only be used by a single thread.+ * thread-safe: it will only be used by a single thread. The behavior is undefined if the doc+ * values type of the given field is not {@link DocValuesType#SORTED_NUMERIC}. The return value is+ * never {@code null}.*/public abstract SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException;/*** Returns {@link SortedSetDocValues} for this field. The returned instance need not be- * thread-safe: it will only be used by a single thread.+ * thread-safe: it will only be used by a single thread. The behavior is undefined if the doc+ * values type of the given field is not {@link DocValuesType#SORTED_SET}. The return value is+ * never {@code null}.*/public abstract SortedSetDocValues getSortedSet(FieldInfo field) throws IOException;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.javaindex d89e5c7c811..6407559228a 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;import java.io.Closeable;import java.io.IOException;+import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.VectorValues;import org.apache.lucene.search.TopDocs;import org.apache.lucene.util.Accountable;@@ -40,7 +41,11 @@ public abstract class KnnVectorsReader implements Closeable, Accountable {*/public abstract void checkIntegrity() throws IOException;- /** Returns the {@link VectorValues} for the given {@code field} */+ /**+ * Returns the {@link VectorValues} for the given {@code field}. The behavior is undefined if the+ * given field doesn't have KNN vectors enabled on its {@link FieldInfo}. The return value is+ * never {@code null}.+ */public abstract VectorValues getVectorValues(String field) throws IOException;/**@@ -53,6 +58,9 @@ public abstract class KnnVectorsReader implements Closeable, Accountable {* true k closest neighbors. For large values of k (for example when k is close to the total* number of documents), the search may also retrieve fewer than k documents.*+ * <p>The behavior is undefined if the given field doesn't have KNN vectors enabled on its {@link+ * FieldInfo}. The return value is never {@code null}.+ ** @param field the vector field to search* @param target the vector-valued query* @param k the number of docs to returndiff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.javaindex 6df7d7eb133..064e8418d59 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java@@ -33,7 +33,8 @@ public abstract class NormsProducer implements Closeable {/*** Returns {@link NumericDocValues} for this field. The returned instance need not be thread-safe:- * it will only be used by a single thread.+ * it will only be used by a single thread. The behavior is undefined if the given field doesn't+ * have norms enabled on its {@link FieldInfo}. The return value is never {@code null}.*/public abstract NumericDocValues getNorms(FieldInfo field) throws IOException;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.javaindex 241c6dcdbcf..cc85dc0184d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsReader.java@@ -18,6 +18,7 @@ package org.apache.lucene.codecs;import java.io.Closeable;import java.io.IOException;+import org.apache.lucene.index.FieldInfo;import org.apache.lucene.index.PointValues;/**@@ -40,7 +41,10 @@ public abstract class PointsReader implements Closeable {*/public abstract void checkIntegrity() throws IOException;- /** Return {@link PointValues} for the given {@code field}. */+ /**+ * Return {@link PointValues} for the given {@code field}. The behavior is undefined if the given+ * field doesn't have points enabled on its {@link FieldInfo}.+ */public abstract PointValues getValues(String field) throws IOException;/**diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90HnswVectorsReader.javaindex b0ac8a975d5..2ae1bc45767 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90HnswVectorsReader.java@@ -226,19 +226,13 @@ public final class Lucene90HnswVectorsReader extends KnnVectorsReader {@Overridepublic VectorValues getVectorValues(String field) throws IOException {FieldEntry fieldEntry = fields.get(field);- if (fieldEntry == null || fieldEntry.dimension == 0) {- return null;- }-return getOffHeapVectorValues(fieldEntry);}@Overridepublic TopDocs search(String field, float[] target, int k, Bits acceptDocs) throws IOException {FieldEntry fieldEntry = fields.get(field);- if (fieldEntry == null || fieldEntry.dimension == 0) {- return null;- }+if (fieldEntry.size() == 0) {return new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), new ScoreDoc[0]);}diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.javaindex 064cdbf2aaf..9c05098eba2 100644--- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java@@ -105,15 +105,13 @@ public abstract class CodecReader extends LeafReader {@Overridepublic final Terms terms(String field) throws IOException {- // ensureOpen(); no; getPostingsReader calls this- FieldsProducer fieldsProducer = getPostingsReader();- if (fieldsProducer == null) {+ ensureOpen();+ FieldInfo fi = getFieldInfos().fieldInfo(field);+ if (fi == null || fi.getIndexOptions() == IndexOptions.NONE) {+ // Field does not exist or does not index postingsreturn null;}- // We could check the FieldInfo IndexOptions but there's no point since- // PostingsReader will simply return null for fields that don't exist or that have no terms- // index.- return fieldsProducer.terms(field);+ return getPostingsReader().terms(field);}// returns the FieldInfo that corresponds to the given field and type, ordiff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.javaindex 6b17d3655ac..55b17d5bbbd 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java@@ -22,6 +22,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.KnnVectorsWriter;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.VectorValues;@@ -45,7 +46,7 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new AssertingKnnVectorsReader(delegate.fieldsReader(state));+ return new AssertingKnnVectorsReader(delegate.fieldsReader(state), state.fieldInfos);}static class AssertingKnnVectorsWriter extends KnnVectorsWriter {@@ -76,10 +77,12 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {static class AssertingKnnVectorsReader extends KnnVectorsReader {final KnnVectorsReader delegate;+ final FieldInfos fis;- AssertingKnnVectorsReader(KnnVectorsReader delegate) {+ AssertingKnnVectorsReader(KnnVectorsReader delegate, FieldInfos fis) {assert delegate != null;this.delegate = delegate;+ this.fis = fis;}@Override@@ -89,17 +92,20 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic VectorValues getVectorValues(String field) throws IOException {+ FieldInfo fi = fis.fieldInfo(field);+ assert fi != null && fi.getVectorDimension() > 0;VectorValues values = delegate.getVectorValues(field);- if (values != null) {- assert values.docID() == -1;- assert values.size() >= 0;- assert values.dimension() > 0;- }+ assert values != null;+ assert values.docID() == -1;+ assert values.size() >= 0;+ assert values.dimension() > 0;return values;}@Overridepublic TopDocs search(String field, float[] target, int k, Bits acceptDocs) throws IOException {+ FieldInfo fi = fis.fieldInfo(field);+ assert fi != null && fi.getVectorDimension() > 0;TopDocs hits = delegate.search(field, target, k, acceptDocs);assert hits != null;assert hits.scoreDocs.length <= k;diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingPointsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingPointsFormat.javaindex 2b68a59e135..eada1e8f49c 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingPointsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingPointsFormat.java@@ -21,6 +21,7 @@ import org.apache.lucene.codecs.PointsFormat;import org.apache.lucene.codecs.PointsReader;import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.FieldInfos;import org.apache.lucene.index.MergeState;import org.apache.lucene.index.PointValues;import org.apache.lucene.index.SegmentReadState;@@ -54,18 +55,21 @@ public final class AssertingPointsFormat extends PointsFormat {@Overridepublic PointsReader fieldsReader(SegmentReadState state) throws IOException {- return new AssertingPointsReader(state.segmentInfo.maxDoc(), in.fieldsReader(state), false);+ return new AssertingPointsReader(+ state.segmentInfo.maxDoc(), in.fieldsReader(state), state.fieldInfos, false);}static class AssertingPointsReader extends PointsReader {private final PointsReader in;private final int maxDoc;+ private final FieldInfos fis;private final boolean merging;private final Thread creationThread;- AssertingPointsReader(int maxDoc, PointsReader in, boolean merging) {+ AssertingPointsReader(int maxDoc, PointsReader in, FieldInfos fis, boolean merging) {this.in = in;this.maxDoc = maxDoc;+ this.fis = fis;this.merging = merging;this.creationThread = Thread.currentThread();// do a few simple checks on init@@ -80,6 +84,8 @@ public final class AssertingPointsFormat extends PointsFormat {@Overridepublic PointValues getValues(String field) throws IOException {+ FieldInfo fi = fis.fieldInfo(field);+ assert fi != null && fi.getPointDimensionCount() > 0;if (merging) {AssertingCodec.assertThread("PointsReader", creationThread);}@@ -97,7 +103,7 @@ public final class AssertingPointsFormat extends PointsFormat {@Overridepublic PointsReader getMergeInstance() {- return new AssertingPointsReader(maxDoc, in.getMergeInstance(), true);+ return new AssertingPointsReader(maxDoc, in.getMergeInstance(), fis, true);}@Override