Skip to content

Apache Lucene open-source search software

Let the structure reading vectors say how they are read (#16682)

Jim Ferenczi79674f38df7d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6c4e131d58c..cad59846c8d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -191,6 +191,13 @@ Improvements
Optimizations
---------------------
+
+* GITHUB#16682: Carry file open hints on merge and flush contexts, which used to drop them. An HNSW
+ format now carries the random access hint for the vectors it reads, rather than the flat format
+ holding them claiming it whatever reads them. Add NoReuseHint, set by stored fields, term vectors
+ and the raw vectors kept only to rescore, so an application can tell those apart from the files it
+ wants to keep in memory. (Jim Ferenczi)
+
* GITHUB#16431: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQuery
and DoubleValuesSource using DocValuesSkipper. (Rajat Jain)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
index 16251990634..a8424c929f8 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene102;
import java.io.IOException;
+import java.util.stream.Stream;
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
@@ -24,6 +25,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.NoReuseHint;
/**
* The binary quantization format used here is a per-vector optimized scalar quantization. These
@@ -123,7 +126,7 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
return new Lucene102BinaryQuantizedVectorsReader(
- state, rawVectorFormat.fieldsReader(state), scorer);
+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), scorer);
}
@Override
@@ -141,4 +144,13 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {
+ rawVectorFormat
+ ")";
}
+
+ /** The raw vectors are only read back to rescore. */
+ private static SegmentReadState rescoreOnly(SegmentReadState state) {
+ IOContext.FileOpenHint[] hints =
+ Stream.concat(state.context.hints().stream(), Stream.of(NoReuseHint.INSTANCE))
+ .distinct()
+ .toArray(IOContext.FileOpenHint[]::new);
+ return new SegmentReadState(state, state.context.withHints(hints));
+ }
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
index e7c6a869150..7770bbc34bf 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
@@ -53,7 +53,6 @@ import org.apache.lucene.search.AcceptDocs;
import org.apache.lucene.search.KnnCollector;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
@@ -128,10 +127,8 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader
versionMeta,
VECTOR_DATA_EXTENSION,
Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
- // Quantized vectors are accessed randomly from their node ID stored in the HNSW
- // graph.
- state.context.withHints(
- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
+ // how these are read is up to whoever wraps this format
+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));
} catch (Throwable t) {
IOUtils.closeWhileSuppressingExceptions(t, this);
throw t;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
index 698cca05b05..4eba00e1f9c 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java
@@ -33,6 +33,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
/**
@@ -119,7 +122,10 @@ public class Lucene102HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
index c1a0bba89b2..26334924b08 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java
@@ -33,6 +33,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
/**
@@ -137,7 +140,10 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
index b7f7e16c5cf..f6a83d2a95b 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java
@@ -18,6 +18,7 @@
package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
+import java.util.stream.Stream;
import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
@@ -26,6 +27,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.NoReuseHint;
/**
* Format supporting vector quantization, storage, and retrieval
@@ -150,6 +153,15 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
return new Lucene99ScalarQuantizedVectorsReader(
- state, rawVectorFormat.fieldsReader(state), flatVectorScorer);
+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), flatVectorScorer);
+ }
+
+ /** The raw vectors are only read back to rescore. */
+ private static SegmentReadState rescoreOnly(SegmentReadState state) {
+ IOContext.FileOpenHint[] hints =
+ Stream.concat(state.context.hints().stream(), Stream.of(NoReuseHint.INSTANCE))
+ .distinct()
+ .toArray(IOContext.FileOpenHint[]::new);
+ return new SegmentReadState(state, state.context.withHints(hints));
}
}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index 57b1f7fe130..e13406ced82 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -44,7 +44,6 @@ import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
@@ -110,10 +109,8 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
versionMeta,
Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION,
Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
- // Quantized vectors are accessed randomly from their node ID stored in the HNSW
- // graph.
- state.context.withHints(
- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
+ // how these are read is up to whoever wraps this format
+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));
} catch (Throwable t) {
IOUtils.closeWhileSuppressingExceptions(t, this);
throw t;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java
index b8f8f1b5b26..b7537fdffc4 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java
@@ -41,6 +41,9 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.IORunnable;
import org.apache.lucene.util.hnsw.HnswGraph;
@@ -147,7 +150,10 @@ public final class HnswBitVectorsFormat extends KnnVectorsFormat {
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
index c7f73cbaa67..5c6634213b4 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java
@@ -34,6 +34,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
@@ -202,7 +205,10 @@ public class Lucene104HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
index 4f5895832c6..c49d8499733 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java
@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.NoReuseHint;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
import org.apache.lucene.util.quantization.OptimizedScalarQuantizer.QuantizationResult;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
@@ -160,7 +161,7 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
@Override
public FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {
return new Lucene104ScalarQuantizedVectorsReader(
- state, rawVectorFormat.fieldsReader(state), scorer);
+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), scorer);
}
@Override
@@ -180,4 +181,9 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {
+ rawVectorFormat
+ ")";
}
+
+ /** The raw vectors are only read back to rescore. */
+ private static SegmentReadState rescoreOnly(SegmentReadState state) {
+ return new SegmentReadState(state, state.context.union(NoReuseHint.INSTANCE));
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index ddb3920a823..13617dd51bd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -26,7 +26,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
-import java.util.stream.Stream;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
@@ -50,7 +49,6 @@ import org.apache.lucene.search.AcceptDocs;
import org.apache.lucene.search.KnnCollector;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
@@ -91,17 +89,6 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
FlatVectorsReader rawVectorsReader,
Lucene104ScalarQuantizedVectorScorer vectorsScorer)
throws IOException {
- // Quantized vectors are accessed randomly from their node ID stored in the HNSW
- // graph.
- this(state, rawVectorsReader, vectorsScorer, DataAccessHint.RANDOM);
- }
-
- public Lucene104ScalarQuantizedVectorsReader(
- SegmentReadState state,
- FlatVectorsReader rawVectorsReader,
- Lucene104ScalarQuantizedVectorScorer vectorsScorer,
- DataAccessHint accessHint)
- throws IOException {
this.vectorScorer = vectorsScorer;
this.rawVectorsReader = rawVectorsReader;
int versionMeta = -1;
@@ -128,17 +115,14 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
CodecUtil.checkFooter(meta, priorE);
}
- final IOContext.FileOpenHint[] hints =
- Stream.of(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, accessHint)
- .filter(Objects::nonNull)
- .toArray(IOContext.FileOpenHint[]::new);
quantizedVectorData =
openDataInput(
state,
versionMeta,
VECTOR_DATA_EXTENSION,
Lucene104ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
- state.context.withHints(hints));
+ // how these are read is up to whoever wraps this format
+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));
} catch (Throwable t) {
IOUtils.closeWhileSuppressingExceptions(t, this);
throw t;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
index 1d0d0e54a81..c44a7025e4a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java
@@ -49,7 +49,10 @@ import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.internal.hppc.FloatArrayList;
import org.apache.lucene.search.VectorScorer;
+import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
@@ -396,6 +399,15 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
}
}
+ /**
+ * The quantized query vectors staged for the merge scorer. A graph build reads them by ordinal as
+ * it walks, so they are read at random.
+ */
+ private IOContext queryDataContext() {
+ return segmentWriteState.context.withHints(
+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM);
+ }
+
/**
* Writes merged index-side and query-side records in one pass. Index-side records go to the
* quantized vector data file, while query-side records go to a temporary file owned by the
@@ -428,7 +440,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
String queryDataName = null;
try (IndexOutput queryData =
segmentWriteState.directory.createTempOutput(
- segmentWriteState.segmentInfo.name, "queries", segmentWriteState.context)) {
+ segmentWriteState.segmentInfo.name, "queries", queryDataContext())) {
queryDataName = queryData.getName();
KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {
@@ -450,11 +462,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {
throw t;
}
return new PreparedQueryData(
- segmentWriteState.directory,
- segmentWriteState.context,
- fieldInfo,
- vectorScorer,
- queryDataName);
+ segmentWriteState.directory, queryDataContext(), fieldInfo, vectorScorer, queryDataName);
}
/** Owns one field's temporary query-side records until its graph scorer takes them over. */
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java
index 868190a4b10..98f7ccb2423 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java
@@ -63,6 +63,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.NoReuseHint;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
@@ -143,7 +144,9 @@ public final class Lucene90CompressingStoredFieldsReader extends StoredFieldsRea
try {
// Open the data file
fieldsStream =
- d.openInput(fieldsStreamFN, context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM));
+ d.openInput(
+ fieldsStreamFN,
+ context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM, NoReuseHint.INSTANCE));
version =
CodecUtil.checkIndexHeader(
fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
index d3364f5ea53..a5c4a200b78 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java
@@ -42,6 +42,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.NoReuseHint;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
@@ -135,7 +136,8 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWri
fieldsStream =
directory.createOutput(
- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);
+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),
+ context.union(NoReuseHint.INSTANCE));
CodecUtil.writeIndexHeader(
fieldsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);
assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java
index fd058ed5c83..7d3aa166326 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java
@@ -62,6 +62,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.NoReuseHint;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -148,7 +149,9 @@ public final class Lucene90CompressingTermVectorsReader extends TermVectorsReade
final String vectorsStreamFN =
IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);
vectorsStream =
- d.openInput(vectorsStreamFN, context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM));
+ d.openInput(
+ vectorsStreamFN,
+ context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM, NoReuseHint.INSTANCE));
version =
CodecUtil.checkIndexHeader(
vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
index c95164b0b86..d47d73abfa9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java
@@ -47,6 +47,7 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.NoReuseHint;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -269,7 +270,8 @@ public final class Lucene90CompressingTermVectorsWriter extends TermVectorsWrite
vectorsStream =
directory.createOutput(
- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);
+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),
+ context.union(NoReuseHint.INSTANCE));
CodecUtil.writeIndexHeader(
vectorsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);
assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
index f29417e66c7..d7aa0f1ff6e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java
@@ -22,8 +22,6 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVe
import java.io.IOException;
import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Stream;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.hnsw.FlatVectorsReader;
import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;
@@ -48,7 +46,6 @@ import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IOContext.FileOpenHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
@@ -72,27 +69,11 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {
public Lucene99FlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer)
throws IOException {
- this(state, scorer, DataAccessHint.RANDOM);
- }
-
- /**
- * Creates a Lucene99FlatVectorsReader.
- *
- * @param state the segment read state
- * @param scorer the flat vectors scorer
- * @param accessHint a data access hint, or null
- */
- public Lucene99FlatVectorsReader(
- SegmentReadState state, FlatVectorsScorer scorer, DataAccessHint accessHint)
- throws IOException {
int versionMeta = readMetadata(state);
this.vectorScorer = scorer;
this.fieldInfos = state.fieldInfos;
- FileOpenHint[] hints =
- Stream.of(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, accessHint)
- .filter(Objects::nonNull)
- .toArray(FileOpenHint[]::new);
- dataContext = state.context.withHints(hints);
+ // how these are read is up to whoever wraps this format
+ dataContext = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);
try {
vectorData =
openDataInput(
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
index 875a6325431..e576a13b3a1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java
@@ -29,6 +29,9 @@ import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.lucene.util.hnsw.HnswGraphBuilder;
@@ -311,7 +314,10 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat {
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
index 17ff9f4e7da..abd76d0fbc8 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java
@@ -51,6 +51,9 @@ import org.apache.lucene.index.Sorter;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IORunnable;
import org.apache.lucene.util.IOUtils;
@@ -485,7 +488,9 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {
if (flatVectorsReader instanceof QuantizedVectorsReader quantizedVectorsReader
&& fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) {
return quantizedVectorsReader.getRandomVectorScorerSupplierForMerge(
- fieldInfo, segmentWriteState);
+ fieldInfo,
+ segmentWriteState.withHints(
+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
}
return null;
}
@@ -537,7 +542,8 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {
segmentWriteState.directory,
segmentWriteState.segmentInfo,
segmentWriteState.fieldInfos,
- segmentWriteState.context,
+ segmentWriteState.context.union(
+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM),
segmentWriteState.segmentSuffix);
flatVectorsReader = flatVectorsFormat.fieldsReader(readState);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
index 2d98549d429..01ae33ecd11 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
@@ -67,6 +67,15 @@ public class SegmentReadState {
this.segmentSuffix = segmentSuffix;
}
+ /** Create a {@code SegmentReadState} with a new {@link IOContext}. */
+ public SegmentReadState(SegmentReadState other, IOContext context) {
+ this.directory = other.directory;
+ this.segmentInfo = other.segmentInfo;
+ this.fieldInfos = other.fieldInfos;
+ this.context = context;
+ this.segmentSuffix = other.segmentSuffix;
+ }
+
/** Create a {@code SegmentReadState}. */
public SegmentReadState(SegmentReadState other, String newSegmentSuffix) {
this.directory = other.directory;
@@ -75,4 +84,9 @@ public class SegmentReadState {
this.context = other.context;
this.segmentSuffix = newSegmentSuffix;
}
+
+ /** Returns a copy of this state whose context carries the given hints in place of its own. */
+ public SegmentReadState withHints(IOContext.FileOpenHint... hints) {
+ return new SegmentReadState(this, context.withHints(hints));
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
index 4854a0a423d..0c94ed558a7 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
@@ -113,6 +113,19 @@ public class SegmentWriteState {
this.context = context;
}
+ /** Create a shallow copy of {@link SegmentWriteState} with a new {@link IOContext}. */
+ public SegmentWriteState(SegmentWriteState state, IOContext context) {
+ infoStream = state.infoStream;
+ directory = state.directory;
+ segmentInfo = state.segmentInfo;
+ fieldInfos = state.fieldInfos;
+ this.context = context;
+ segmentSuffix = state.segmentSuffix;
+ segUpdates = state.segUpdates;
+ delCountOnFlush = state.delCountOnFlush;
+ liveDocs = state.liveDocs;
+ }
+
/** Create a shallow copy of {@link SegmentWriteState} with a new segment suffix. */
public SegmentWriteState(SegmentWriteState state, String segmentSuffix) {
infoStream = state.infoStream;
@@ -144,4 +157,9 @@ public class SegmentWriteState {
}
return true;
}
+
+ /** Returns a copy of this state whose context carries the given hints in place of its own. */
+ public SegmentWriteState withHints(IOContext.FileOpenHint... hints) {
+ return new SegmentWriteState(this, context.withHints(hints));
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java b/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java
index e3e4553a8df..8f8b387990f 100644
--- a/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java
+++ b/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java
@@ -25,16 +25,20 @@ import java.util.stream.Collectors;
record DefaultIOContext(Set<FileOpenHint> hints) implements IOContext {
public DefaultIOContext(Set<FileOpenHint> hints) {
- this.hints = Set.copyOf(Objects.requireNonNull(hints));
+ this.hints = checkOneHintPerType(hints);
+ }
- // there should only be one hint of each type in the IOContext
+ /** Returns an immutable copy of {@code hints}, rejecting any type given more than once. */
+ static Set<FileOpenHint> checkOneHintPerType(Set<FileOpenHint> hints) {
+ Set<FileOpenHint> copy = Set.copyOf(Objects.requireNonNull(hints));
Map<Class<? extends FileOpenHint>, List<FileOpenHint>> hintClasses =
- hints.stream().collect(Collectors.groupingBy(IOContext.FileOpenHint::getClass));
+ copy.stream().collect(Collectors.groupingBy(IOContext.FileOpenHint::getClass));
for (var hintType : hintClasses.entrySet()) {
if (hintType.getValue().size() > 1) {
throw new IllegalArgumentException("Multiple hints of type " + hintType + " specified");
}
}
+ return copy;
}
public DefaultIOContext(FileOpenHint... hints) {
diff --git a/lucene/core/src/java/org/apache/lucene/store/IOContext.java b/lucene/core/src/java/org/apache/lucene/store/IOContext.java
index 357e23ab0b8..dabbe9c6d1d 100644
--- a/lucene/core/src/java/org/apache/lucene/store/IOContext.java
+++ b/lucene/core/src/java/org/apache/lucene/store/IOContext.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.store;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Stream;
@@ -64,7 +66,12 @@ public interface IOContext {
/** Returns an {@link IOContext} for merging with the specified {@link MergeInfo} */
static IOContext merge(MergeInfo mergeInfo) {
+ return merge(mergeInfo, Set.of());
+ }
+
+ private static IOContext merge(MergeInfo mergeInfo, Set<FileOpenHint> hints) {
Objects.requireNonNull(mergeInfo);
+ Set<FileOpenHint> copy = DefaultIOContext.checkOneHintPerType(hints);
return new IOContext() {
@Override
public Context context() {
@@ -83,19 +90,24 @@ public interface IOContext {
@Override
public Set<FileOpenHint> hints() {
- return Set.of();
+ return copy;
}
@Override
public IOContext withHints(FileOpenHint... hints) {
- return this;
+ return merge(mergeInfo, Set.of(hints));
}
};
}
/** Returns an {@link IOContext} for flushing with the specified {@link FlushInfo} */
static IOContext flush(FlushInfo flushInfo) {
+ return flush(flushInfo, Set.of());
+ }
+
+ private static IOContext flush(FlushInfo flushInfo, Set<FileOpenHint> hints) {
Objects.requireNonNull(flushInfo);
+ Set<FileOpenHint> copy = DefaultIOContext.checkOneHintPerType(hints);
return new IOContext() {
@Override
public Context context() {
@@ -114,12 +126,12 @@ public interface IOContext {
@Override
public Set<FileOpenHint> hints() {
- return Set.of();
+ return copy;
}
@Override
public IOContext withHints(FileOpenHint... hints) {
- return this;
+ return flush(flushInfo, Set.of(hints));
}
};
}
@@ -142,8 +154,7 @@ public interface IOContext {
}
/**
- * Returns an IOContext with the given hints, if it makes sense to do so for this specific
- * context. Otherwise, returns this context.
+ * Returns an IOContext with the given hints.
*
* <p>The returned context has the same {@link #context()}, {@link #mergeInfo()}, and {@link
* #flushInfo()} as this context.
@@ -151,4 +162,16 @@ public interface IOContext {
* <p>This instance is immutable and unaffected by this method call.
*/
IOContext withHints(FileOpenHint... hints);
+
+ /**
+ * Returns an IOContext with the given hints added to the ones it already has. A hint it already
+ * has is added once.
+ *
+ * @throws IllegalArgumentException if that would give it two different hints of the same type
+ */
+ default IOContext union(FileOpenHint... hints) {
+ Set<FileOpenHint> merged = new HashSet<>(hints());
+ Collections.addAll(merged, hints);
+ return withHints(merged.toArray(FileOpenHint[]::new));
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.java b/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.java
new file mode 100644
index 00000000000..e112d5dd0ad
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.store;
+
+/**
+ * A hint that a file is not worth keeping in memory. It may be read often; what it lacks is reuse
+ * worth caching for, because its reads repeat in no pattern a cache can exploit, or because it is
+ * large enough next to the rest of the index that holding it would push out data that does benefit.
+ *
+ * <p>Close to {@link ReadOnceHint}, and weaker: it does not claim the file is read once, or in
+ * order, or even that a read never revisits bytes another read has touched.
+ */
+public enum NoReuseHint implements IOContext.FileOpenHint {
+ INSTANCE
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.java
new file mode 100644
index 00000000000..89f856b499b
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene99;
+
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.not;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.ReadOnceHint;
+import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
+import org.hamcrest.MatcherAssert;
+
+/**
+ * Who says how vector files are read: the format that walks them, rather than the one that holds
+ * them. Directories decide what to make of it.
+ */
+public class TestVectorAccessHints extends LuceneTestCase {
+
+ private static final int DIM = 8;
+ private static final String FIELD = "field";
+
+ /** The vectors the graph build scores against, reopened from the segment the merge just wrote. */
+ public void testMergedVectorsStateTheirAccessPattern() throws Exception {
+ Opens opens = new Opens();
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ // threshold 0: always build the graph, whatever the segment size
+ try (IndexWriter w =
+ new IndexWriter(
+ dir,
+ writerConfig(
+ new Lucene99HnswVectorsFormat(
+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+ 0)))) {
+ for (int segment = 0; segment < 2; segment++) {
+ addDocuments(w, 128);
+ w.commit();
+ }
+ opens.clear();
+ w.forceMerge(1);
+ }
+ assertVectorHints(opens.endingWith(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION), opens);
+ }
+ }
+
+ /** The quantized vectors a merge writes to a temporary file and scores against. */
+ public void testMergeTemporaryFileStatesItsAccessPattern() throws Exception {
+ Opens opens = new Opens();
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ // an asymmetric encoding quantizes the query side differently, which is what makes the merge
+ // write these query vectors to a temporary file
+ try (IndexWriter w =
+ new IndexWriter(
+ dir,
+ writerConfig(
+ new Lucene104HnswScalarQuantizedVectorsFormat(
+ ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE,
+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+ 1,
+ null,
+ 0)))) {
+ for (int segment = 0; segment < 2; segment++) {
+ addDocuments(w, 128);
+ w.commit();
+ }
+ opens.clear();
+ w.forceMerge(1);
+ }
+
+ List<Open> temps = new ArrayList<>();
+ for (Open open : opens.endingWith("tmp")) {
+ if (open.name().contains("_queries_")) {
+ temps.add(open);
+ }
+ }
+ MatcherAssert.assertThat("no temporary file was read back: " + opens, temps, not(empty()));
+ assertVectorHints(temps, opens);
+ }
+ }
+
+ /**
+ * A flat format on its own says nothing about how its vectors are read. Only a format that walks
+ * them, like HNSW, knows that, so only it asks for random access.
+ */
+ public void testFlatFormatClaimsNoAccessPattern() throws Exception {
+ Opens opens = new Opens();
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ try (IndexWriter w = new IndexWriter(dir, writerConfig(new Lucene99HnswVectorsFormat()))) {
+ addDocuments(w, 16);
+ w.commit();
+ }
+
+ opens.clear();
+ new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())
+ .fieldsReader(flatReadState(dir, IOContext.DEFAULT))
+ .close();
+
+ for (Open open : vectorFileOpens(opens)) {
+ assertNull("a flat format assumed how its vectors are read: " + open, open.hint());
+ }
+ }
+ }
+
+ /** A caller that says how it reads the vectors is followed, whatever the format would say. */
+ public void testReadStateAccessHintWins() throws Exception {
+ Opens opens = new Opens();
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ try (IndexWriter w = new IndexWriter(dir, writerConfig(new Lucene99HnswVectorsFormat()))) {
+ addDocuments(w, 16);
+ w.commit();
+ }
+
+ opens.clear();
+ new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())
+ .fieldsReader(
+ flatReadState(
+ dir,
+ IOContext.DEFAULT.withHints(
+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.SEQUENTIAL)))
+ .close();
+
+ for (Open open : vectorFileOpens(opens)) {
+ assertEquals(
+ "the format ignored what the caller asked for: " + open,
+ DataAccessHint.SEQUENTIAL,
+ open.hint());
+ }
+ }
+ }
+
+ /**
+ * A graph scores the quantized vectors as it walks and reads the raw ones back to rescore. Both
+ * are read at random, and both take the access pattern asked of the format holding them.
+ */
+ public void testQuantizedAndRawVectorsAreReadAtRandom() throws Exception {
+ Opens opens = new Opens();
+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
+ try (IndexWriter w =
+ new IndexWriter(dir, writerConfig(new Lucene104HnswScalarQuantizedVectorsFormat()))) {
+ opens.clear();
+ addDocuments(w, 16);
+ w.commit();
+ try (DirectoryReader reader = DirectoryReader.open(w)) {
+ assertEquals(1, reader.leaves().size());
+ }
+ }
+
+ List<Open> quantized = opens.endingWith("veq");
+ MatcherAssert.assertThat(
+ "no quantized vectors were opened: " + opens, quantized, not(empty()));
+ for (Open open : quantized) {
+ if (open.context().hints().contains(ReadOnceHint.INSTANCE) == false) {
+ assertEquals(
+ "the quantized vectors are scored in graph order: " + open,
+ DataAccessHint.RANDOM,
+ open.hint());
+ }
+ }
+
+ for (Open open : vectorFileOpens(opens)) {
+ assertEquals(
+ "the raw vectors are rescored at random: " + open, DataAccessHint.RANDOM, open.hint());
+ }
+ }
+ }
+
+ /** A read state for the flat format holding the vectors of the single segment in {@code dir}. */
+ private static SegmentReadState flatReadState(Directory dir, IOContext context)
+ throws IOException {
+ SegmentInfo info = SegmentInfos.readLatestCommit(dir).info(0).info;
+ FieldInfos fieldInfos =
+ info.getCodec().fieldInfosFormat().read(dir, info, "", IOContext.DEFAULT);
+ return new SegmentReadState(dir, info, fieldInfos, context, "Lucene99HnswVectorsFormat_0");
+ }
+
+ /** Opens of the vectors data file, skipping the ones integrity checks make. */
+ private static List<Open> vectorFileOpens(Opens opens) {
+ List<Open> vectorOpens =
+ opens.endingWith(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION).stream()
+ .filter(open -> open.context().hints().contains(ReadOnceHint.INSTANCE) == false)
+ .toList();
+ MatcherAssert.assertThat("no vectors were opened: " + opens, vectorOpens, not(empty()));
+ return vectorOpens;
+ }
+
+ private static void assertVectorHints(List<Open> allOpens, Opens opens) {
+ List<Open> vectorOpens = new ArrayList<>();
+ for (Open open : allOpens) {
+ // integrity checks read the file once, front to back, and say so with READONCE
+ if (open.context().hints().contains(ReadOnceHint.INSTANCE) == false) {
+ vectorOpens.add(open);
+ }
+ }
+ MatcherAssert.assertThat("no vectors were opened: " + opens, vectorOpens, not(empty()));
+ for (Open open : vectorOpens) {
+ MatcherAssert.assertThat(
+ "opened without saying it holds vectors: " + open,
+ open.context().hints(),
+ hasItem(FileDataHint.KNN_VECTORS));
+ MatcherAssert.assertThat(
+ "opened without saying how it is read: " + open,
+ open.context().hints(),
+ hasItem(DataAccessHint.RANDOM));
+ }
+ }
+
+ private static IndexWriterConfig writerConfig(KnnVectorsFormat format) {
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(format));
+ // keep the vector files out of a compound file so the directory sees them by name
+ iwc.setUseCompoundFile(false);
+ return iwc;
+ }
+
+ private static void addDocuments(IndexWriter w, int count) throws IOException {
+ for (int i = 0; i < count; i++) {
+ Document doc = new Document();
+ doc.add(
+ new KnnFloatVectorField(
+ FIELD,
+ BaseKnnVectorsFormatTestCase.randomNormalizedVector(DIM),
+ VectorSimilarityFunction.DOT_PRODUCT));
+ w.addDocument(doc);
+ }
+ }
+
+ /** One {@link Directory#openInput} call. */
+ private record Open(String name, IOContext context) {
+ DataAccessHint hint() {
+ return context.hints(DataAccessHint.class).findFirst().orElse(null);
+ }
+
+ @Override
+ public String toString() {
+ return name + " " + context.hints();
+ }
+ }
+
+ private static final class Opens {
+ private final List<Open> opens = new ArrayList<>();
+
+ synchronized void record(String name, IOContext context) {
+ opens.add(new Open(name, context));
+ }
+
+ synchronized void clear() {
+ opens.clear();
+ }
+
+ synchronized List<Open> endingWith(String extension) {
+ List<Open> matching = new ArrayList<>();
+ for (Open open : opens) {
+ if (open.name().endsWith("." + extension)) {
+ matching.add(open);
+ }
+ }
+ return matching;
+ }
+
+ @Override
+ public synchronized String toString() {
+ return opens.toString();
+ }
+ }
+
+ private static final class RecordingDirectory extends FilterDirectory {
+ private final Opens opens;
+
+ RecordingDirectory(Directory in, Opens opens) {
+ super(in);
+ this.opens = opens;
+ }
+
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ opens.record(name, context);
+ return super.openInput(name, context);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java b/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java
index 19460d6c583..7a717ed1851 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java
@@ -38,12 +38,16 @@ public class TestIOContext extends LuceneTestCase {
assertEquals(context.context(), newContext.context());
assertEquals(context.mergeInfo(), newContext.mergeInfo());
assertEquals(context.flushInfo(), newContext.flushInfo());
- if (context != newContext) {
- assertTrue(context.mergeInfo() == null && context.flushInfo() == null);
- assertEquals(Set.of(newHint), newContext.hints());
- } else {
- assertTrue(context.mergeInfo() != null || context.flushInfo() != null);
- }
+ assertEquals(Set.of(newHint), newContext.hints());
+ }
+ }
+
+ /** Every context rejects two hints of one type, since a hint is looked up by its type. */
+ public void testRejectsRepeatedHintType() {
+ for (var context : getContexts()) {
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> context.withHints(DataAccessHint.RANDOM, DataAccessHint.SEQUENTIAL));
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
index 3d040d1ca85..538c73a1c64 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java
@@ -148,6 +148,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {
hints.addAll(Arrays.asList(FileTypeHint.values()));
hints.addAll(Arrays.asList(PreloadHint.values()));
hints.addAll(Arrays.asList(ReadOnceHint.values()));
+ hints.addAll(Arrays.asList(NoReuseHint.values()));
for (var hint : hints) {
var context = new DefaultIOContext(hint);
assertEquals(Constants.DEFAULT_READADVICE, func.apply("e", context).get());
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
index 87f3c51eadd..4007a27b111 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java
@@ -44,7 +44,6 @@ import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
@@ -198,10 +197,8 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);
- IOContext.FileOpenHint[] hints = {
- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM
- };
- IOContext context = state.context.withHints(hints);
+ // how these are read is up to whoever wraps this format
+ IOContext context = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);
IndexInput in = null;
boolean success = false;
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java
index a39816de55d..c273949b04a 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java
@@ -36,6 +36,9 @@ import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;
@@ -222,7 +225,10 @@ public final class DedupHnswScalarQuantizedVectorsFormat extends KnnVectorsForma
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ flatVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
index 2c568638b5b..c36b4705af2 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java
@@ -36,6 +36,9 @@ import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.TaskExecutor;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.util.hnsw.HnswGraph;
/**
@@ -208,7 +211,10 @@ public final class DedupHnswVectorsFormat extends KnnVectorsFormat {
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene99HnswVectorsReader(state, FORMAT.fieldsReader(state));
+ return new Lucene99HnswVectorsReader(
+ state,
+ FORMAT.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
index d0f1a4337d3..68378f34408 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java
@@ -48,7 +48,6 @@ import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.R
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;
import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
@@ -274,10 +273,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReader
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);
- IOContext.FileOpenHint[] hints = {
- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM
- };
- IOContext context = state.context.withHints(hints);
+ // how these are read is up to whoever wraps this format
+ IOContext context = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);
IndexInput in = null;
boolean success = false;
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java
index 8cef8d29ce7..f4613d569ad 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java
@@ -29,6 +29,9 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
/**
* A Faiss-based format to create and search vector indexes, using {@link FaissLibrary} to interact
@@ -104,7 +107,10 @@ public final class FaissKnnVectorsFormat extends KnnVectorsFormat {
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new FaissKnnVectorsReader(state, rawVectorsFormat.fieldsReader(state));
+ return new FaissKnnVectorsReader(
+ state,
+ rawVectorsFormat.fieldsReader(
+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));
}
@Override