Apache Lucene open-source search software
Let the structure reading vectors say how they are read (#16682)
Jim Ferenczi79674f38df7d
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 6c4e131d58c..cad59846c8d 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -191,6 +191,13 @@ ImprovementsOptimizations---------------------++* GITHUB#16682: Carry file open hints on merge and flush contexts, which used to drop them. An HNSW+ format now carries the random access hint for the vectors it reads, rather than the flat format+ holding them claiming it whatever reads them. Add NoReuseHint, set by stored fields, term vectors+ and the raw vectors kept only to rescore, so an application can tell those apart from the files it+ wants to keep in memory. (Jim Ferenczi)+* GITHUB#16431: Add support for Block-Max WAND dynamic score skipping in FunctionScoreQueryand DoubleValuesSource using DocValuesSkipper. (Rajat Jain)diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.javaindex 16251990634..a8424c929f8 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsFormat.java@@ -17,6 +17,7 @@package org.apache.lucene.backward_codecs.lucene102;import java.io.IOException;+import java.util.stream.Stream;import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -24,6 +25,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.NoReuseHint;/*** The binary quantization format used here is a per-vector optimized scalar quantization. These@@ -123,7 +126,7 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {@Overridepublic FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {return new Lucene102BinaryQuantizedVectorsReader(- state, rawVectorFormat.fieldsReader(state), scorer);+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), scorer);}@Override@@ -141,4 +144,13 @@ public class Lucene102BinaryQuantizedVectorsFormat extends FlatVectorsFormat {+ rawVectorFormat+ ")";}++ /** The raw vectors are only read back to rescore. */+ private static SegmentReadState rescoreOnly(SegmentReadState state) {+ IOContext.FileOpenHint[] hints =+ Stream.concat(state.context.hints().stream(), Stream.of(NoReuseHint.INSTANCE))+ .distinct()+ .toArray(IOContext.FileOpenHint[]::new);+ return new SegmentReadState(state, state.context.withHints(hints));+ }}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.javaindex e7c6a869150..7770bbc34bf 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java@@ -53,7 +53,6 @@ import org.apache.lucene.search.AcceptDocs;import org.apache.lucene.search.KnnCollector;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;@@ -128,10 +127,8 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReaderversionMeta,VECTOR_DATA_EXTENSION,Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,- // Quantized vectors are accessed randomly from their node ID stored in the HNSW- // graph.- state.context.withHints(- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));+ // how these are read is up to whoever wraps this format+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));} catch (Throwable t) {IOUtils.closeWhileSuppressingExceptions(t, this);throw t;diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.javaindex 698cca05b05..4eba00e1f9c 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102HnswBinaryQuantizedVectorsFormat.java@@ -33,6 +33,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;/**@@ -119,7 +122,10 @@ public class Lucene102HnswBinaryQuantizedVectorsFormat extends KnnVectorsFormat@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.javaindex c1a0bba89b2..26334924b08 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99HnswScalarQuantizedVectorsFormat.java@@ -33,6 +33,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;/**@@ -137,7 +140,10 @@ public class Lucene99HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.javaindex b7f7e16c5cf..f6a83d2a95b 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsFormat.java@@ -18,6 +18,7 @@package org.apache.lucene.backward_codecs.lucene99;import java.io.IOException;+import java.util.stream.Stream;import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -26,6 +27,8 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.NoReuseHint;/*** Format supporting vector quantization, storage, and retrieval@@ -150,6 +153,15 @@ public class Lucene99ScalarQuantizedVectorsFormat extends FlatVectorsFormat {@Overridepublic FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {return new Lucene99ScalarQuantizedVectorsReader(- state, rawVectorFormat.fieldsReader(state), flatVectorScorer);+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), flatVectorScorer);+ }++ /** The raw vectors are only read back to rescore. */+ private static SegmentReadState rescoreOnly(SegmentReadState state) {+ IOContext.FileOpenHint[] hints =+ Stream.concat(state.context.hints().stream(), Stream.of(NoReuseHint.INSTANCE))+ .distinct()+ .toArray(IOContext.FileOpenHint[]::new);+ return new SegmentReadState(state, state.context.withHints(hints));}}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex 57b1f7fe130..e13406ced82 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -44,7 +44,6 @@ import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.internal.hppc.IntObjectHashMap;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;@@ -110,10 +109,8 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadeversionMeta,Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION,Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,- // Quantized vectors are accessed randomly from their node ID stored in the HNSW- // graph.- state.context.withHints(- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));+ // how these are read is up to whoever wraps this format+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));} catch (Throwable t) {IOUtils.closeWhileSuppressingExceptions(t, this);throw t;diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.javaindex b8f8f1b5b26..b7537fdffc4 100644--- a/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bitvectors/HnswBitVectorsFormat.java@@ -41,6 +41,9 @@ import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.Sorter;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.IORunnable;import org.apache.lucene.util.hnsw.HnswGraph;@@ -147,7 +150,10 @@ public final class HnswBitVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.javaindex c7f73cbaa67..5c6634213b4 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104HnswScalarQuantizedVectorsFormat.java@@ -34,6 +34,9 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;@@ -202,7 +205,10 @@ public class Lucene104HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.javaindex 4f5895832c6..c49d8499733 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsFormat.java@@ -25,6 +25,7 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.NoReuseHint;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;import org.apache.lucene.util.quantization.OptimizedScalarQuantizer.QuantizationResult;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;@@ -160,7 +161,7 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {@Overridepublic FlatVectorsReader fieldsReader(SegmentReadState state) throws IOException {return new Lucene104ScalarQuantizedVectorsReader(- state, rawVectorFormat.fieldsReader(state), scorer);+ state, rawVectorFormat.fieldsReader(rescoreOnly(state)), scorer);}@Override@@ -180,4 +181,9 @@ public class Lucene104ScalarQuantizedVectorsFormat extends FlatVectorsFormat {+ rawVectorFormat+ ")";}++ /** The raw vectors are only read back to rescore. */+ private static SegmentReadState rescoreOnly(SegmentReadState state) {+ return new SegmentReadState(state, state.context.union(NoReuseHint.INSTANCE));+ }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex ddb3920a823..13617dd51bd 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -26,7 +26,6 @@ import java.io.IOException;import java.util.HashMap;import java.util.Map;import java.util.Objects;-import java.util.stream.Stream;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.KnnVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -50,7 +49,6 @@ import org.apache.lucene.search.AcceptDocs;import org.apache.lucene.search.KnnCollector;import org.apache.lucene.search.VectorScorer;import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.Directory;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;@@ -91,17 +89,6 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderFlatVectorsReader rawVectorsReader,Lucene104ScalarQuantizedVectorScorer vectorsScorer)throws IOException {- // Quantized vectors are accessed randomly from their node ID stored in the HNSW- // graph.- this(state, rawVectorsReader, vectorsScorer, DataAccessHint.RANDOM);- }-- public Lucene104ScalarQuantizedVectorsReader(- SegmentReadState state,- FlatVectorsReader rawVectorsReader,- Lucene104ScalarQuantizedVectorScorer vectorsScorer,- DataAccessHint accessHint)- throws IOException {this.vectorScorer = vectorsScorer;this.rawVectorsReader = rawVectorsReader;int versionMeta = -1;@@ -128,17 +115,14 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderCodecUtil.checkFooter(meta, priorE);}- final IOContext.FileOpenHint[] hints =- Stream.of(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, accessHint)- .filter(Objects::nonNull)- .toArray(IOContext.FileOpenHint[]::new);quantizedVectorData =openDataInput(state,versionMeta,VECTOR_DATA_EXTENSION,Lucene104ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,- state.context.withHints(hints));+ // how these are read is up to whoever wraps this format+ state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS));} catch (Throwable t) {IOUtils.closeWhileSuppressingExceptions(t, this);throw t;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.javaindex 1d0d0e54a81..c44a7025e4a 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsWriter.java@@ -49,7 +49,10 @@ import org.apache.lucene.index.Sorter;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.internal.hppc.FloatArrayList;import org.apache.lucene.search.VectorScorer;+import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.IOUtils;@@ -396,6 +399,15 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {}}+ /**+ * The quantized query vectors staged for the merge scorer. A graph build reads them by ordinal as+ * it walks, so they are read at random.+ */+ private IOContext queryDataContext() {+ return segmentWriteState.context.withHints(+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM);+ }+/*** Writes merged index-side and query-side records in one pass. Index-side records go to the* quantized vector data file, while query-side records go to a temporary file owned by the@@ -428,7 +440,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {String queryDataName = null;try (IndexOutput queryData =segmentWriteState.directory.createTempOutput(- segmentWriteState.segmentInfo.name, "queries", segmentWriteState.context)) {+ segmentWriteState.segmentInfo.name, "queries", queryDataContext())) {queryDataName = queryData.getName();KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();for (int docV = iterator.nextDoc(); docV != NO_MORE_DOCS; docV = iterator.nextDoc()) {@@ -450,11 +462,7 @@ public class Lucene104ScalarQuantizedVectorsWriter extends FlatVectorsWriter {throw t;}return new PreparedQueryData(- segmentWriteState.directory,- segmentWriteState.context,- fieldInfo,- vectorScorer,- queryDataName);+ segmentWriteState.directory, queryDataContext(), fieldInfo, vectorScorer, queryDataName);}/** Owns one field's temporary query-side records until its graph scorer takes them over. */diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.javaindex 868190a4b10..98f7ccb2423 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java@@ -63,6 +63,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.NoReuseHint;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;@@ -143,7 +144,9 @@ public final class Lucene90CompressingStoredFieldsReader extends StoredFieldsReatry {// Open the data filefieldsStream =- d.openInput(fieldsStreamFN, context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM));+ d.openInput(+ fieldsStreamFN,+ context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM, NoReuseHint.INSTANCE));version =CodecUtil.checkIndexHeader(fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.javaindex d3364f5ea53..a5c4a200b78 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java@@ -42,6 +42,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.NoReuseHint;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;@@ -135,7 +136,8 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWrifieldsStream =directory.createOutput(- IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION), context);+ IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION),+ context.union(NoReuseHint.INSTANCE));CodecUtil.writeIndexHeader(fieldsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.javaindex fd058ed5c83..7d3aa166326 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java@@ -62,6 +62,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.NoReuseHint;import org.apache.lucene.store.RandomAccessInput;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;@@ -148,7 +149,9 @@ public final class Lucene90CompressingTermVectorsReader extends TermVectorsReadefinal String vectorsStreamFN =IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);vectorsStream =- d.openInput(vectorsStreamFN, context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM));+ d.openInput(+ vectorsStreamFN,+ context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM, NoReuseHint.INSTANCE));version =CodecUtil.checkIndexHeader(vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.javaindex c95164b0b86..d47d73abfa9 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java@@ -47,6 +47,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.NoReuseHint;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;@@ -269,7 +270,8 @@ public final class Lucene90CompressingTermVectorsWriter extends TermVectorsWritevectorsStream =directory.createOutput(- IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION), context);+ IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION),+ context.union(NoReuseHint.INSTANCE));CodecUtil.writeIndexHeader(vectorsStream, formatName, VERSION_CURRENT, si.getId(), segmentSuffix);assert CodecUtil.indexHeaderLength(formatName, segmentSuffix)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.javaindex f29417e66c7..d7aa0f1ff6e 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java@@ -22,8 +22,6 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVeimport java.io.IOException;import java.util.Map;-import java.util.Objects;-import java.util.stream.Stream;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;import org.apache.lucene.codecs.hnsw.FlatVectorsScorer;@@ -48,7 +46,6 @@ import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;-import org.apache.lucene.store.IOContext.FileOpenHint;import org.apache.lucene.store.IndexInput;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;@@ -72,27 +69,11 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {public Lucene99FlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer)throws IOException {- this(state, scorer, DataAccessHint.RANDOM);- }-- /**- * Creates a Lucene99FlatVectorsReader.- *- * @param state the segment read state- * @param scorer the flat vectors scorer- * @param accessHint a data access hint, or null- */- public Lucene99FlatVectorsReader(- SegmentReadState state, FlatVectorsScorer scorer, DataAccessHint accessHint)- throws IOException {int versionMeta = readMetadata(state);this.vectorScorer = scorer;this.fieldInfos = state.fieldInfos;- FileOpenHint[] hints =- Stream.of(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, accessHint)- .filter(Objects::nonNull)- .toArray(FileOpenHint[]::new);- dataContext = state.context.withHints(hints);+ // how these are read is up to whoever wraps this format+ dataContext = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);try {vectorData =openDataInput(diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.javaindex 875a6325431..e576a13b3a1 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsFormat.java@@ -29,6 +29,9 @@ import org.apache.lucene.index.MergeScheduler;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;import org.apache.lucene.util.hnsw.HnswGraphBuilder;@@ -311,7 +314,10 @@ public final class Lucene99HnswVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.javaindex 17ff9f4e7da..abd76d0fbc8 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsWriter.java@@ -51,6 +51,9 @@ import org.apache.lucene.index.Sorter;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.util.IORunnable;import org.apache.lucene.util.IOUtils;@@ -485,7 +488,9 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {if (flatVectorsReader instanceof QuantizedVectorsReader quantizedVectorsReader&& fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32)) {return quantizedVectorsReader.getRandomVectorScorerSupplierForMerge(- fieldInfo, segmentWriteState);+ fieldInfo,+ segmentWriteState.withHints(+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));}return null;}@@ -537,7 +542,8 @@ public final class Lucene99HnswVectorsWriter extends KnnVectorsWriter {segmentWriteState.directory,segmentWriteState.segmentInfo,segmentWriteState.fieldInfos,- segmentWriteState.context,+ segmentWriteState.context.union(+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM),segmentWriteState.segmentSuffix);flatVectorsReader = flatVectorsFormat.fieldsReader(readState);}diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.javaindex 2d98549d429..01ae33ecd11 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java@@ -67,6 +67,15 @@ public class SegmentReadState {this.segmentSuffix = segmentSuffix;}+ /** Create a {@code SegmentReadState} with a new {@link IOContext}. */+ public SegmentReadState(SegmentReadState other, IOContext context) {+ this.directory = other.directory;+ this.segmentInfo = other.segmentInfo;+ this.fieldInfos = other.fieldInfos;+ this.context = context;+ this.segmentSuffix = other.segmentSuffix;+ }+/** Create a {@code SegmentReadState}. */public SegmentReadState(SegmentReadState other, String newSegmentSuffix) {this.directory = other.directory;@@ -75,4 +84,9 @@ public class SegmentReadState {this.context = other.context;this.segmentSuffix = newSegmentSuffix;}++ /** Returns a copy of this state whose context carries the given hints in place of its own. */+ public SegmentReadState withHints(IOContext.FileOpenHint... hints) {+ return new SegmentReadState(this, context.withHints(hints));+ }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.javaindex 4854a0a423d..0c94ed558a7 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java@@ -113,6 +113,19 @@ public class SegmentWriteState {this.context = context;}+ /** Create a shallow copy of {@link SegmentWriteState} with a new {@link IOContext}. */+ public SegmentWriteState(SegmentWriteState state, IOContext context) {+ infoStream = state.infoStream;+ directory = state.directory;+ segmentInfo = state.segmentInfo;+ fieldInfos = state.fieldInfos;+ this.context = context;+ segmentSuffix = state.segmentSuffix;+ segUpdates = state.segUpdates;+ delCountOnFlush = state.delCountOnFlush;+ liveDocs = state.liveDocs;+ }+/** Create a shallow copy of {@link SegmentWriteState} with a new segment suffix. */public SegmentWriteState(SegmentWriteState state, String segmentSuffix) {infoStream = state.infoStream;@@ -144,4 +157,9 @@ public class SegmentWriteState {}return true;}++ /** Returns a copy of this state whose context carries the given hints in place of its own. */+ public SegmentWriteState withHints(IOContext.FileOpenHint... hints) {+ return new SegmentWriteState(this, context.withHints(hints));+ }}diff --git a/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java b/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.javaindex e3e4553a8df..8f8b387990f 100644--- a/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java+++ b/lucene/core/src/java/org/apache/lucene/store/DefaultIOContext.java@@ -25,16 +25,20 @@ import java.util.stream.Collectors;record DefaultIOContext(Set<FileOpenHint> hints) implements IOContext {public DefaultIOContext(Set<FileOpenHint> hints) {- this.hints = Set.copyOf(Objects.requireNonNull(hints));+ this.hints = checkOneHintPerType(hints);+ }- // there should only be one hint of each type in the IOContext+ /** Returns an immutable copy of {@code hints}, rejecting any type given more than once. */+ static Set<FileOpenHint> checkOneHintPerType(Set<FileOpenHint> hints) {+ Set<FileOpenHint> copy = Set.copyOf(Objects.requireNonNull(hints));Map<Class<? extends FileOpenHint>, List<FileOpenHint>> hintClasses =- hints.stream().collect(Collectors.groupingBy(IOContext.FileOpenHint::getClass));+ copy.stream().collect(Collectors.groupingBy(IOContext.FileOpenHint::getClass));for (var hintType : hintClasses.entrySet()) {if (hintType.getValue().size() > 1) {throw new IllegalArgumentException("Multiple hints of type " + hintType + " specified");}}+ return copy;}public DefaultIOContext(FileOpenHint... hints) {diff --git a/lucene/core/src/java/org/apache/lucene/store/IOContext.java b/lucene/core/src/java/org/apache/lucene/store/IOContext.javaindex 357e23ab0b8..dabbe9c6d1d 100644--- a/lucene/core/src/java/org/apache/lucene/store/IOContext.java+++ b/lucene/core/src/java/org/apache/lucene/store/IOContext.java@@ -16,6 +16,8 @@*/package org.apache.lucene.store;+import java.util.Collections;+import java.util.HashSet;import java.util.Objects;import java.util.Set;import java.util.stream.Stream;@@ -64,7 +66,12 @@ public interface IOContext {/** Returns an {@link IOContext} for merging with the specified {@link MergeInfo} */static IOContext merge(MergeInfo mergeInfo) {+ return merge(mergeInfo, Set.of());+ }++ private static IOContext merge(MergeInfo mergeInfo, Set<FileOpenHint> hints) {Objects.requireNonNull(mergeInfo);+ Set<FileOpenHint> copy = DefaultIOContext.checkOneHintPerType(hints);return new IOContext() {@Overridepublic Context context() {@@ -83,19 +90,24 @@ public interface IOContext {@Overridepublic Set<FileOpenHint> hints() {- return Set.of();+ return copy;}@Overridepublic IOContext withHints(FileOpenHint... hints) {- return this;+ return merge(mergeInfo, Set.of(hints));}};}/** Returns an {@link IOContext} for flushing with the specified {@link FlushInfo} */static IOContext flush(FlushInfo flushInfo) {+ return flush(flushInfo, Set.of());+ }++ private static IOContext flush(FlushInfo flushInfo, Set<FileOpenHint> hints) {Objects.requireNonNull(flushInfo);+ Set<FileOpenHint> copy = DefaultIOContext.checkOneHintPerType(hints);return new IOContext() {@Overridepublic Context context() {@@ -114,12 +126,12 @@ public interface IOContext {@Overridepublic Set<FileOpenHint> hints() {- return Set.of();+ return copy;}@Overridepublic IOContext withHints(FileOpenHint... hints) {- return this;+ return flush(flushInfo, Set.of(hints));}};}@@ -142,8 +154,7 @@ public interface IOContext {}/**- * Returns an IOContext with the given hints, if it makes sense to do so for this specific- * context. Otherwise, returns this context.+ * Returns an IOContext with the given hints.** <p>The returned context has the same {@link #context()}, {@link #mergeInfo()}, and {@link* #flushInfo()} as this context.@@ -151,4 +162,16 @@ public interface IOContext {* <p>This instance is immutable and unaffected by this method call.*/IOContext withHints(FileOpenHint... hints);++ /**+ * Returns an IOContext with the given hints added to the ones it already has. A hint it already+ * has is added once.+ *+ * @throws IllegalArgumentException if that would give it two different hints of the same type+ */+ default IOContext union(FileOpenHint... hints) {+ Set<FileOpenHint> merged = new HashSet<>(hints());+ Collections.addAll(merged, hints);+ return withHints(merged.toArray(FileOpenHint[]::new));+ }}diff --git a/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.java b/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.javanew file mode 100644index 00000000000..e112d5dd0ad--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/store/NoReuseHint.java@@ -0,0 +1,29 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.store;++/**+ * A hint that a file is not worth keeping in memory. It may be read often; what it lacks is reuse+ * worth caching for, because its reads repeat in no pattern a cache can exploit, or because it is+ * large enough next to the rest of the index that holding it would push out data that does benefit.+ *+ * <p>Close to {@link ReadOnceHint}, and weaker: it does not claim the file is read once, or in+ * order, or even that a read never revisits bytes another read has touched.+ */+public enum NoReuseHint implements IOContext.FileOpenHint {+ INSTANCE+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.javanew file mode 100644index 00000000000..89f856b499b--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorAccessHints.java@@ -0,0 +1,321 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene99;++import static org.hamcrest.Matchers.empty;+import static org.hamcrest.Matchers.hasItem;+import static org.hamcrest.Matchers.not;++import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import org.apache.lucene.codecs.KnnVectorsFormat;+import org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.FieldInfos;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.SegmentInfo;+import org.apache.lucene.index.SegmentInfos;+import org.apache.lucene.index.SegmentReadState;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.ReadOnceHint;+import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;+import org.hamcrest.MatcherAssert;++/**+ * Who says how vector files are read: the format that walks them, rather than the one that holds+ * them. Directories decide what to make of it.+ */+public class TestVectorAccessHints extends LuceneTestCase {++ private static final int DIM = 8;+ private static final String FIELD = "field";++ /** The vectors the graph build scores against, reopened from the segment the merge just wrote. */+ public void testMergedVectorsStateTheirAccessPattern() throws Exception {+ Opens opens = new Opens();+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {+ // threshold 0: always build the graph, whatever the segment size+ try (IndexWriter w =+ new IndexWriter(+ dir,+ writerConfig(+ new Lucene99HnswVectorsFormat(+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,+ 0)))) {+ for (int segment = 0; segment < 2; segment++) {+ addDocuments(w, 128);+ w.commit();+ }+ opens.clear();+ w.forceMerge(1);+ }+ assertVectorHints(opens.endingWith(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION), opens);+ }+ }++ /** The quantized vectors a merge writes to a temporary file and scores against. */+ public void testMergeTemporaryFileStatesItsAccessPattern() throws Exception {+ Opens opens = new Opens();+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {+ // an asymmetric encoding quantizes the query side differently, which is what makes the merge+ // write these query vectors to a temporary file+ try (IndexWriter w =+ new IndexWriter(+ dir,+ writerConfig(+ new Lucene104HnswScalarQuantizedVectorsFormat(+ ScalarEncoding.SINGLE_BIT_QUERY_NIBBLE,+ Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,+ Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,+ 1,+ null,+ 0)))) {+ for (int segment = 0; segment < 2; segment++) {+ addDocuments(w, 128);+ w.commit();+ }+ opens.clear();+ w.forceMerge(1);+ }++ List<Open> temps = new ArrayList<>();+ for (Open open : opens.endingWith("tmp")) {+ if (open.name().contains("_queries_")) {+ temps.add(open);+ }+ }+ MatcherAssert.assertThat("no temporary file was read back: " + opens, temps, not(empty()));+ assertVectorHints(temps, opens);+ }+ }++ /**+ * A flat format on its own says nothing about how its vectors are read. Only a format that walks+ * them, like HNSW, knows that, so only it asks for random access.+ */+ public void testFlatFormatClaimsNoAccessPattern() throws Exception {+ Opens opens = new Opens();+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {+ try (IndexWriter w = new IndexWriter(dir, writerConfig(new Lucene99HnswVectorsFormat()))) {+ addDocuments(w, 16);+ w.commit();+ }++ opens.clear();+ new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())+ .fieldsReader(flatReadState(dir, IOContext.DEFAULT))+ .close();++ for (Open open : vectorFileOpens(opens)) {+ assertNull("a flat format assumed how its vectors are read: " + open, open.hint());+ }+ }+ }++ /** A caller that says how it reads the vectors is followed, whatever the format would say. */+ public void testReadStateAccessHintWins() throws Exception {+ Opens opens = new Opens();+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {+ try (IndexWriter w = new IndexWriter(dir, writerConfig(new Lucene99HnswVectorsFormat()))) {+ addDocuments(w, 16);+ w.commit();+ }++ opens.clear();+ new Lucene99FlatVectorsFormat(FlatVectorScorerUtil.getLucene99FlatVectorsScorer())+ .fieldsReader(+ flatReadState(+ dir,+ IOContext.DEFAULT.withHints(+ FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.SEQUENTIAL)))+ .close();++ for (Open open : vectorFileOpens(opens)) {+ assertEquals(+ "the format ignored what the caller asked for: " + open,+ DataAccessHint.SEQUENTIAL,+ open.hint());+ }+ }+ }++ /**+ * A graph scores the quantized vectors as it walks and reads the raw ones back to rescore. Both+ * are read at random, and both take the access pattern asked of the format holding them.+ */+ public void testQuantizedAndRawVectorsAreReadAtRandom() throws Exception {+ Opens opens = new Opens();+ try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {+ try (IndexWriter w =+ new IndexWriter(dir, writerConfig(new Lucene104HnswScalarQuantizedVectorsFormat()))) {+ opens.clear();+ addDocuments(w, 16);+ w.commit();+ try (DirectoryReader reader = DirectoryReader.open(w)) {+ assertEquals(1, reader.leaves().size());+ }+ }++ List<Open> quantized = opens.endingWith("veq");+ MatcherAssert.assertThat(+ "no quantized vectors were opened: " + opens, quantized, not(empty()));+ for (Open open : quantized) {+ if (open.context().hints().contains(ReadOnceHint.INSTANCE) == false) {+ assertEquals(+ "the quantized vectors are scored in graph order: " + open,+ DataAccessHint.RANDOM,+ open.hint());+ }+ }++ for (Open open : vectorFileOpens(opens)) {+ assertEquals(+ "the raw vectors are rescored at random: " + open, DataAccessHint.RANDOM, open.hint());+ }+ }+ }++ /** A read state for the flat format holding the vectors of the single segment in {@code dir}. */+ private static SegmentReadState flatReadState(Directory dir, IOContext context)+ throws IOException {+ SegmentInfo info = SegmentInfos.readLatestCommit(dir).info(0).info;+ FieldInfos fieldInfos =+ info.getCodec().fieldInfosFormat().read(dir, info, "", IOContext.DEFAULT);+ return new SegmentReadState(dir, info, fieldInfos, context, "Lucene99HnswVectorsFormat_0");+ }++ /** Opens of the vectors data file, skipping the ones integrity checks make. */+ private static List<Open> vectorFileOpens(Opens opens) {+ List<Open> vectorOpens =+ opens.endingWith(Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION).stream()+ .filter(open -> open.context().hints().contains(ReadOnceHint.INSTANCE) == false)+ .toList();+ MatcherAssert.assertThat("no vectors were opened: " + opens, vectorOpens, not(empty()));+ return vectorOpens;+ }++ private static void assertVectorHints(List<Open> allOpens, Opens opens) {+ List<Open> vectorOpens = new ArrayList<>();+ for (Open open : allOpens) {+ // integrity checks read the file once, front to back, and say so with READONCE+ if (open.context().hints().contains(ReadOnceHint.INSTANCE) == false) {+ vectorOpens.add(open);+ }+ }+ MatcherAssert.assertThat("no vectors were opened: " + opens, vectorOpens, not(empty()));+ for (Open open : vectorOpens) {+ MatcherAssert.assertThat(+ "opened without saying it holds vectors: " + open,+ open.context().hints(),+ hasItem(FileDataHint.KNN_VECTORS));+ MatcherAssert.assertThat(+ "opened without saying how it is read: " + open,+ open.context().hints(),+ hasItem(DataAccessHint.RANDOM));+ }+ }++ private static IndexWriterConfig writerConfig(KnnVectorsFormat format) {+ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(format));+ // keep the vector files out of a compound file so the directory sees them by name+ iwc.setUseCompoundFile(false);+ return iwc;+ }++ private static void addDocuments(IndexWriter w, int count) throws IOException {+ for (int i = 0; i < count; i++) {+ Document doc = new Document();+ doc.add(+ new KnnFloatVectorField(+ FIELD,+ BaseKnnVectorsFormatTestCase.randomNormalizedVector(DIM),+ VectorSimilarityFunction.DOT_PRODUCT));+ w.addDocument(doc);+ }+ }++ /** One {@link Directory#openInput} call. */+ private record Open(String name, IOContext context) {+ DataAccessHint hint() {+ return context.hints(DataAccessHint.class).findFirst().orElse(null);+ }++ @Override+ public String toString() {+ return name + " " + context.hints();+ }+ }++ private static final class Opens {+ private final List<Open> opens = new ArrayList<>();++ synchronized void record(String name, IOContext context) {+ opens.add(new Open(name, context));+ }++ synchronized void clear() {+ opens.clear();+ }++ synchronized List<Open> endingWith(String extension) {+ List<Open> matching = new ArrayList<>();+ for (Open open : opens) {+ if (open.name().endsWith("." + extension)) {+ matching.add(open);+ }+ }+ return matching;+ }++ @Override+ public synchronized String toString() {+ return opens.toString();+ }+ }++ private static final class RecordingDirectory extends FilterDirectory {+ private final Opens opens;++ RecordingDirectory(Directory in, Opens opens) {+ super(in);+ this.opens = opens;+ }++ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ opens.record(name, context);+ return super.openInput(name, context);+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java b/lucene/core/src/test/org/apache/lucene/store/TestIOContext.javaindex 19460d6c583..7a717ed1851 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestIOContext.java@@ -38,12 +38,16 @@ public class TestIOContext extends LuceneTestCase {assertEquals(context.context(), newContext.context());assertEquals(context.mergeInfo(), newContext.mergeInfo());assertEquals(context.flushInfo(), newContext.flushInfo());- if (context != newContext) {- assertTrue(context.mergeInfo() == null && context.flushInfo() == null);- assertEquals(Set.of(newHint), newContext.hints());- } else {- assertTrue(context.mergeInfo() != null || context.flushInfo() != null);- }+ assertEquals(Set.of(newHint), newContext.hints());+ }+ }++ /** Every context rejects two hints of one type, since a hint is looked up by its type. */+ public void testRejectsRepeatedHintType() {+ for (var context : getContexts()) {+ expectThrows(+ IllegalArgumentException.class,+ () -> context.withHints(DataAccessHint.RANDOM, DataAccessHint.SEQUENTIAL));}}diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.javaindex 3d040d1ca85..538c73a1c64 100644--- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java+++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java@@ -148,6 +148,7 @@ public class TestMMapDirectory extends BaseDirectoryTestCase {hints.addAll(Arrays.asList(FileTypeHint.values()));hints.addAll(Arrays.asList(PreloadHint.values()));hints.addAll(Arrays.asList(ReadOnceHint.values()));+ hints.addAll(Arrays.asList(NoReuseHint.values()));for (var hint : hints) {var context = new DefaultIOContext(hint);assertEquals(Constants.DEFAULT_READADVICE, func.apply("e", context).get());diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.javaindex 87f3c51eadd..4007a27b111 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupFlatVectorsReader.java@@ -44,7 +44,6 @@ import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;@@ -198,10 +197,8 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);- IOContext.FileOpenHint[] hints = {- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM- };- IOContext context = state.context.withHints(hints);+ // how these are read is up to whoever wraps this format+ IOContext context = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);IndexInput in = null;boolean success = false;diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.javaindex a39816de55d..c273949b04a 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswScalarQuantizedVectorsFormat.java@@ -36,6 +36,9 @@ import org.apache.lucene.index.MergeScheduler;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;import org.apache.lucene.util.quantization.QuantizedByteVectorValues.ScalarEncoding;@@ -222,7 +225,10 @@ public final class DedupHnswScalarQuantizedVectorsFormat extends KnnVectorsForma@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ flatVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.javaindex 2c568638b5b..c36b4705af2 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupHnswVectorsFormat.java@@ -36,6 +36,9 @@ import org.apache.lucene.index.MergeScheduler;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.search.TaskExecutor;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.util.hnsw.HnswGraph;/**@@ -208,7 +211,10 @@ public final class DedupHnswVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new Lucene99HnswVectorsReader(state, FORMAT.fieldsReader(state));+ return new Lucene99HnswVectorsReader(+ state,+ FORMAT.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Overridediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.javaindex d0f1a4337d3..68378f34408 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorsReader.java@@ -48,7 +48,6 @@ import org.apache.lucene.sandbox.codecs.dedup.DedupScalarQuantizedVectorValues.Rimport org.apache.lucene.sandbox.codecs.dedup.DedupUtil.GroupInfo;import org.apache.lucene.sandbox.codecs.dedup.DedupUtil.ReadFieldInfo;import org.apache.lucene.store.ChecksumIndexInput;-import org.apache.lucene.store.DataAccessHint;import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;@@ -274,10 +273,8 @@ final class DedupScalarQuantizedVectorsReader extends FlatVectorsReaderIndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, vectorDataExtension);- IOContext.FileOpenHint[] hints = {- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM- };- IOContext context = state.context.withHints(hints);+ // how these are read is up to whoever wraps this format+ IOContext context = state.context.union(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);IndexInput in = null;boolean success = false;diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.javaindex 8cef8d29ce7..f4613d569ad 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/faiss/FaissKnnVectorsFormat.java@@ -29,6 +29,9 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat;import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat;import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.SegmentWriteState;+import org.apache.lucene.store.DataAccessHint;+import org.apache.lucene.store.FileDataHint;+import org.apache.lucene.store.FileTypeHint;/*** A Faiss-based format to create and search vector indexes, using {@link FaissLibrary} to interact@@ -104,7 +107,10 @@ public final class FaissKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {- return new FaissKnnVectorsReader(state, rawVectorsFormat.fieldsReader(state));+ return new FaissKnnVectorsReader(+ state,+ rawVectorsFormat.fieldsReader(+ state.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM)));}@Override