Apache Lucene open-source search software
Use `ReadAdvice#RANDOM` when appropriate. (#13222)
Adrien Grand4ea2bae1198b
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.javaindex 569ca55400b..1b5c307294b 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsReader.java@@ -58,6 +58,7 @@ import org.apache.lucene.store.DataInput;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BitUtil;import org.apache.lucene.util.BytesRef;@@ -128,7 +129,7 @@ public final class Lucene90CompressingStoredFieldsReader extends StoredFieldsReaChecksumIndexInput metaIn = null;try {// Open the data file- fieldsStream = d.openInput(fieldsStreamFN, context);+ fieldsStream = d.openInput(fieldsStreamFN, context.withReadAdvice(ReadAdvice.RANDOM));version =CodecUtil.checkIndexHeader(fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.javaindex 73b9eafaa4d..b06da63153c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsReader.java@@ -59,6 +59,7 @@ import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.RandomAccessInput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;@@ -134,7 +135,7 @@ public final class Lucene90CompressingTermVectorsReader extends TermVectorsReade// Open the data filefinal String vectorsStreamFN =IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);- vectorsStream = d.openInput(vectorsStreamFN, context);+ vectorsStream = d.openInput(vectorsStreamFN, context.withReadAdvice(ReadAdvice.RANDOM));version =CodecUtil.checkIndexHeader(vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.javaindex 81ea9b07023..b08d3ceeb91 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java@@ -38,7 +38,9 @@ import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;@@ -66,7 +68,10 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {state,versionMeta,Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION,- Lucene99FlatVectorsFormat.VECTOR_DATA_CODEC_NAME);+ Lucene99FlatVectorsFormat.VECTOR_DATA_CODEC_NAME,+ // Flat formats are used to randomly access vectors from their node ID that is stored+ // in the HNSW graph.+ state.context.withReadAdvice(ReadAdvice.RANDOM));success = true;} finally {if (success == false) {@@ -102,11 +107,15 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {}private static IndexInput openDataInput(- SegmentReadState state, int versionMeta, String fileExtension, String codecName)+ SegmentReadState state,+ int versionMeta,+ String fileExtension,+ String codecName,+ IOContext context)throws IOException {String fileName =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension);- IndexInput in = state.directory.openInput(fileName, state.context);+ IndexInput in = state.directory.openInput(fileName, context);boolean success = false;try {int versionVectorData =diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.javaindex 9f1239a1781..0491507aaf4 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsWriter.java@@ -44,8 +44,10 @@ import org.apache.lucene.index.SegmentWriteState;import org.apache.lucene.index.Sorter;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.search.DocIdSetIterator;+import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.ArrayUtil;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;@@ -283,10 +285,13 @@ public final class Lucene99FlatVectorsWriter extends FlatVectorsWriter {CodecUtil.writeFooter(tempVectorData);IOUtils.close(tempVectorData);- // copy the temporary file vectors to the actual data file+ // This temp file will be accessed in a random-access fashion to construct the HNSW graph.+ // Note: don't use the context from the state, which is a flush/merge context, not expecting+ // to perform random reads.vectorDataInput =segmentWriteState.directory.openInput(- tempVectorData.getName(), segmentWriteState.context);+ tempVectorData.getName(), IOContext.DEFAULT.withReadAdvice(ReadAdvice.RANDOM));+ // copy the temporary file vectors to the actual data filevectorData.copyBytes(vectorDataInput, vectorDataInput.length() - CodecUtil.footerLength());CodecUtil.retrieveChecksum(vectorDataInput);long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.javaindex 517ef765684..a72e42425fb 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java@@ -40,8 +40,10 @@ import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.search.KnnCollector;import org.apache.lucene.store.ChecksumIndexInput;import org.apache.lucene.store.DataInput;+import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.RandomAccessInput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.Bits;import org.apache.lucene.util.IOUtils;@@ -102,7 +104,8 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReaderstate,versionMeta,Lucene99HnswVectorsFormat.VECTOR_INDEX_EXTENSION,- Lucene99HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME);+ Lucene99HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME,+ state.context.withReadAdvice(ReadAdvice.RANDOM));success = true;} finally {if (success == false) {@@ -112,11 +115,15 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader}private static IndexInput openDataInput(- SegmentReadState state, int versionMeta, String fileExtension, String codecName)+ SegmentReadState state,+ int versionMeta,+ String fileExtension,+ String codecName,+ IOContext context)throws IOException {String fileName =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension);- IndexInput in = state.directory.openInput(fileName, state.context);+ IndexInput in = state.directory.openInput(fileName, context);boolean success = false;try {int versionVectorData =diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.javaindex 3c8b8f0c490..dfd447de24c 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java@@ -36,7 +36,9 @@ import org.apache.lucene.index.SegmentReadState;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.ChecksumIndexInput;+import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;+import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.Accountable;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;@@ -93,7 +95,10 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReadestate,versionMeta,Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_EXTENSION,- Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME);+ Lucene99ScalarQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,+ // Quantized vectors are accessed randomly from their node ID stored in the HNSW+ // graph.+ state.context.withReadAdvice(ReadAdvice.RANDOM));success = true;} finally {if (success == false) {@@ -166,11 +171,15 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade}private static IndexInput openDataInput(- SegmentReadState state, int versionMeta, String fileExtension, String codecName)+ SegmentReadState state,+ int versionMeta,+ String fileExtension,+ String codecName,+ IOContext context)throws IOException {String fileName =IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, fileExtension);- IndexInput in = state.directory.openInput(fileName, state.context);+ IndexInput in = state.directory.openInput(fileName, context);boolean success = false;try {int versionVectorData =