Apache Lucene open-source search software
Update the IOContext rather than the ReadAdvice on IndexInput (#14702)
Simon Cooper9936b20153bc
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.javaindex 219e89a4f6e..2a34aa69313 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsReader.java@@ -124,7 +124,7 @@ public abstract class KnnVectorsReader implements Closeable {** <p>The default implementation returns {@code this}*/- public KnnVectorsReader getMergeInstance() {+ public KnnVectorsReader getMergeInstance() throws IOException {return this;}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatVectorsReader.javaindex b03e42cfbb4..4816f8ad4ff 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/hnsw/FlatVectorsReader.java@@ -96,7 +96,7 @@ public abstract class FlatVectorsReader extends KnnVectorsReader implements Acco* <p>The default implementation returns {@code this}*/@Override- public FlatVectorsReader getMergeInstance() {+ public FlatVectorsReader getMergeInstance() throws IOException {return this;}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.javaindex b42f9e8a63d..607f1154875 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99FlatVectorsReader.java@@ -21,7 +21,6 @@ import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readSiimport static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.readVectorEncoding;import java.io.IOException;-import java.io.UncheckedIOException;import java.util.Map;import org.apache.lucene.codecs.CodecUtil;import org.apache.lucene.codecs.hnsw.FlatVectorsReader;@@ -45,7 +44,6 @@ import org.apache.lucene.store.FileDataHint;import org.apache.lucene.store.FileTypeHint;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.RamUsageEstimator;import org.apache.lucene.util.hnsw.RandomVectorScorer;@@ -63,12 +61,17 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {private final IntObjectHashMap<FieldEntry> fields = new IntObjectHashMap<>();private final IndexInput vectorData;private final FieldInfos fieldInfos;+ private final IOContext dataContext;public Lucene99FlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer)throws IOException {super(scorer);int versionMeta = readMetadata(state);this.fieldInfos = state.fieldInfos;+ // Flat formats are used to randomly access vectors from their node ID that is stored+ // in the HNSW graph.+ dataContext =+ state.context.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM);try {vectorData =openDataInput(@@ -76,10 +79,7 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {versionMeta,Lucene99FlatVectorsFormat.VECTOR_DATA_EXTENSION,Lucene99FlatVectorsFormat.VECTOR_DATA_CODEC_NAME,- // Flat formats are used to randomly access vectors from their node ID that is stored- // in the HNSW graph.- state.context.withHints(- FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));+ dataContext);} catch (Throwable t) {IOUtils.closeWhileSuppressingExceptions(t, this);throw t;@@ -177,14 +177,10 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {}@Override- public FlatVectorsReader getMergeInstance() {- try {- // Update the read advice since vectors are guaranteed to be accessed sequentially for merge- this.vectorData.updateReadAdvice(ReadAdvice.SEQUENTIAL);- return this;- } catch (IOException exception) {- throw new UncheckedIOException(exception);- }+ public FlatVectorsReader getMergeInstance() throws IOException {+ // Update the read advice since vectors are guaranteed to be accessed sequentially for merge+ vectorData.updateIOContext(dataContext.withHints(DataAccessHint.SEQUENTIAL));+ return this;}private FieldEntry getFieldEntryOrThrow(String field) {@@ -276,7 +272,7 @@ public final class Lucene99FlatVectorsReader extends FlatVectorsReader {public void finishMerge() throws IOException {// This makes sure that the access pattern hint is reverted back since HNSW implementation// needs it- this.vectorData.updateReadAdvice(ReadAdvice.RANDOM);+ vectorData.updateIOContext(dataContext);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.javaindex a83b1635032..c86df8e947d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99HnswVectorsReader.java@@ -131,7 +131,7 @@ public final class Lucene99HnswVectorsReader extends KnnVectorsReader}@Override- public KnnVectorsReader getMergeInstance() {+ public KnnVectorsReader getMergeInstance() throws IOException {return new Lucene99HnswVectorsReader(this, this.flatVectorsReader.getMergeInstance());}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.javaindex 1fd0f329a8a..1df71485713 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldKnnVectorsFormat.java@@ -238,7 +238,7 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {}}- private FieldsReader(final FieldsReader fieldsReader) {+ private FieldsReader(final FieldsReader fieldsReader) throws IOException {this.fieldInfos = fieldsReader.fieldInfos;for (FieldInfo fi : this.fieldInfos) {if (fi.hasVectorValues() && fieldsReader.fields.containsKey(fi.number)) {@@ -248,7 +248,7 @@ public abstract class PerFieldKnnVectorsFormat extends KnnVectorsFormat {}@Override- public KnnVectorsReader getMergeInstance() {+ public KnnVectorsReader getMergeInstance() throws IOException {return new FieldsReader(this);}diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java b/lucene/core/src/java/org/apache/lucene/store/IndexInput.javaindex 4997732ec3f..0a6911d116e 100644--- a/lucene/core/src/java/org/apache/lucene/store/IndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/IndexInput.java@@ -227,12 +227,12 @@ public abstract class IndexInput extends DataInput implements Closeable {public void prefetch(long offset, long length) throws IOException {}/**- * Optional method: Give a hint to this input about the change in read access pattern. IndexInput+ * Optional method: Updates the {@code IOContext} to specify a new read access pattern. IndexInput* implementations may take advantage of this hint to optimize reads from storage.** <p>The default implementation is a no-op.*/- public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {}+ public void updateIOContext(IOContext context) throws IOException {}/*** Returns a hint whether all the contents of this input are resident in physical memory. It's adiff --git a/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.javaindex 889333f1763..3967b400a41 100644--- a/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java+++ b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java@@ -357,7 +357,11 @@ abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegme}@Override- public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {+ public void updateIOContext(IOContext context) throws IOException {+ updateReadAdvice(toReadAdvice.apply(context));+ }++ private void updateReadAdvice(ReadAdvice readAdvice) throws IOException {if (NATIVE_ACCESS.isEmpty()) {return;}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.javaindex 16b4ab652f7..ccccf35844e 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/asserting/AssertingKnnVectorsFormat.java@@ -113,16 +113,17 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {public static class AssertingKnnVectorsReader extends KnnVectorsReaderimplements HnswGraphProvider {public final KnnVectorsReader delegate;- final FieldInfos fis;- final boolean mergeInstance;- AtomicInteger mergeInstanceCount = new AtomicInteger();- AtomicInteger finishMergeCount = new AtomicInteger();+ private final FieldInfos fis;+ private final boolean mergeInstance;+ private final AtomicInteger mergeInstanceCount = new AtomicInteger();+ private final AtomicInteger finishMergeCount = new AtomicInteger();- AssertingKnnVectorsReader(KnnVectorsReader delegate, FieldInfos fis) {+ private AssertingKnnVectorsReader(KnnVectorsReader delegate, FieldInfos fis) {this(delegate, fis, false);}- AssertingKnnVectorsReader(KnnVectorsReader delegate, FieldInfos fis, boolean mergeInstance) {+ private AssertingKnnVectorsReader(+ KnnVectorsReader delegate, FieldInfos fis, boolean mergeInstance) {assert delegate != null;this.delegate = delegate;this.fis = fis;@@ -136,6 +137,8 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic FloatVectorValues getFloatVectorValues(String field) throws IOException {+ assert mergeInstanceCount.get() == finishMergeCount.get() || mergeInstance+ : "Called on the wrong instance";FieldInfo fi = fis.fieldInfo(field);assert fi != null&& fi.getVectorDimension() > 0@@ -150,6 +153,8 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic ByteVectorValues getByteVectorValues(String field) throws IOException {+ assert mergeInstanceCount.get() == finishMergeCount.get() || mergeInstance+ : "Called on the wrong instance";FieldInfo fi = fis.fieldInfo(field);assert fi != null&& fi.getVectorDimension() > 0@@ -165,7 +170,7 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs)throws IOException {- assert !mergeInstance;+ assert mergeInstanceCount.get() == finishMergeCount.get() : "There is an open merge instance";FieldInfo fi = fis.fieldInfo(field);assert fi != null&& fi.getVectorDimension() > 0@@ -176,7 +181,7 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs)throws IOException {- assert !mergeInstance;+ assert mergeInstanceCount.get() == finishMergeCount.get() : "There is an open merge instance";FieldInfo fi = fis.fieldInfo(field);assert fi != null&& fi.getVectorDimension() > 0@@ -185,15 +190,28 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {}@Override- public KnnVectorsReader getMergeInstance() {- assert !mergeInstance;+ public KnnVectorsReader getMergeInstance() throws IOException {var mergeVectorsReader = delegate.getMergeInstance();assert mergeVectorsReader != null;mergeInstanceCount.incrementAndGet();+ AtomicInteger parentMergeFinishCount = this.finishMergeCount;- final var parent = this;return new AssertingKnnVectorsReader(mergeVectorsReader, AssertingKnnVectorsReader.this.fis, true) {+ private boolean finished;++ @Override+ public void search(+ String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) {+ assert false : "This instance should only be used for merging";+ }++ @Override+ public void search(+ String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) {+ assert false : "This instance should only be used for merging";+ }+@Overridepublic KnnVectorsReader getMergeInstance() {assert false; // merging from a merge instance it not allowed@@ -202,9 +220,10 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic void finishMerge() throws IOException {- assert mergeInstance;+ assert !finished : "Merging already finished";+ finished = true;delegate.finishMerge();- parent.finishMergeCount.incrementAndGet();+ parentMergeFinishCount.incrementAndGet();}@Override@@ -216,9 +235,7 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic void finishMerge() throws IOException {- assert mergeInstance;- delegate.finishMerge();- finishMergeCount.incrementAndGet();+ assert false; // can only finish merge on the merge instance}@Override@@ -228,10 +245,8 @@ public class AssertingKnnVectorsFormat extends KnnVectorsFormat {@Overridepublic void close() throws IOException {- assert !mergeInstance;- delegate.close();+ assert mergeInstanceCount.get() == finishMergeCount.get();delegate.close();- assert finishMergeCount.get() <= 0 || mergeInstanceCount.get() == finishMergeCount.get();}@Overridediff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.javaindex cbcd953edbf..9b9dc9e6ff4 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java@@ -61,7 +61,6 @@ import org.apache.lucene.store.IndexInput;import org.apache.lucene.store.IndexOutput;import org.apache.lucene.store.MMapDirectory;import org.apache.lucene.store.RandomAccessInput;-import org.apache.lucene.store.ReadAdvice;import org.apache.lucene.tests.mockfile.ExtrasFS;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;@@ -1571,38 +1570,6 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {doTestPrefetch(TestUtil.nextInt(random(), 1, 1024));}- public void testUpdateReadAdvice() throws IOException {- try (Directory dir = getDirectory(createTempDir("testUpdateReadAdvice"))) {- final int totalLength = TestUtil.nextInt(random(), 16384, 65536);- byte[] arr = new byte[totalLength];- random().nextBytes(arr);- try (IndexOutput out = dir.createOutput("temp.bin", IOContext.DEFAULT)) {- out.writeBytes(arr, arr.length);- }-- try (IndexInput orig = dir.openInput("temp.bin", IOContext.DEFAULT)) {- IndexInput in = random().nextBoolean() ? orig.clone() : orig;- // Read advice updated at start- in.updateReadAdvice(randomFrom(random(), ReadAdvice.values()));- for (int i = 0; i < totalLength; i++) {- int offset = TestUtil.nextInt(random(), 0, (int) in.length() - 1);- in.seek(offset);- assertEquals(arr[offset], in.readByte());- }-- // Updating readAdvice in the middle- for (int i = 0; i < 10_000; ++i) {- int offset = TestUtil.nextInt(random(), 0, (int) in.length() - 1);- in.seek(offset);- assertEquals(arr[offset], in.readByte());- if (random().nextBoolean()) {- in.updateReadAdvice(randomFrom(random(), ReadAdvice.values()));- }- }- }- }- }-private void doTestPrefetch(int startOffset) throws IOException {try (Directory dir = getDirectory(createTempDir())) {final int totalLength = startOffset + TestUtil.nextInt(random(), 16384, 65536);diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.javaindex 7410c258954..fcf29fcb365 100644--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockIndexInputWrapper.java@@ -25,7 +25,6 @@ import org.apache.lucene.internal.tests.TestSecrets;import org.apache.lucene.store.FilterIndexInput;import org.apache.lucene.store.IOContext;import org.apache.lucene.store.IndexInput;-import org.apache.lucene.store.ReadAdvice;/*** Used by MockDirectoryWrapper to create an input stream that keeps track of when it's been closed.@@ -186,10 +185,10 @@ public class MockIndexInputWrapper extends FilterIndexInput {}@Override- public void updateReadAdvice(ReadAdvice readAdvice) throws IOException {+ public void updateIOContext(IOContext context) throws IOException {ensureOpen();ensureAccessible();- in.updateReadAdvice(readAdvice);+ in.updateIOContext(context);}@Override