Skip to content

Apache Lucene open-source search software

Propagate getMergeInstance() through the quantized vector readers (#16622)

John Mauricecbad01e83ef4
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f8647ceb5f5..87d4071990e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -492,6 +492,11 @@ Optimizations
vectors writers now state what file they are writing, and the raw vectors a quantized format keeps
only to rescore say so as they are written. (Jim Ferenczi)
+* GITHUB#16622: Merges of quantized vector fields now also open the raw vectors for themselves.
+ Lucene104ScalarQuantizedVectorsReader, and the backward-codecs Lucene99 and Lucene102 quantized
+ vector readers, pass getMergeInstance() and finishMerge() on to their raw vectors reader.
+ (John Maurice)
+
* GITHUB#16684: Merges now open stored fields for themselves and state that they read the data file
sequentially, instead of reusing the mapping searches read at random. (Jim Ferenczi)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
index 7770bbc34bf..99c17c736b0 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java
@@ -78,7 +78,7 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Lucene102BinaryQuantizedVectorsReader.class);
- private final Map<String, FieldEntry> fields = new HashMap<>();
+ private final Map<String, FieldEntry> fields;
private final IndexInput quantizedVectorData;
private final FlatVectorsReader rawVectorsReader;
private final Lucene102BinaryFlatVectorsScorer vectorScorer;
@@ -96,6 +96,7 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader
FlatVectorsReader rawVectorsReader,
Lucene102BinaryFlatVectorsScorer vectorsScorer)
throws IOException {
+ this.fields = new HashMap<>();
this.vectorScorer = vectorsScorer;
this.rawVectorsReader = rawVectorsReader;
int versionMeta = -1;
@@ -135,6 +136,37 @@ public class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader
}
}
+ /**
+ * Copy constructor for {@link #getMergeInstance()}: the copy shares {@code reader}'s open state
+ * and reads raw vectors through {@code rawVectorsReader}, normally the original raw reader's
+ * merge instance. It is used only by the merging thread and is never closed: {@link
+ * #finishMerge()} releases the raw merge instance.
+ */
+ protected Lucene102BinaryQuantizedVectorsReader(
+ Lucene102BinaryQuantizedVectorsReader reader, FlatVectorsReader rawVectorsReader) {
+ this.fields = reader.fields;
+ this.quantizedVectorData = reader.quantizedVectorData;
+ this.rawVectorsReader = rawVectorsReader;
+ this.vectorScorer = reader.vectorScorer;
+ }
+
+ /**
+ * Returns a copy of this reader that reads raw vectors through the raw reader's merge instance. A
+ * subclass must override this method and build its own copy through the {@link
+ * #Lucene102BinaryQuantizedVectorsReader(Lucene102BinaryQuantizedVectorsReader,
+ * FlatVectorsReader) copy constructor}, or its merge instance will be a plain {@code
+ * Lucene102BinaryQuantizedVectorsReader}.
+ */
+ @Override
+ public FlatVectorsReader getMergeInstance() throws IOException {
+ return new Lucene102BinaryQuantizedVectorsReader(this, rawVectorsReader.getMergeInstance());
+ }
+
+ @Override
+ public void finishMerge() throws IOException {
+ rawVectorsReader.finishMerge();
+ }
+
private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
index e13406ced82..aacce895b4d 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScalarQuantizedVectorsReader.java
@@ -67,7 +67,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Lucene99ScalarQuantizedVectorsReader.class);
- private final IntObjectHashMap<FieldEntry> fields = new IntObjectHashMap<>();
+ private final IntObjectHashMap<FieldEntry> fields;
private final FlatVectorsScorer vectorScorer;
private final IndexInput quantizedVectorData;
private final FlatVectorsReader rawVectorsReader;
@@ -77,6 +77,7 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
public Lucene99ScalarQuantizedVectorsReader(
SegmentReadState state, FlatVectorsReader rawVectorsReader, FlatVectorsScorer scorer)
throws IOException {
+ this.fields = new IntObjectHashMap<>();
this.vectorScorer = scorer;
this.rawVectorsReader = rawVectorsReader;
this.fieldInfos = state.fieldInfos;
@@ -117,6 +118,31 @@ public final class Lucene99ScalarQuantizedVectorsReader extends FlatVectorsReade
}
}
+ /**
+ * Copy constructor for {@link #getMergeInstance()}: the copy shares {@code reader}'s open state
+ * and reads raw vectors through {@code rawVectorsReader}, normally the original raw reader's
+ * merge instance. It is used only by the merging thread and is never closed: {@link
+ * #finishMerge()} releases the raw merge instance.
+ */
+ private Lucene99ScalarQuantizedVectorsReader(
+ Lucene99ScalarQuantizedVectorsReader reader, FlatVectorsReader rawVectorsReader) {
+ this.fields = reader.fields;
+ this.vectorScorer = reader.vectorScorer;
+ this.quantizedVectorData = reader.quantizedVectorData;
+ this.rawVectorsReader = rawVectorsReader;
+ this.fieldInfos = reader.fieldInfos;
+ }
+
+ @Override
+ public FlatVectorsReader getMergeInstance() throws IOException {
+ return new Lucene99ScalarQuantizedVectorsReader(this, rawVectorsReader.getMergeInstance());
+ }
+
+ @Override
+ public void finishMerge() throws IOException {
+ rawVectorsReader.finishMerge();
+ }
+
private void readFields(ChecksumIndexInput meta, int versionMeta, FieldInfos infos)
throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
index 29eb4819dc4..81e2b0e0add 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene102/TestLucene102HnswBinaryQuantizedVectorsFormat.java
@@ -25,7 +25,11 @@ import static org.hamcrest.Matchers.oneOf;
import java.io.IOException;
import java.util.Arrays;
+import java.util.List;
import java.util.Locale;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
@@ -38,6 +42,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.VectorEncoding;
@@ -45,6 +50,10 @@ import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.search.AcceptDocs;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.SameThreadExecutorService;
@@ -188,4 +197,70 @@ public class TestLucene102HnswBinaryQuantizedVectorsFormat extends BaseKnnVector
protected boolean supportsFloatVectorFallback() {
return false;
}
+
+ /**
+ * A merge instance reaches the raw vectors reader, which opens the raw vectors again for the
+ * merge, and finishing the merge closes them.
+ */
+ public void testMergeInstanceOpensTheRawVectors() throws IOException {
+ List<AtomicBoolean> mergeOpens = new CopyOnWriteArrayList<>();
+ AtomicInteger mergeSlices = new AtomicInteger();
+ try (Directory dir =
+ new FilterDirectory(newDirectory()) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ if (name.endsWith(".vec") == false || context.context() != IOContext.Context.MERGE) {
+ return in;
+ }
+ AtomicBoolean closed = new AtomicBoolean();
+ mergeOpens.add(closed);
+ return new FilterIndexInput(name, in) {
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length)
+ throws IOException {
+ mergeSlices.incrementAndGet(); // the vectors are read through slices
+ return super.slice(sliceDescription, offset, length);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed.set(true);
+ super.close();
+ }
+ };
+ }
+ }) {
+ int dims = random().nextInt(4, 65);
+ try (IndexWriter w =
+ new IndexWriter(
+ dir, new IndexWriterConfig().setCodec(getCodec()).setUseCompoundFile(false))) {
+ for (int i = 0; i < 16; i++) {
+ Document doc = new Document();
+ doc.add(
+ new KnnFloatVectorField("f", randomVector(dims), VectorSimilarityFunction.EUCLIDEAN));
+ w.addDocument(doc);
+ }
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ KnnVectorsReader vectors =
+ ((CodecReader) getOnlyLeafReader(reader)).getVectorReader().unwrapReaderForField("f");
+ assertEquals(List.of(), mergeOpens);
+ KnnVectorsReader merging = vectors.getMergeInstance();
+ assertEquals("the raw vectors were not opened for the merge", 1, mergeOpens.size());
+
+ FloatVectorValues actual = merging.getFloatVectorValues("f");
+ assertTrue("the merge instance did not read what was opened for it", mergeSlices.get() > 0);
+ FloatVectorValues expected = vectors.getFloatVectorValues("f");
+ assertEquals(expected.size(), actual.size());
+ for (int ord = 0; ord < expected.size(); ord++) {
+ assertArrayEquals(expected.vectorValue(ord), actual.vectorValue(ord), 0f);
+ }
+
+ merging.finishMerge();
+ assertTrue("finishing the merge did not close them", mergeOpens.get(0).get());
+ }
+ }
+ }
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
index 54e88675925..27142a71873 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
@@ -20,17 +20,28 @@ package org.apache.lucene.backward_codecs.lucene99;
import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;
import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.CodecReader;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FloatVectorValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.VectorEncoding;
+import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
@@ -75,4 +86,70 @@ public class TestLucene99HnswScalarQuantizedVectorsFormat extends BaseKnnVectors
protected VectorEncoding randomVectorEncoding() {
return random().nextBoolean() ? VectorEncoding.BYTE : VectorEncoding.FLOAT32;
}
+
+ /**
+ * A merge instance reaches the raw vectors reader, which opens the raw vectors again for the
+ * merge, and finishing the merge closes them.
+ */
+ public void testMergeInstanceOpensTheRawVectors() throws IOException {
+ List<AtomicBoolean> mergeOpens = new CopyOnWriteArrayList<>();
+ AtomicInteger mergeSlices = new AtomicInteger();
+ try (Directory dir =
+ new FilterDirectory(newDirectory()) {
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ if (name.endsWith(".vec") == false || context.context() != IOContext.Context.MERGE) {
+ return in;
+ }
+ AtomicBoolean closed = new AtomicBoolean();
+ mergeOpens.add(closed);
+ return new FilterIndexInput(name, in) {
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length)
+ throws IOException {
+ mergeSlices.incrementAndGet(); // the vectors are read through slices
+ return super.slice(sliceDescription, offset, length);
+ }
+
+ @Override
+ public void close() throws IOException {
+ closed.set(true);
+ super.close();
+ }
+ };
+ }
+ }) {
+ int dims = random().nextInt(4, 65);
+ try (IndexWriter w =
+ new IndexWriter(
+ dir, new IndexWriterConfig().setCodec(getCodec()).setUseCompoundFile(false))) {
+ for (int i = 0; i < 16; i++) {
+ Document doc = new Document();
+ doc.add(
+ new KnnFloatVectorField("f", randomVector(dims), VectorSimilarityFunction.EUCLIDEAN));
+ w.addDocument(doc);
+ }
+ w.commit();
+ }
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ KnnVectorsReader vectors =
+ ((CodecReader) getOnlyLeafReader(reader)).getVectorReader().unwrapReaderForField("f");
+ assertEquals(List.of(), mergeOpens);
+ KnnVectorsReader merging = vectors.getMergeInstance();
+ assertEquals("the raw vectors were not opened for the merge", 1, mergeOpens.size());
+
+ FloatVectorValues actual = merging.getFloatVectorValues("f");
+ assertTrue("the merge instance did not read what was opened for it", mergeSlices.get() > 0);
+ FloatVectorValues expected = vectors.getFloatVectorValues("f");
+ assertEquals(expected.size(), actual.size());
+ for (int ord = 0; ord < expected.size(); ord++) {
+ assertArrayEquals(expected.vectorValue(ord), actual.vectorValue(ord), 0f);
+ }
+
+ merging.finishMerge();
+ assertTrue("finishing the merge did not close them", mergeOpens.get(0).get());
+ }
+ }
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index c50b22cdefb..b5406cfc0a6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -78,7 +78,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
private static final long SHALLOW_SIZE =
RamUsageEstimator.shallowSizeOfInstance(Lucene104ScalarQuantizedVectorsReader.class);
- private final Map<String, FieldEntry> fields = new HashMap<>();
+ private final Map<String, FieldEntry> fields;
private final IndexInput quantizedVectorData;
private final FlatVectorsReader rawVectorsReader;
private final Lucene104ScalarQuantizedVectorScorer vectorScorer;
@@ -89,6 +89,7 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
FlatVectorsReader rawVectorsReader,
Lucene104ScalarQuantizedVectorScorer vectorsScorer)
throws IOException {
+ this.fields = new HashMap<>();
this.vectorScorer = vectorsScorer;
this.rawVectorsReader = rawVectorsReader;
int versionMeta = -1;
@@ -129,6 +130,37 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
}
}
+ /**
+ * Copy constructor for {@link #getMergeInstance()}: the copy shares {@code reader}'s open state
+ * and reads raw vectors through {@code rawVectorsReader}, normally the original raw reader's
+ * merge instance. It is used only by the merging thread and is never closed: {@link
+ * #finishMerge()} releases the raw merge instance.
+ */
+ protected Lucene104ScalarQuantizedVectorsReader(
+ Lucene104ScalarQuantizedVectorsReader reader, FlatVectorsReader rawVectorsReader) {
+ this.fields = reader.fields;
+ this.quantizedVectorData = reader.quantizedVectorData;
+ this.rawVectorsReader = rawVectorsReader;
+ this.vectorScorer = reader.vectorScorer;
+ }
+
+ /**
+ * Returns a copy of this reader that reads raw vectors through the raw reader's merge instance. A
+ * subclass must override this method and build its own copy through the {@link
+ * #Lucene104ScalarQuantizedVectorsReader(Lucene104ScalarQuantizedVectorsReader,
+ * FlatVectorsReader) copy constructor}, or its merge instance will be a plain {@code
+ * Lucene104ScalarQuantizedVectorsReader}.
+ */
+ @Override
+ public FlatVectorsReader getMergeInstance() throws IOException {
+ return new Lucene104ScalarQuantizedVectorsReader(this, rawVectorsReader.getMergeInstance());
+ }
+
+ @Override
+ public void finishMerge() throws IOException {
+ rawVectorsReader.finishMerge();
+ }
+
private void readFields(ChecksumIndexInput meta, FieldInfos infos) throws IOException {
for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) {
FieldInfo info = infos.fieldInfo(fieldNumber);
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
index df7d7b20c5f..fa0317de6d0 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestVectorsMergeReadAdvice.java
@@ -21,7 +21,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
import org.apache.lucene.index.CodecReader;
@@ -49,10 +51,18 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
private static final int DIM = 8;
public void testMergeOpensItsOwnVectors() throws Exception {
+ assertMergeOpensItsOwnVectors(new Lucene99HnswVectorsFormat());
+ }
+
+ public void testQuantizedMergeOpensItsOwnVectors() throws Exception {
+ assertMergeOpensItsOwnVectors(new Lucene104HnswScalarQuantizedVectorsFormat());
+ }
+
+ private static void assertMergeOpensItsOwnVectors(KnnVectorsFormat format) throws Exception {
Opens opens = new Opens();
try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
IndexWriterConfig iwc = new IndexWriterConfig();
- iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswVectorsFormat()));
+ iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(format));
iwc.setUseCompoundFile(false); // so the directory sees the data file by name
try (IndexWriter w = new IndexWriter(dir, iwc)) {
for (int segment = 0; segment < 2; segment++) {
@@ -76,6 +86,7 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
assertFalse(
"the merge never opened the vectors for itself: " + opens, sequential.isEmpty());
for (Open open : sequential) {
+ assertTrue("the merge did not read the vectors it opened: " + open, open.read());
assertTrue("the merge kept its vectors open: " + open, open.closed());
}
}
@@ -89,10 +100,20 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
}
public void testTheMappingIsSharedThenReleasedByTheLastMergeInstance() throws Exception {
+ assertMappingIsSharedThenReleasedByTheLastMergeInstance(new Lucene99HnswVectorsFormat());
+ }
+
+ public void testQuantizedMappingIsSharedThenReleasedByTheLastMergeInstance() throws Exception {
+ assertMappingIsSharedThenReleasedByTheLastMergeInstance(
+ new Lucene104HnswScalarQuantizedVectorsFormat());
+ }
+
+ private static void assertMappingIsSharedThenReleasedByTheLastMergeInstance(
+ KnnVectorsFormat format) throws Exception {
Opens opens = new Opens();
try (Directory dir = new RecordingDirectory(newDirectory(), opens)) {
IndexWriterConfig iwc = new IndexWriterConfig();
- iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswVectorsFormat()));
+ iwc.setCodec(TestUtil.alwaysKnnVectorsFormat(format));
iwc.setUseCompoundFile(false);
try (IndexWriter w = new IndexWriter(dir, iwc)) {
for (int i = 0; i < 16; i++) {
@@ -246,6 +267,7 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
private final String name;
private final IOContext context;
private volatile boolean closed;
+ private volatile boolean read;
Open(String name, IOContext context) {
this.name = name;
@@ -272,9 +294,14 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
return closed;
}
+ /** Whether the input was sliced, which is how the vectors are read. */
+ boolean read() {
+ return read;
+ }
+
@Override
public String toString() {
- return name + " [hint=" + hint() + " closed=" + closed + "]";
+ return name + " [hint=" + hint() + " closed=" + closed + " read=" + read + "]";
}
}
@@ -358,6 +385,7 @@ public class TestVectorsMergeReadAdvice extends LuceneTestCase {
@Override
public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ open.read = true;
return new RecordingIndexInput(in.slice(sliceDescription, offset, length), name, open, opens);
}
}