Skip to content

Apache Lucene open-source search software

Prefetch full-precision vectors when rescoring (#16705)

goankur82d73908cb18
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 42567bd644e..3a66cee3964 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -7,6 +7,11 @@ http://s.apache.org/luceneversions
API Changes
---------------------
+* GITHUB#16705: KnnVectorValues#prefetch(int ord, int count) replaces prefetch(int ord). It prefetches
+ a run of consecutive vectors with a single read and returns true if a prefetch was issued, like
+ IndexInput#prefetch. The int[] form now coalesces ordinals that are consecutive in the array over
+ it. (Ankur Goel, Jim Ferenczi)
+
* GITHUB#15113: Move long[] group varint support from core to backward-codecs. The legacy long[]
GroupVIntUtil and DataOutput APIs were removed from core and are now only available for backward
codecs. (Subhramit Basu)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
index 4e205208816..c50b22cdefb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java
@@ -724,6 +724,12 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
return rawVectorValues.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ // vectorValue()/rescorer() read the raw full-precision vectors, so prefetch those.
+ return rawVectorValues.prefetch(ord, count);
+ }
+
@Override
public ScalarQuantizedVectorValues copy() throws IOException {
return new ScalarQuantizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy());
@@ -785,6 +791,11 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReader
return rawVectorValues.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return rawVectorValues.prefetch(ord, count);
+ }
+
@Override
public ScalarQuantizedFloat16VectorValues copy() throws IOException {
return new ScalarQuantizedFloat16VectorValues(
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java
index b2be57c5b31..b1de04cea9d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java
@@ -52,6 +52,11 @@ final class NormalizedFloatVectorValues extends FloatVectorValues {
return normalizedVector;
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return values.prefetch(ord, count);
+ }
+
@Override
public DocIndexIterator iterator() {
return values.iterator();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
index 6bc9963a579..46e4505d931 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java
@@ -82,20 +82,13 @@ public abstract class OffHeapByteVectorValues extends ByteVectorValues implement
}
@Override
- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {
- if (ordsToPrefetch == null) {
- return;
- }
- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);
- if (finalNumOrds <= 1) {
- return;
- }
-
- // 1. calculate offset and prefetch immediately
- for (int i = 0; i < finalNumOrds; i++) {
- long offset = (long) ordsToPrefetch[i] * byteSize;
- slice.prefetch(offset, byteSize);
+ public boolean prefetch(int ord, int count) throws IOException {
+ if (ord < 0 || ord >= size || count <= 0) {
+ return false;
}
+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.
+ final int runLength = Math.min(count, size - ord);
+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java
index 3c8e1f67d6c..497d03c3712 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java
@@ -87,26 +87,18 @@ public abstract class OffHeapFloat16VectorValues extends Float16VectorValues
}
@Override
- public VectorEncoding getEncoding() {
- return VectorEncoding.FLOAT16;
+ public boolean prefetch(int ord, int count) throws IOException {
+ if (ord < 0 || ord >= size || count <= 0) {
+ return false;
+ }
+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.
+ final int runLength = Math.min(count, size - ord);
+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);
}
@Override
- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {
- if (ordsToPrefetch == null) {
- return;
- }
-
- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);
- if (finalNumOrds <= 1) {
- return;
- }
-
- // calculate offset and prefetch immediately
- for (int i = 0; i < finalNumOrds; i++) {
- long offset = (long) ordsToPrefetch[i] * byteSize;
- slice.prefetch(offset, byteSize);
- }
+ public VectorEncoding getEncoding() {
+ return VectorEncoding.FLOAT16;
}
public static OffHeapFloat16VectorValues load(
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
index 7219f536121..e286441d494 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java
@@ -86,21 +86,13 @@ public abstract class OffHeapFloatVectorValues extends FloatVectorValues impleme
}
@Override
- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {
- if (ordsToPrefetch == null) {
- return;
- }
-
- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);
- if (finalNumOrds <= 1) {
- return;
- }
-
- // 1. calculate offset and prefetch immediately
- for (int i = 0; i < finalNumOrds; i++) {
- long offset = (long) ordsToPrefetch[i] * byteSize;
- slice.prefetch(offset, byteSize);
+ public boolean prefetch(int ord, int count) throws IOException {
+ if (ord < 0 || ord >= size || count <= 0) {
+ return false;
}
+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.
+ final int runLength = Math.min(count, size - ord);
+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);
}
public static OffHeapFloatVectorValues load(
diff --git a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
index d437df74a62..5441d267153 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java
@@ -502,6 +502,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
return vectorValues.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return vectorValues.prefetch(ord, count);
+ }
+
@Override
public int ordToDoc(int ord) {
return vectorValues.ordToDoc(ord);
@@ -523,8 +528,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
@Override
- public FloatVectorValues copy() {
- throw new UnsupportedOperationException();
+ public FloatVectorValues copy() throws IOException {
+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still
+ // has to get one that checks the timeout.
+ return new ExitableFloatVectorValues(vectorValues.copy());
}
}
@@ -545,6 +552,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
return vectorValues.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return vectorValues.prefetch(ord, count);
+ }
+
@Override
public int ordToDoc(int ord) {
return vectorValues.ordToDoc(ord);
@@ -566,8 +578,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
@Override
- public Float16VectorValues copy() {
- throw new UnsupportedOperationException();
+ public Float16VectorValues copy() throws IOException {
+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still
+ // has to get one that checks the timeout.
+ return new ExitableFloat16VectorValues(vectorValues.copy());
}
}
@@ -593,6 +607,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
return vectorValues.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return vectorValues.prefetch(ord, count);
+ }
+
@Override
public int ordToDoc(int ord) {
return vectorValues.ordToDoc(ord);
@@ -609,8 +628,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {
}
@Override
- public ByteVectorValues copy() {
- throw new UnsupportedOperationException();
+ public ByteVectorValues copy() throws IOException {
+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still
+ // has to get one that checks the timeout.
+ return new ExitableByteVectorValues(vectorValues.copy());
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java
index a5718192435..b568ff8f389 100644
--- a/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java
@@ -50,12 +50,47 @@ public abstract class KnnVectorValues {
}
/**
- * Prefetches the provided ordinals
+ * Prefetches {@code count} consecutive vectors starting at the given ordinal, so that later calls
+ * to read them are more likely to hit memory. Implementations should start the reads and return
+ * without waiting for them, and are free to prefetch fewer vectors than asked for, including none
+ * at all. {@code count} is clamped to the number of vectors remaining after {@code ord}. The
+ * default implementation is a no-op.
+ *
+ * @param ord the ordinal of the first vector to prefetch
+ * @param count how many consecutive vectors to prefetch, starting at {@code ord}
+ * @return true if prefetch actually prefetched something, hence the caller can benefit from
+ * deferring the reads
+ */
+ public boolean prefetch(int ord, int count) throws IOException {
+ return false;
+ }
+
+ /**
+ * Prefetches the provided ordinals. Ordinals that are consecutive in the array are prefetched
+ * together with a single call to {@link #prefetch(int, int)}.
*
* @param ordsToPrefetch a list of ordinals to prefetch
* @param numOrds number of ords to prefetch from ordsToPrefetch array
*/
- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {}
+ public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {
+ if (ordsToPrefetch == null) {
+ return;
+ }
+ final int count = Math.min(numOrds, ordsToPrefetch.length);
+ if (count <= 1) {
+ // A single ord is read right after this call, so prefetching it buys no overlap. Callers that
+ // do want a lone value loaded ahead of time should call prefetch(ord, 1) directly.
+ return;
+ }
+ int i = 0;
+ while (i < count) {
+ final int runStart = i++;
+ while (i < count && ordsToPrefetch[i] == ordsToPrefetch[i - 1] + 1) {
+ i++;
+ }
+ prefetch(ordsToPrefetch[runStart], i - runStart);
+ }
+ }
/**
* Creates a new copy of this {@link KnnVectorValues}. This is helpful when you need to access
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
index a4750214b98..394a82f6db9 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
@@ -326,6 +326,12 @@ public final class SortingCodecReader extends FilterCodecReader {
return delegate.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ // ords are interpreted in the delegate's ord-space, as in vectorValue().
+ return delegate.prefetch(ord, count);
+ }
+
@Override
public int dimension() {
return delegate.dimension();
@@ -362,6 +368,12 @@ public final class SortingCodecReader extends FilterCodecReader {
return delegate.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ // ords are interpreted in the delegate's ord-space, as in vectorValue().
+ return delegate.prefetch(ord, count);
+ }
+
@Override
public DocIndexIterator iterator() {
return iteratorSupplier.get();
@@ -399,6 +411,12 @@ public final class SortingCodecReader extends FilterCodecReader {
return delegate.vectorValue(ord);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ // ords are interpreted in the delegate's ord-space, as in vectorValue().
+ return delegate.prefetch(ord, count);
+ }
+
@Override
public DocIndexIterator iterator() {
return iteratorSupplier.get();
diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java
index 77e0231d3ee..8aedcfd42da 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java
@@ -60,6 +60,11 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleV
+ fi.getVectorDimension());
}
+ // A separate view is used for prefetching so that running ahead of the scoring position does
+ // not disturb the iterator that advanceExact()/doubleValue() rely on.
+ final KnnVectorValues prefetchValues = vectorValues.copy();
+ final KnnVectorValues.DocIndexIterator prefetchIterator = prefetchValues.iterator();
+
if (vectorSimilarityFunction == null) {
VectorScorer scorer = fullPrecisionRescorer(vectorValues);
if (scorer == null) {
@@ -77,6 +82,14 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleV
return doc >= iterator.docID()
&& (iterator.docID() == doc || iterator.advance(doc) == doc);
}
+
+ @Override
+ public void prefetch(int doc) throws IOException {
+ if (doc >= prefetchIterator.docID()
+ && (prefetchIterator.docID() == doc || prefetchIterator.advance(doc) == doc)) {
+ prefetchValues.prefetch(prefetchIterator.index(), 1);
+ }
+ }
};
}
final KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();
@@ -90,6 +103,14 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleV
public boolean advanceExact(int doc) throws IOException {
return doc >= iterator.docID() && (iterator.docID() == doc || iterator.advance(doc) == doc);
}
+
+ @Override
+ public void prefetch(int doc) throws IOException {
+ if (doc >= prefetchIterator.docID()
+ && (prefetchIterator.docID() == doc || prefetchIterator.advance(doc) == doc)) {
+ prefetchValues.prefetch(prefetchIterator.index(), 1);
+ }
+ }
};
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
index 33b6f07e654..20a2d812b47 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java
@@ -40,6 +40,18 @@ public abstract class DoubleValues {
return DocIdSetIterator.NO_MORE_DOCS;
}
+ /**
+ * Optionally start loading whatever is needed to compute the value of the given document, without
+ * waiting for it to become available. This lets a caller that knows several upcoming documents
+ * keep more than one load in flight, which matters when the values live on slow storage. Callers
+ * must still call {@link #advanceExact} before {@link #doubleValue()}; documents must be
+ * prefetched in the same order in which they are later advanced to. The default is a no-op.
+ *
+ * @param doc the document whose value will be needed later
+ * @lucene.experimental
+ */
+ public void prefetch(int doc) throws IOException {}
+
/**
* Return the maximum score that documents between the current position and {@code upTo} can
* produce. Default implementation returns {@link Float#POSITIVE_INFINITY}.
diff --git a/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java
index b481ebf2927..6015a1c430a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java
@@ -17,8 +17,10 @@
package org.apache.lucene.search;
import java.io.IOException;
+import java.util.List;
import java.util.Objects;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.VectorSimilarityFunction;
/**
@@ -33,6 +35,13 @@ import org.apache.lucene.index.VectorSimilarityFunction;
*/
public class RescoreTopNQuery extends Query {
+ /**
+ * Number of candidates whose loads are kept in flight while rescoring. Large enough to cover the
+ * queue depth a block device needs to reach its peak read rate, small enough that the buffer does
+ * not grow with the rescoring set.
+ */
+ private static final int PREFETCH_WINDOW = 128;
+
private final int n;
private final Query query;
private final DoubleValuesSource valuesSource;
@@ -62,6 +71,25 @@ public class RescoreTopNQuery extends Query {
Query rewritten = indexSearcher.rewrite(query);
Weight weight = indexSearcher.createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
HitQueue queue = new HitQueue(n, false);
+ // A value source that reads the inner scorer's score must be consumed at the scorer's current
+ // position, so those cannot be prefetched ahead and are scored as we go.
+ int originalCount =
+ valuesSource.needsScores()
+ ? rescoreInline(reader, weight, rewrittenValueSource, queue)
+ : rescoreWithPrefetch(reader, weight, rewrittenValueSource, queue);
+ int i = 0;
+ ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()];
+ for (ScoreDoc topDoc : queue) {
+ scoreDocs[i++] = topDoc;
+ }
+ TopDocs topDocs =
+ new TopDocs(new TotalHits(originalCount, TotalHits.Relation.EQUAL_TO), scoreDocs);
+ return DocAndScoreQuery.createDocAndScoreQuery(reader, topDocs, 0);
+ }
+
+ private int rescoreInline(
+ IndexReader reader, Weight weight, DoubleValuesSource rewrittenValueSource, HitQueue queue)
+ throws IOException {
int originalCount = 0;
for (var leaf : reader.leaves()) {
Scorer innerScorer = weight.scorer(leaf);
@@ -70,25 +98,133 @@ public class RescoreTopNQuery extends Query {
}
DoubleValues rescores = rewrittenValueSource.getValues(leaf, getDoubleValues(innerScorer));
DocIdSetIterator iterator = innerScorer.iterator();
+ while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ rescoreInto(queue, rescores, leaf.docBase, iterator.docID());
+ originalCount++;
+ }
+ }
+ return originalCount;
+ }
+
+ /**
+ * Starts the loads for the candidates ahead of the scoring position, so that more than one read
+ * is in flight when the values live on slow storage.
+ *
+ * <p>Candidates are held in a fixed-size ring rather than buffered in full: a rerank shortlist
+ * can be large, and its size is a function of the query rather than of the index. Each iteration
+ * issues the prefetch for one candidate and, once the ring is full, scores the oldest one, which
+ * by then has had a whole window of candidates' worth of time to load. That keeps {@link
+ * #PREFETCH_WINDOW} reads in flight from end to end instead of draining every time a batch is
+ * scored. The ring spans segments, because a shortlist is spread over all of them and a
+ * segment-at-a-time window would go idle at every boundary.
+ *
+ * <p>Only doc ids are buffered, never values.
+ */
+ private int rescoreWithPrefetch(
+ IndexReader reader, Weight weight, DoubleValuesSource rewrittenValueSource, HitQueue queue)
+ throws IOException {
+ final List<LeafReaderContext> leaves = reader.leaves();
+ final DoubleValues[] leafValues = new DoubleValues[leaves.size()];
+ final PrefetchRing ring = new PrefetchRing(PREFETCH_WINDOW);
+ int originalCount = 0;
+
+ for (int i = 0; i < leaves.size(); i++) {
+ final LeafReaderContext leaf = leaves.get(i);
+ Scorer innerScorer = weight.scorer(leaf);
+ if (innerScorer == null) {
+ continue;
+ }
+ DoubleValues rescores = rewrittenValueSource.getValues(leaf, null);
+ leafValues[i] = rescores;
+ DocIdSetIterator iterator = innerScorer.iterator();
while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int docId = iterator.docID();
- if (rescores.advanceExact(docId)) {
- double v = rescores.doubleValue();
- queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, (float) v));
- } else {
- queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, 0f));
+ // Issue the load first, so that the new read is in flight while we block on the oldest one.
+ rescores.prefetch(docId);
+ if (ring.isFull()) {
+ scoreOldest(ring, leaves, leafValues, queue);
+ originalCount++;
}
- originalCount++;
+ ring.append(i, docId);
}
}
- int i = 0;
- ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()];
- for (ScoreDoc topDoc : queue) {
- scoreDocs[i++] = topDoc;
+
+ while (ring.isEmpty() == false) {
+ scoreOldest(ring, leaves, leafValues, queue);
+ originalCount++;
+ }
+ return originalCount;
+ }
+
+ /**
+ * Scores the candidate that has been in the ring longest, and so has had the most time to load,
+ * then drops it.
+ */
+ private static void scoreOldest(
+ PrefetchRing ring, List<LeafReaderContext> leaves, DoubleValues[] leafValues, HitQueue queue)
+ throws IOException {
+ int leafOrd = ring.oldestLeaf();
+ rescoreInto(queue, leafValues[leafOrd], leaves.get(leafOrd).docBase, ring.oldestDoc());
+ ring.removeOldest();
+ }
+
+ /**
+ * Fixed-size FIFO of the candidates whose loads have been issued but whose scores have not been
+ * read yet. Holds doc ids and the segment they belong to, never values, and never grows with the
+ * size of the rescoring set.
+ */
+ private static final class PrefetchRing {
+ private final int[] docs;
+ private final int[] leafOrds;
+ private int head;
+ private int size;
+
+ PrefetchRing(int capacity) {
+ this.docs = new int[capacity];
+ this.leafOrds = new int[capacity];
+ }
+
+ boolean isFull() {
+ return size == docs.length;
+ }
+
+ boolean isEmpty() {
+ return size == 0;
+ }
+
+ /** Adds a candidate. A window may straddle segments, so the segment travels with the doc id. */
+ void append(int leafOrd, int doc) {
+ assert isFull() == false : "ring is full";
+ int tail = (head + size) % docs.length;
+ docs[tail] = doc;
+ leafOrds[tail] = leafOrd;
+ size++;
+ }
+
+ int oldestDoc() {
+ assert isEmpty() == false : "ring is empty";
+ return docs[head];
+ }
+
+ int oldestLeaf() {
+ assert isEmpty() == false : "ring is empty";
+ return leafOrds[head];
+ }
+
+ void removeOldest() {
+ assert isEmpty() == false : "ring is empty";
+ head = (head + 1) % docs.length;
+ size--;
+ }
+ }
+
+ private static void rescoreInto(HitQueue queue, DoubleValues values, int docBase, int docId)
+ throws IOException {
+ if (values.advanceExact(docId)) {
+ queue.insertWithOverflow(new ScoreDoc(docBase + docId, (float) values.doubleValue()));
+ } else {
+ queue.insertWithOverflow(new ScoreDoc(docBase + docId, 0f));
}
- TopDocs topDocs =
- new TopDocs(new TotalHits(originalCount, TotalHits.Relation.EQUAL_TO), scoreDocs);
- return DocAndScoreQuery.createDocAndScoreQuery(reader, topDocs, 0);
}
private DoubleValues getDoubleValues(Scorer innerScorer) {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.java
new file mode 100644
index 00000000000..faed0bf63b5
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene104;
+
+import java.io.IOException;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedFloat16VectorValues;
+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues;
+import org.apache.lucene.index.Float16VectorValues;
+import org.apache.lucene.index.FloatVectorValues;
+import org.apache.lucene.index.KnnVectorValues;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+/**
+ * The {@code FloatVectorValues} handed out for a quantized field wraps the raw full-precision
+ * values. Any of these wrappers that forwards {@code vectorValue()} but not {@link
+ * KnnVectorValues#prefetch(int, int)} inherits the no-op default instead, which silently leaves
+ * full-precision rescoring at one outstanding read per candidate. Nothing else fails, so these
+ * assertions are the only thing that catches a dropped override.
+ */
+public class TestQuantizedVectorValuesPrefetch extends LuceneTestCase {
+
+ public void testScalarQuantizedValuesPrefetchRawVectors() throws IOException {
+ RecordingFloatVectorValues raw = new RecordingFloatVectorValues();
+ // The quantized side is unused by prefetch, which reads the raw vectors.
+ ScalarQuantizedVectorValues values = new ScalarQuantizedVectorValues(raw, null);
+
+ assertTrue(values.prefetch(7, 3));
+ assertEquals(1, raw.calls);
+ assertEquals(7, raw.lastOrd);
+ assertEquals(3, raw.lastCount);
+ }
+
+ public void testScalarQuantizedFloat16ValuesPrefetchRawVectors() throws IOException {
+ RecordingFloat16VectorValues raw = new RecordingFloat16VectorValues();
+ ScalarQuantizedFloat16VectorValues values = new ScalarQuantizedFloat16VectorValues(raw, null);
+
+ assertTrue(values.prefetch(2, 5));
+ assertEquals(1, raw.calls);
+ assertEquals(2, raw.lastOrd);
+ assertEquals(5, raw.lastCount);
+ }
+
+ public void testNormalizedValuesPrefetchUnderlyingVectors() throws IOException {
+ RecordingFloatVectorValues delegate = new RecordingFloatVectorValues();
+ NormalizedFloatVectorValues values = new NormalizedFloatVectorValues(delegate);
+
+ assertTrue(values.prefetch(4, 2));
+ assertEquals(1, delegate.calls);
+ assertEquals(4, delegate.lastOrd);
+ assertEquals(2, delegate.lastCount);
+ }
+
+ /** Records what a wrapper forwards, and claims the prefetch was issued. */
+ private static class RecordingFloatVectorValues extends FloatVectorValues {
+ int calls;
+ int lastOrd = -1;
+ int lastCount = -1;
+
+ @Override
+ public boolean prefetch(int ord, int count) {
+ calls++;
+ lastOrd = ord;
+ lastCount = count;
+ return true;
+ }
+
+ @Override
+ public float[] vectorValue(int ord) {
+ return new float[dimension()];
+ }
+
+ @Override
+ public FloatVectorValues copy() {
+ return this;
+ }
+
+ @Override
+ public int dimension() {
+ return 4;
+ }
+
+ @Override
+ public int size() {
+ return 16;
+ }
+ }
+
+ private static class RecordingFloat16VectorValues extends Float16VectorValues {
+ int calls;
+ int lastOrd = -1;
+ int lastCount = -1;
+
+ @Override
+ public boolean prefetch(int ord, int count) {
+ calls++;
+ lastOrd = ord;
+ lastCount = count;
+ return true;
+ }
+
+ @Override
+ public short[] vectorValue(int ord) {
+ return new short[dimension()];
+ }
+
+ @Override
+ public Float16VectorValues copy() {
+ return this;
+ }
+
+ @Override
+ public int dimension() {
+ return 4;
+ }
+
+ @Override
+ public int size() {
+ return 16;
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java
index abb6017ad07..fec29554f22 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java
@@ -26,20 +26,81 @@ import org.apache.lucene.tests.util.LuceneTestCase;
public class TestOffHeapVectorValues extends LuceneTestCase {
public void testPrefetchVectorValuesWithMoreThanOneOrds() throws IOException {
+ // {1, 2} are consecutive, so they are prefetched with a single read.
CountingIndexInput byteIndexInput = new CountingIndexInput();
OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);
values.prefetch(new int[] {1, 2, 5}, 2);
- assertEquals(2, byteIndexInput.getPrefetchCount());
+ assertEquals(1, byteIndexInput.getPrefetchCount());
CountingIndexInput floatIndexInput = new CountingIndexInput();
OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);
floatValues.prefetch(new int[] {1, 2, 5}, 2);
- assertEquals(2, floatIndexInput.getPrefetchCount());
+ assertEquals(1, floatIndexInput.getPrefetchCount());
CountingIndexInput float16IndexInput = new CountingIndexInput();
OffHeapFloat16VectorValues float16Values = createTestFloat16VectorValues(float16IndexInput);
float16Values.prefetch(new int[] {1, 2, 5}, 2);
- assertEquals(2, float16IndexInput.getPrefetchCount());
+ assertEquals(1, float16IndexInput.getPrefetchCount());
+ }
+
+ public void testPrefetchCoalescesConsecutiveOrdsOnly() throws IOException {
+ // One read per run of consecutive ords: {1, 2, 3}, {7}, {9, 10}.
+ CountingIndexInput byteIndexInput = new CountingIndexInput();
+ OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);
+ values.prefetch(new int[] {1, 2, 3, 7, 9, 10}, 6);
+ assertEquals(3, byteIndexInput.getPrefetchCount());
+
+ // Ords are not sorted here, so no run forms and every ord costs a read.
+ CountingIndexInput floatIndexInput = new CountingIndexInput();
+ OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);
+ floatValues.prefetch(new int[] {10, 3, 2, 8}, 4);
+ assertEquals(4, floatIndexInput.getPrefetchCount());
+ }
+
+ public void testPrefetchContiguousRun() throws IOException {
+ // byteSize is 1 here, so a run of n ords is a single n-byte read at offset ord.
+ CountingIndexInput byteIndexInput = new CountingIndexInput();
+ OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);
+ assertTrue(values.prefetch(10, 4));
+ assertEquals(1, byteIndexInput.getPrefetchCount());
+ assertEquals(10, byteIndexInput.lastOffset);
+ assertEquals(4, byteIndexInput.lastLength);
+
+ // A run reaching past the last ord is clamped to the vectors that exist.
+ assertTrue(values.prefetch(98, 50));
+ assertEquals(2, byteIndexInput.getPrefetchCount());
+ assertEquals(2, byteIndexInput.lastLength);
+
+ // Nothing to prefetch: out of range ord, or a non-positive count.
+ assertFalse(values.prefetch(100, 1));
+ assertFalse(values.prefetch(-1, 1));
+ assertFalse(values.prefetch(1, 0));
+ assertEquals(2, byteIndexInput.getPrefetchCount());
+ }
+
+ public void testPrefetchFloat16VectorValues() throws IOException {
+ // fp16 stores vectors contiguously by ordinal just like fp32 and byte, so it prefetches the
+ // same way: one read per run of consecutive ords.
+ CountingIndexInput indexInput = new CountingIndexInput();
+ OffHeapFloat16VectorValues values = createTestFloat16VectorValues(indexInput);
+ assertTrue(values.prefetch(3, 2));
+ assertEquals(1, indexInput.getPrefetchCount());
+ assertEquals(3 * 2, indexInput.lastOffset);
+ assertEquals(2 * 2, indexInput.lastLength);
+
+ values.prefetch(new int[] {1, 2, 3, 7}, 4);
+ assertEquals(3, indexInput.getPrefetchCount());
+
+ assertFalse(values.prefetch(100, 1));
+ assertEquals(3, indexInput.getPrefetchCount());
+ }
+
+ public void testPrefetchReportsNothingPrefetchedWhenInputDeclinesIt() throws IOException {
+ // The default IndexInput#prefetch is a no-op returning false: callers gain nothing by
+ // deferring.
+ OffHeapFloatVectorValues floatValues =
+ createTestFloatVectorValues(new NoopPrefetchIndexInput());
+ assertFalse(floatValues.prefetch(1, 4));
}
public void testPrefetchVectorValuesWithLessThanOneOrds() throws IOException {
@@ -70,18 +131,21 @@ public class TestOffHeapVectorValues extends LuceneTestCase {
// array length so it only prefetches the available ords instead of running past the array end.
CountingIndexInput byteIndexInput = new CountingIndexInput();
OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);
- values.prefetch(new int[] {1, 2}, 5);
+ values.prefetch(new int[] {1, 3}, 5);
assertEquals(2, byteIndexInput.getPrefetchCount());
CountingIndexInput floatIndexInput = new CountingIndexInput();
OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);
- floatValues.prefetch(new int[] {1, 2}, 5);
+ floatValues.prefetch(new int[] {1, 3}, 5);
assertEquals(2, floatIndexInput.getPrefetchCount());
CountingIndexInput float16IndexInput = new CountingIndexInput();
OffHeapFloat16VectorValues float16Values = createTestFloat16VectorValues(float16IndexInput);
+ // Bounding is independent of coalescing: {1, 2} are consecutive, so the two in-bounds ords form
+ // one run and cost a single read, where the ords above use {1, 3} and cost two. Either way the
+ // trailing three slots of the length-2 array are never touched.
float16Values.prefetch(new int[] {1, 2}, 5);
- assertEquals(2, float16IndexInput.getPrefetchCount());
+ assertEquals(1, float16IndexInput.getPrefetchCount());
}
private OffHeapByteVectorValues createTestByteVectorValues(IndexInput indexInput)
@@ -102,9 +166,20 @@ public class TestOffHeapVectorValues extends LuceneTestCase {
100, 100, indexInput, 2, null, VectorSimilarityFunction.EUCLIDEAN);
}
- private static final class CountingIndexInput extends IndexInput {
+ /** An input that declines to prefetch, like the default {@link IndexInput#prefetch}. */
+ private static final class NoopPrefetchIndexInput extends CountingIndexInput {
+
+ @Override
+ public boolean prefetch(long offset, long length) throws IOException {
+ return false;
+ }
+ }
+
+ private static class CountingIndexInput extends IndexInput {
AtomicInteger counter;
+ long lastOffset = -1;
+ long lastLength = -1;
public CountingIndexInput() {
super("closing index input");
@@ -114,6 +189,8 @@ public class TestOffHeapVectorValues extends LuceneTestCase {
@Override
public boolean prefetch(long offset, long length) throws IOException {
counter.incrementAndGet();
+ lastOffset = offset;
+ lastLength = length;
return true;
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.java
new file mode 100644
index 00000000000..955baeeff92
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RescoreTopNQuery;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Filter readers that wrap {@link KnnVectorValues} have to forward {@link
+ * KnnVectorValues#prefetch(int, int)} along with everything else. A wrapper that forwards {@code
+ * vectorValue()} but not {@code prefetch()} inherits the no-op default, so prefetching silently
+ * stops reaching the store and reads go back to one at a time. Nothing else changes, which is why
+ * this needs asserting rather than eyeballing.
+ */
+public class TestFilterReadersForwardVectorPrefetch extends LuceneTestCase {
+
+ private static final String FIELD = "vector";
+ private static final String SORT_FIELD = "sort";
+ private static final int DIMS = 4;
+
+ public void testExitableDirectoryReaderForwardsPrefetch() throws Exception {
+ try (PrefetchCountingDirectory dir = newIndex()) {
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ // A timeout that never fires: the wrapping, not the timeout, is what is under test.
+ DirectoryReader exitable = new ExitableDirectoryReader(reader, () -> false);
+ LeafReader leaf = getOnlyLeafReader(exitable);
+ assertPrefetchReachesStore("ExitableDirectoryReader", dir, leaf);
+ }
+ }
+ }
+
+ public void testSortingCodecReaderForwardsPrefetch() throws Exception {
+ try (PrefetchCountingDirectory dir = newIndex()) {
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ Sort sort = new Sort(new SortField(SORT_FIELD, SortField.Type.LONG, true));
+ CodecReader sorting =
+ SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(getOnlyLeafReader(reader)), sort);
+ assertPrefetchReachesStore("SortingCodecReader", dir, sorting);
+ }
+ }
+ }
+
+ public void testFullPrecisionRescoringWorksThroughExitableReader() throws Exception {
+ // Prefetching runs over a second copy() of the vector values, so a wrapper whose copy() throws
+ // breaks rescoring outright rather than just losing the prefetch. A search carrying a timeout
+ // goes through ExitableDirectoryReader, so this is the common case, not an exotic one.
+ try (PrefetchCountingDirectory dir = newIndex()) {
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ DirectoryReader exitable = new ExitableDirectoryReader(reader, () -> false);
+ IndexSearcher searcher = new IndexSearcher(exitable);
+ float[] target = new float[DIMS];
+ target[0] = 1f;
+ Query knn = new KnnFloatVectorQuery(FIELD, target, 8);
+ TopDocs topDocs =
+ searcher.search(
+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knn, target, FIELD, 4), 4);
+ assertEquals(4, topDocs.scoreDocs.length);
+ }
+ }
+ }
+
+ public void testUnwrappedReaderPrefetchesForComparison() throws Exception {
+ // Guards the test itself: if the plain reader stopped prefetching, the assertions above would
+ // pass or fail for reasons that have nothing to do with the wrappers.
+ try (PrefetchCountingDirectory dir = newIndex()) {
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertPrefetchReachesStore("unwrapped reader", dir, getOnlyLeafReader(reader));
+ }
+ }
+ }
+
+ private static void assertPrefetchReachesStore(
+ String what, PrefetchCountingDirectory dir, LeafReader leaf) throws IOException {
+ FloatVectorValues values = leaf.getFloatVectorValues(FIELD);
+ assertNotNull(what + " has no vector values", values);
+ int before = dir.prefetchCount(".vec");
+ values.prefetch(0, 2);
+ assertTrue(
+ what
+ + " dropped prefetch: no read was started on the raw vector file. Counted prefetches "
+ + "per extension: "
+ + dir.counts,
+ dir.prefetchCount(".vec") > before);
+ }
+
+ private PrefetchCountingDirectory newIndex() throws IOException {
+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());
+ IndexWriterConfig config = new IndexWriterConfig();
+ config.setCodec(
+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));
+ // Attribute prefetches to the vector file itself rather than to an enclosing .cfs, and keep
+ // everything in one segment so getOnlyLeafReader() applies.
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(64);
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int i = 0; i < 16; i++) {
+ Document doc = new Document();
+ float[] vector = new float[DIMS];
+ vector[i % DIMS] = 1f;
+ doc.add(new KnnFloatVectorField(FIELD, vector, VectorSimilarityFunction.DOT_PRODUCT));
+ doc.add(new NumericDocValuesField(SORT_FIELD, i));
+ writer.addDocument(doc);
+ }
+ writer.forceMerge(1);
+ }
+ return dir;
+ }
+
+ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */
+ private static class PrefetchCountingDirectory extends FilterDirectory {
+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();
+
+ PrefetchCountingDirectory(Directory in) {
+ super(in);
+ }
+
+ int prefetchCount(String extension) {
+ AtomicInteger counter = counts.get(extension);
+ return counter == null ? 0 : counter.get();
+ }
+
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ int dot = name.lastIndexOf('.');
+ String extension = dot < 0 ? name : name.substring(dot);
+ AtomicInteger counter = counts.computeIfAbsent(extension, _ -> new AtomicInteger());
+ return new CountingIndexInput(in, counter);
+ }
+ }
+
+ private static class CountingIndexInput extends FilterIndexInput {
+ private final AtomicInteger counter;
+
+ CountingIndexInput(IndexInput in, AtomicInteger counter) {
+ super("CountingIndexInput(" + in + ")", in);
+ this.counter = counter;
+ }
+
+ @Override
+ public boolean prefetch(long offset, long length) throws IOException {
+ counter.incrementAndGet();
+ // The count, not the return value, is what the assertions look at: ByteBuffersDirectory does
+ // not implement prefetch, so the delegate reports false even though the call arrived.
+ return in.prefetch(offset, length);
+ }
+
+ @Override
+ public IndexInput clone() {
+ return new CountingIndexInput(in.clone(), counter);
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.java
new file mode 100644
index 00000000000..35d92564960
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.KnnFloat16VectorField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.apache.lucene.util.TestVectorUtil;
+
+/**
+ * The float16 counterpart of {@link TestRescoreTopNQueryPrefetch}: verifies that full-precision
+ * rescoring of a float16 field also reaches {@link IndexInput#prefetch} on the raw vector file.
+ *
+ * <p>Both float32 and float16 rescoring inherit {@code DoubleValues#prefetch} from {@code
+ * AbstractFullPrecisionVectorSimilarityValuesSource}, so one test passing while the other fails
+ * would mean the shared implementation is reached on only one of the two paths. A missing override
+ * anywhere along it silently degrades rescoring to a single outstanding read per candidate, which
+ * no correctness test can detect.
+ */
+public class TestRescoreTopNQueryFloat16Prefetch extends LuceneTestCase {
+
+ private static final String FIELD = "vector";
+ private static final int DIMS = 8;
+ private static final VectorSimilarityFunction SIMILARITY = VectorSimilarityFunction.DOT_PRODUCT;
+
+ public void testFloat16RescoringPrefetchesRawVectors() throws Exception {
+ Map<Integer, short[]> vectors = new HashMap<>();
+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());
+ try {
+ IndexWriterConfig config = new IndexWriterConfig();
+ config.setCodec(
+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));
+ // Keep files uncompounded so prefetches can be attributed to the raw vector file itself
+ // rather than to the enclosing .cfs.
+ config.setUseCompoundFile(false);
+ int perSegment = 60;
+ int segments = 3;
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int s = 0; s < segments; s++) {
+ for (int i = 0; i < perSegment; i++) {
+ int id = s * perSegment + i;
+ short[] vector = TestVectorUtil.randomVectorFloat16(DIMS);
+ Document doc = new Document();
+ doc.add(new IntField("id", id, Field.Store.YES));
+ doc.add(new KnnFloat16VectorField(FIELD, vector, SIMILARITY));
+ writer.addDocument(doc);
+ vectors.put(id, vector);
+ }
+ writer.flush();
+ }
+ }
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ short[] target = TestVectorUtil.randomVectorFloat16(DIMS);
+ int k = 10;
+ int shortlist = k * 5;
+
+ // Baseline: the first-phase query alone must not touch the raw vectors at all.
+ KnnFloat16VectorQuery knnQuery = new KnnFloat16VectorQuery(FIELD, target, shortlist);
+ searcher.search(knnQuery, shortlist);
+ int afterFirstPhase = dir.prefetchCount(".vec");
+
+ Query rescored =
+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, target, FIELD, k);
+ TopDocs topDocs = searcher.search(rescored, k);
+
+ int prefetched = dir.prefetchCount(".vec") - afterFirstPhase;
+ assertTrue(
+ "float16 rescoring issued no prefetch on the raw vector file; counted prefetches per "
+ + "extension: "
+ + dir.counts
+ + ". A missing prefetch override on the float16 rescore path leaves the store at "
+ + "one outstanding read per candidate.",
+ prefetched > 0);
+
+ // Rescoring reads one raw vector per candidate, so prefetches should scale with the
+ // shortlist rather than being a token few.
+ assertTrue(
+ "expected prefetches to scale with the rescored candidate count, got " + prefetched,
+ prefetched >= topDocs.scoreDocs.length);
+
+ // Deferring the scoring behind the prefetch window must not change the scores.
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ Document doc = searcher.storedFields().document(scoreDoc.doc);
+ int id = doc.getField("id").numericValue().intValue();
+ short[] docVector = vectors.get(id);
+ assertNotNull("vector for id " + id + " not found", docVector);
+ assertEquals(
+ "score does not match full-precision similarity for id " + id,
+ SIMILARITY.compare(target, docVector),
+ scoreDoc.score,
+ 1e-5);
+ }
+ }
+ } finally {
+ dir.close();
+ }
+ }
+
+ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */
+ private static class PrefetchCountingDirectory extends FilterDirectory {
+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();
+
+ PrefetchCountingDirectory(Directory in) {
+ super(in);
+ }
+
+ int prefetchCount(String extension) {
+ AtomicInteger counter = counts.get(extension);
+ return counter == null ? 0 : counter.get();
+ }
+
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ int dot = name.lastIndexOf('.');
+ if (dot < 0) {
+ return in;
+ }
+ AtomicInteger counter = counts.computeIfAbsent(name.substring(dot), _ -> new AtomicInteger());
+ return new CountingIndexInput(in, counter);
+ }
+ }
+
+ private static class CountingIndexInput extends FilterIndexInput {
+ private final AtomicInteger counter;
+
+ CountingIndexInput(IndexInput in, AtomicInteger counter) {
+ super("CountingIndexInput(" + in + ")", in);
+ this.counter = counter;
+ }
+
+ @Override
+ public boolean prefetch(long offset, long length) throws IOException {
+ counter.incrementAndGet();
+ return in.prefetch(offset, length);
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);
+ }
+
+ @Override
+ public IndexInput clone() {
+ return new CountingIndexInput(in.clone(), counter);
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.java
new file mode 100644
index 00000000000..d3f2c36bfea
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.KnnFloatVectorField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.VectorSimilarityFunction;
+import org.apache.lucene.store.ByteBuffersDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.apache.lucene.store.FilterIndexInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+/**
+ * Verifies that full-precision rescoring actually reaches {@link IndexInput#prefetch} on the raw
+ * vector file. The prefetch has to travel from {@link RescoreTopNQuery} through {@link
+ * DoubleValues} and the quantized {@code FloatVectorValues} wrapper down to the store; a missing
+ * override anywhere on that path silently degrades rescoring to one outstanding read at a time,
+ * which is invisible in correctness tests.
+ */
+public class TestRescoreTopNQueryPrefetch extends LuceneTestCase {
+
+ private static final String FIELD = "vector";
+ private static final int DIMS = 8;
+ private static final VectorSimilarityFunction SIMILARITY = VectorSimilarityFunction.DOT_PRODUCT;
+
+ public void testRescoringPrefetchesRawVectors() throws Exception {
+ Random random = random();
+ Map<Integer, float[]> vectors = new HashMap<>();
+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());
+ try {
+ IndexWriterConfig config = new IndexWriterConfig();
+ config.setCodec(
+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));
+ // Keep files uncompounded so prefetches can be attributed to the raw vector file itself
+ // rather than to the enclosing .cfs.
+ config.setUseCompoundFile(false);
+ int perSegment = 60;
+ int segments = 3;
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int s = 0; s < segments; s++) {
+ for (int i = 0; i < perSegment; i++) {
+ int id = s * perSegment + i;
+ float[] vector = randomVector(random);
+ Document doc = new Document();
+ doc.add(new IntField("id", id, Field.Store.YES));
+ doc.add(new KnnFloatVectorField(FIELD, vector, SIMILARITY));
+ writer.addDocument(doc);
+ vectors.put(id, vector);
+ }
+ writer.flush();
+ }
+ }
+
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ float[] target = randomVector(random);
+ int k = 10;
+ int shortlist = k * 5;
+
+ // Baseline: the first-phase query alone must not touch the raw vectors at all.
+ KnnFloatVectorQuery knnQuery = new KnnFloatVectorQuery(FIELD, target, shortlist);
+ searcher.search(knnQuery, shortlist);
+ int afterFirstPhase = dir.prefetchCount(".vec");
+
+ Query rescored =
+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, target, FIELD, k);
+ TopDocs topDocs = searcher.search(rescored, k);
+
+ int prefetched = dir.prefetchCount(".vec") - afterFirstPhase;
+ assertTrue(
+ "rescoring issued no prefetch on the raw vector file; counted prefetches per extension: "
+ + dir.counts
+ + ". A missing prefetch override on the rescore path leaves the store at one "
+ + "outstanding read per candidate.",
+ prefetched > 0);
+
+ // Rescoring reads one raw vector per candidate, so prefetches should scale with the
+ // shortlist rather than being a token few.
+ assertTrue(
+ "expected prefetches to scale with the rescored candidate count, got " + prefetched,
+ prefetched >= topDocs.scoreDocs.length);
+
+ // Deferring the scoring behind the prefetch window must not change the scores.
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ Document doc = searcher.storedFields().document(scoreDoc.doc);
+ int id = doc.getField("id").numericValue().intValue();
+ float[] docVector = vectors.get(id);
+ assertNotNull("vector for id " + id + " not found", docVector);
+ assertEquals(
+ "score does not match full-precision similarity for id " + id,
+ SIMILARITY.compare(target, docVector),
+ scoreDoc.score,
+ 1e-5);
+ }
+ }
+ } finally {
+ dir.close();
+ }
+ }
+
+ private static float[] randomVector(Random random) {
+ float[] v = new float[DIMS];
+ for (int i = 0; i < v.length; i++) {
+ v[i] = random.nextFloat() - 0.5f;
+ }
+ VectorUtilSupport.normalize(v);
+ return v;
+ }
+
+ /** Normalizes so DOT_PRODUCT stays in range. */
+ private static class VectorUtilSupport {
+ static void normalize(float[] v) {
+ double sum = 0;
+ for (float f : v) {
+ sum += (double) f * f;
+ }
+ float norm = (float) Math.sqrt(sum);
+ if (norm == 0) {
+ v[0] = 1f;
+ return;
+ }
+ for (int i = 0; i < v.length; i++) {
+ v[i] /= norm;
+ }
+ }
+ }
+
+ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */
+ private static class PrefetchCountingDirectory extends FilterDirectory {
+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();
+
+ PrefetchCountingDirectory(Directory in) {
+ super(in);
+ }
+
+ int prefetchCount(String extension) {
+ AtomicInteger counter = counts.get(extension);
+ return counter == null ? 0 : counter.get();
+ }
+
+ @Override
+ public IndexInput openInput(String name, IOContext context) throws IOException {
+ IndexInput in = super.openInput(name, context);
+ int dot = name.lastIndexOf('.');
+ if (dot < 0) {
+ return in;
+ }
+ AtomicInteger counter = counts.computeIfAbsent(name.substring(dot), _ -> new AtomicInteger());
+ return new CountingIndexInput(in, counter);
+ }
+ }
+
+ private static class CountingIndexInput extends FilterIndexInput {
+ private final AtomicInteger counter;
+
+ CountingIndexInput(IndexInput in, AtomicInteger counter) {
+ super("CountingIndexInput(" + in + ")", in);
+ this.counter = counter;
+ }
+
+ @Override
+ public boolean prefetch(long offset, long length) throws IOException {
+ counter.incrementAndGet();
+ return in.prefetch(offset, length);
+ }
+
+ @Override
+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {
+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);
+ }
+
+ @Override
+ public IndexInput clone() {
+ return new CountingIndexInput(in.clone(), counter);
+ }
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
index b3cacda9f81..b4acc025f82 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java
@@ -200,6 +200,11 @@ final class DedupScalarQuantizedVectorValues {
groupView.prefetch(scratch, numOrds);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());
+ }
+
@Override
public byte[] vectorValue(int ord) throws IOException {
return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));
@@ -304,8 +309,8 @@ final class DedupScalarQuantizedVectorValues {
}
@Override
- public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {
- rawValues.prefetch(ordsToPrefetch, numOrds);
+ public boolean prefetch(int ord, int count) throws IOException {
+ return rawValues.prefetch(ord, count);
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
index 23656f3fcab..70981af05c5 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;
import org.apache.lucene.index.DocsWithFieldSet;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.KnnVectorValues;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;
@@ -181,6 +182,32 @@ final class DedupUtil {
}
}
+ /**
+ * Prefetches a run of {@code count} field ords starting at {@code ord} from the group view. Field
+ * ords that are consecutive need not map to consecutive group ords, so the run is remapped and
+ * prefetched ord by ord rather than as a single contiguous read.
+ *
+ * @return true if a prefetch was actually issued for any of them
+ */
+ static boolean prefetchRemapped(
+ KnnVectorValues groupView,
+ FieldOrdToGroupOrd fieldOrdToGroupOrd,
+ int ord,
+ int count,
+ int size)
+ throws IOException {
+ if (ord < 0 || ord >= size || count <= 0) {
+ return false;
+ }
+ final int runLength = Math.min(count, size - ord);
+ boolean prefetched = false;
+ for (int i = 0; i < runLength; i++) {
+ // Not short-circuiting: every ord in the run has to be issued.
+ prefetched |= groupView.prefetch(fieldOrdToGroupOrd.get(ord + i), 1);
+ }
+ return prefetched;
+ }
+
static long hashBytes(byte[] bytes) {
return murmurhash3_x64_128(bytes, 0, bytes.length, GOOD_FAST_HASH_SEED)[0];
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
index 2e79cf65377..fffe388ff6d 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java
@@ -162,6 +162,11 @@ sealed interface DedupVectorValues
groupView.prefetch(scratch, numOrds);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());
+ }
+
@Override
public byte[] vectorValue(int ord) throws IOException {
return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));
@@ -287,6 +292,11 @@ sealed interface DedupVectorValues
groupView.prefetch(scratch, numOrds);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());
+ }
+
@Override
public float[] vectorValue(int ord) throws IOException {
return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));
@@ -412,6 +422,11 @@ sealed interface DedupVectorValues
groupView.prefetch(scratch, numOrds);
}
+ @Override
+ public boolean prefetch(int ord, int count) throws IOException {
+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());
+ }
+
@Override
public short[] vectorValue(int ord) throws IOException {
return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));