Apache Lucene open-source search software
Prefetch full-precision vectors when rescoring (#16705)
goankur82d73908cb18
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 42567bd644e..3a66cee3964 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -7,6 +7,11 @@ http://s.apache.org/luceneversionsAPI Changes---------------------+* GITHUB#16705: KnnVectorValues#prefetch(int ord, int count) replaces prefetch(int ord). It prefetches+ a run of consecutive vectors with a single read and returns true if a prefetch was issued, like+ IndexInput#prefetch. The int[] form now coalesces ordinals that are consecutive in the array over+ it. (Ankur Goel, Jim Ferenczi)+* GITHUB#15113: Move long[] group varint support from core to backward-codecs. The legacy long[]GroupVIntUtil and DataOutput APIs were removed from core and are now only available for backwardcodecs. (Subhramit Basu)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.javaindex 4e205208816..c50b22cdefb 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/Lucene104ScalarQuantizedVectorsReader.java@@ -724,6 +724,12 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderreturn rawVectorValues.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ // vectorValue()/rescorer() read the raw full-precision vectors, so prefetch those.+ return rawVectorValues.prefetch(ord, count);+ }+@Overridepublic ScalarQuantizedVectorValues copy() throws IOException {return new ScalarQuantizedVectorValues(rawVectorValues.copy(), quantizedVectorValues.copy());@@ -785,6 +791,11 @@ public class Lucene104ScalarQuantizedVectorsReader extends FlatVectorsReaderreturn rawVectorValues.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return rawVectorValues.prefetch(ord, count);+ }+@Overridepublic ScalarQuantizedFloat16VectorValues copy() throws IOException {return new ScalarQuantizedFloat16VectorValues(diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.javaindex b2be57c5b31..b1de04cea9d 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene104/NormalizedFloatVectorValues.java@@ -52,6 +52,11 @@ final class NormalizedFloatVectorValues extends FloatVectorValues {return normalizedVector;}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return values.prefetch(ord, count);+ }+@Overridepublic DocIndexIterator iterator() {return values.iterator();diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.javaindex 6bc9963a579..46e4505d931 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapByteVectorValues.java@@ -82,20 +82,13 @@ public abstract class OffHeapByteVectorValues extends ByteVectorValues implement}@Override- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {- if (ordsToPrefetch == null) {- return;- }- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);- if (finalNumOrds <= 1) {- return;- }-- // 1. calculate offset and prefetch immediately- for (int i = 0; i < finalNumOrds; i++) {- long offset = (long) ordsToPrefetch[i] * byteSize;- slice.prefetch(offset, byteSize);+ public boolean prefetch(int ord, int count) throws IOException {+ if (ord < 0 || ord >= size || count <= 0) {+ return false;}+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.+ final int runLength = Math.min(count, size - ord);+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);}@Overridediff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.javaindex 3c8e1f67d6c..497d03c3712 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloat16VectorValues.java@@ -87,26 +87,18 @@ public abstract class OffHeapFloat16VectorValues extends Float16VectorValues}@Override- public VectorEncoding getEncoding() {- return VectorEncoding.FLOAT16;+ public boolean prefetch(int ord, int count) throws IOException {+ if (ord < 0 || ord >= size || count <= 0) {+ return false;+ }+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.+ final int runLength = Math.min(count, size - ord);+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);}@Override- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {- if (ordsToPrefetch == null) {- return;- }-- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);- if (finalNumOrds <= 1) {- return;- }-- // calculate offset and prefetch immediately- for (int i = 0; i < finalNumOrds; i++) {- long offset = (long) ordsToPrefetch[i] * byteSize;- slice.prefetch(offset, byteSize);- }+ public VectorEncoding getEncoding() {+ return VectorEncoding.FLOAT16;}public static OffHeapFloat16VectorValues load(diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.javaindex 7219f536121..e286441d494 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapFloatVectorValues.java@@ -86,21 +86,13 @@ public abstract class OffHeapFloatVectorValues extends FloatVectorValues impleme}@Override- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {- if (ordsToPrefetch == null) {- return;- }-- int finalNumOrds = Math.min(numOrds, ordsToPrefetch.length);- if (finalNumOrds <= 1) {- return;- }-- // 1. calculate offset and prefetch immediately- for (int i = 0; i < finalNumOrds; i++) {- long offset = (long) ordsToPrefetch[i] * byteSize;- slice.prefetch(offset, byteSize);+ public boolean prefetch(int ord, int count) throws IOException {+ if (ord < 0 || ord >= size || count <= 0) {+ return false;}+ // Vectors are laid out contiguously by ordinal, so a run of them is a single read.+ final int runLength = Math.min(count, size - ord);+ return slice.prefetch((long) ord * byteSize, (long) runLength * byteSize);}public static OffHeapFloatVectorValues load(diff --git a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.javaindex d437df74a62..5441d267153 100644--- a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java@@ -502,6 +502,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {return vectorValues.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return vectorValues.prefetch(ord, count);+ }+@Overridepublic int ordToDoc(int ord) {return vectorValues.ordToDoc(ord);@@ -523,8 +528,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {}@Override- public FloatVectorValues copy() {- throw new UnsupportedOperationException();+ public FloatVectorValues copy() throws IOException {+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still+ // has to get one that checks the timeout.+ return new ExitableFloatVectorValues(vectorValues.copy());}}@@ -545,6 +552,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {return vectorValues.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return vectorValues.prefetch(ord, count);+ }+@Overridepublic int ordToDoc(int ord) {return vectorValues.ordToDoc(ord);@@ -566,8 +578,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {}@Override- public Float16VectorValues copy() {- throw new UnsupportedOperationException();+ public Float16VectorValues copy() throws IOException {+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still+ // has to get one that checks the timeout.+ return new ExitableFloat16VectorValues(vectorValues.copy());}}@@ -593,6 +607,11 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {return vectorValues.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return vectorValues.prefetch(ord, count);+ }+@Overridepublic int ordToDoc(int ord) {return vectorValues.ordToDoc(ord);@@ -609,8 +628,10 @@ public class ExitableDirectoryReader extends FilterDirectoryReader {}@Override- public ByteVectorValues copy() {- throw new UnsupportedOperationException();+ public ByteVectorValues copy() throws IOException {+ // A caller that needs a second view, e.g. to prefetch ahead of the scoring position, still+ // has to get one that checks the timeout.+ return new ExitableByteVectorValues(vectorValues.copy());}}}diff --git a/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java b/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.javaindex a5718192435..b568ff8f389 100644--- a/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java+++ b/lucene/core/src/java/org/apache/lucene/index/KnnVectorValues.java@@ -50,12 +50,47 @@ public abstract class KnnVectorValues {}/**- * Prefetches the provided ordinals+ * Prefetches {@code count} consecutive vectors starting at the given ordinal, so that later calls+ * to read them are more likely to hit memory. Implementations should start the reads and return+ * without waiting for them, and are free to prefetch fewer vectors than asked for, including none+ * at all. {@code count} is clamped to the number of vectors remaining after {@code ord}. The+ * default implementation is a no-op.+ *+ * @param ord the ordinal of the first vector to prefetch+ * @param count how many consecutive vectors to prefetch, starting at {@code ord}+ * @return true if prefetch actually prefetched something, hence the caller can benefit from+ * deferring the reads+ */+ public boolean prefetch(int ord, int count) throws IOException {+ return false;+ }++ /**+ * Prefetches the provided ordinals. Ordinals that are consecutive in the array are prefetched+ * together with a single call to {@link #prefetch(int, int)}.** @param ordsToPrefetch a list of ordinals to prefetch* @param numOrds number of ords to prefetch from ordsToPrefetch array*/- public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {}+ public void prefetch(final int[] ordsToPrefetch, int numOrds) throws IOException {+ if (ordsToPrefetch == null) {+ return;+ }+ final int count = Math.min(numOrds, ordsToPrefetch.length);+ if (count <= 1) {+ // A single ord is read right after this call, so prefetching it buys no overlap. Callers that+ // do want a lone value loaded ahead of time should call prefetch(ord, 1) directly.+ return;+ }+ int i = 0;+ while (i < count) {+ final int runStart = i++;+ while (i < count && ordsToPrefetch[i] == ordsToPrefetch[i - 1] + 1) {+ i++;+ }+ prefetch(ordsToPrefetch[runStart], i - runStart);+ }+ }/*** Creates a new copy of this {@link KnnVectorValues}. This is helpful when you need to accessdiff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.javaindex a4750214b98..394a82f6db9 100644--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java@@ -326,6 +326,12 @@ public final class SortingCodecReader extends FilterCodecReader {return delegate.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ // ords are interpreted in the delegate's ord-space, as in vectorValue().+ return delegate.prefetch(ord, count);+ }+@Overridepublic int dimension() {return delegate.dimension();@@ -362,6 +368,12 @@ public final class SortingCodecReader extends FilterCodecReader {return delegate.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ // ords are interpreted in the delegate's ord-space, as in vectorValue().+ return delegate.prefetch(ord, count);+ }+@Overridepublic DocIndexIterator iterator() {return iteratorSupplier.get();@@ -399,6 +411,12 @@ public final class SortingCodecReader extends FilterCodecReader {return delegate.vectorValue(ord);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ // ords are interpreted in the delegate's ord-space, as in vectorValue().+ return delegate.prefetch(ord, count);+ }+@Overridepublic DocIndexIterator iterator() {return iteratorSupplier.get();diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java b/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.javaindex 77e0231d3ee..8aedcfd42da 100644--- a/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractFullPrecisionVectorSimilarityValuesSource.java@@ -60,6 +60,11 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleV+ fi.getVectorDimension());}+ // A separate view is used for prefetching so that running ahead of the scoring position does+ // not disturb the iterator that advanceExact()/doubleValue() rely on.+ final KnnVectorValues prefetchValues = vectorValues.copy();+ final KnnVectorValues.DocIndexIterator prefetchIterator = prefetchValues.iterator();+if (vectorSimilarityFunction == null) {VectorScorer scorer = fullPrecisionRescorer(vectorValues);if (scorer == null) {@@ -77,6 +82,14 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleVreturn doc >= iterator.docID()&& (iterator.docID() == doc || iterator.advance(doc) == doc);}++ @Override+ public void prefetch(int doc) throws IOException {+ if (doc >= prefetchIterator.docID()+ && (prefetchIterator.docID() == doc || prefetchIterator.advance(doc) == doc)) {+ prefetchValues.prefetch(prefetchIterator.index(), 1);+ }+ }};}final KnnVectorValues.DocIndexIterator iterator = vectorValues.iterator();@@ -90,6 +103,14 @@ abstract class AbstractFullPrecisionVectorSimilarityValuesSource extends DoubleVpublic boolean advanceExact(int doc) throws IOException {return doc >= iterator.docID() && (iterator.docID() == doc || iterator.advance(doc) == doc);}++ @Override+ public void prefetch(int doc) throws IOException {+ if (doc >= prefetchIterator.docID()+ && (prefetchIterator.docID() == doc || prefetchIterator.advance(doc) == doc)) {+ prefetchValues.prefetch(prefetchIterator.index(), 1);+ }+ }};}diff --git a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.javaindex 33b6f07e654..20a2d812b47 100644--- a/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java+++ b/lucene/core/src/java/org/apache/lucene/search/DoubleValues.java@@ -40,6 +40,18 @@ public abstract class DoubleValues {return DocIdSetIterator.NO_MORE_DOCS;}+ /**+ * Optionally start loading whatever is needed to compute the value of the given document, without+ * waiting for it to become available. This lets a caller that knows several upcoming documents+ * keep more than one load in flight, which matters when the values live on slow storage. Callers+ * must still call {@link #advanceExact} before {@link #doubleValue()}; documents must be+ * prefetched in the same order in which they are later advanced to. The default is a no-op.+ *+ * @param doc the document whose value will be needed later+ * @lucene.experimental+ */+ public void prefetch(int doc) throws IOException {}+/*** Return the maximum score that documents between the current position and {@code upTo} can* produce. Default implementation returns {@link Float#POSITIVE_INFINITY}.diff --git a/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.javaindex b481ebf2927..6015a1c430a 100644--- a/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java@@ -17,8 +17,10 @@package org.apache.lucene.search;import java.io.IOException;+import java.util.List;import java.util.Objects;import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.VectorSimilarityFunction;/**@@ -33,6 +35,13 @@ import org.apache.lucene.index.VectorSimilarityFunction;*/public class RescoreTopNQuery extends Query {+ /**+ * Number of candidates whose loads are kept in flight while rescoring. Large enough to cover the+ * queue depth a block device needs to reach its peak read rate, small enough that the buffer does+ * not grow with the rescoring set.+ */+ private static final int PREFETCH_WINDOW = 128;+private final int n;private final Query query;private final DoubleValuesSource valuesSource;@@ -62,6 +71,25 @@ public class RescoreTopNQuery extends Query {Query rewritten = indexSearcher.rewrite(query);Weight weight = indexSearcher.createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1.0f);HitQueue queue = new HitQueue(n, false);+ // A value source that reads the inner scorer's score must be consumed at the scorer's current+ // position, so those cannot be prefetched ahead and are scored as we go.+ int originalCount =+ valuesSource.needsScores()+ ? rescoreInline(reader, weight, rewrittenValueSource, queue)+ : rescoreWithPrefetch(reader, weight, rewrittenValueSource, queue);+ int i = 0;+ ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()];+ for (ScoreDoc topDoc : queue) {+ scoreDocs[i++] = topDoc;+ }+ TopDocs topDocs =+ new TopDocs(new TotalHits(originalCount, TotalHits.Relation.EQUAL_TO), scoreDocs);+ return DocAndScoreQuery.createDocAndScoreQuery(reader, topDocs, 0);+ }++ private int rescoreInline(+ IndexReader reader, Weight weight, DoubleValuesSource rewrittenValueSource, HitQueue queue)+ throws IOException {int originalCount = 0;for (var leaf : reader.leaves()) {Scorer innerScorer = weight.scorer(leaf);@@ -70,25 +98,133 @@ public class RescoreTopNQuery extends Query {}DoubleValues rescores = rewrittenValueSource.getValues(leaf, getDoubleValues(innerScorer));DocIdSetIterator iterator = innerScorer.iterator();+ while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {+ rescoreInto(queue, rescores, leaf.docBase, iterator.docID());+ originalCount++;+ }+ }+ return originalCount;+ }++ /**+ * Starts the loads for the candidates ahead of the scoring position, so that more than one read+ * is in flight when the values live on slow storage.+ *+ * <p>Candidates are held in a fixed-size ring rather than buffered in full: a rerank shortlist+ * can be large, and its size is a function of the query rather than of the index. Each iteration+ * issues the prefetch for one candidate and, once the ring is full, scores the oldest one, which+ * by then has had a whole window of candidates' worth of time to load. That keeps {@link+ * #PREFETCH_WINDOW} reads in flight from end to end instead of draining every time a batch is+ * scored. The ring spans segments, because a shortlist is spread over all of them and a+ * segment-at-a-time window would go idle at every boundary.+ *+ * <p>Only doc ids are buffered, never values.+ */+ private int rescoreWithPrefetch(+ IndexReader reader, Weight weight, DoubleValuesSource rewrittenValueSource, HitQueue queue)+ throws IOException {+ final List<LeafReaderContext> leaves = reader.leaves();+ final DoubleValues[] leafValues = new DoubleValues[leaves.size()];+ final PrefetchRing ring = new PrefetchRing(PREFETCH_WINDOW);+ int originalCount = 0;++ for (int i = 0; i < leaves.size(); i++) {+ final LeafReaderContext leaf = leaves.get(i);+ Scorer innerScorer = weight.scorer(leaf);+ if (innerScorer == null) {+ continue;+ }+ DoubleValues rescores = rewrittenValueSource.getValues(leaf, null);+ leafValues[i] = rescores;+ DocIdSetIterator iterator = innerScorer.iterator();while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {int docId = iterator.docID();- if (rescores.advanceExact(docId)) {- double v = rescores.doubleValue();- queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, (float) v));- } else {- queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, 0f));+ // Issue the load first, so that the new read is in flight while we block on the oldest one.+ rescores.prefetch(docId);+ if (ring.isFull()) {+ scoreOldest(ring, leaves, leafValues, queue);+ originalCount++;}- originalCount++;+ ring.append(i, docId);}}- int i = 0;- ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()];- for (ScoreDoc topDoc : queue) {- scoreDocs[i++] = topDoc;++ while (ring.isEmpty() == false) {+ scoreOldest(ring, leaves, leafValues, queue);+ originalCount++;+ }+ return originalCount;+ }++ /**+ * Scores the candidate that has been in the ring longest, and so has had the most time to load,+ * then drops it.+ */+ private static void scoreOldest(+ PrefetchRing ring, List<LeafReaderContext> leaves, DoubleValues[] leafValues, HitQueue queue)+ throws IOException {+ int leafOrd = ring.oldestLeaf();+ rescoreInto(queue, leafValues[leafOrd], leaves.get(leafOrd).docBase, ring.oldestDoc());+ ring.removeOldest();+ }++ /**+ * Fixed-size FIFO of the candidates whose loads have been issued but whose scores have not been+ * read yet. Holds doc ids and the segment they belong to, never values, and never grows with the+ * size of the rescoring set.+ */+ private static final class PrefetchRing {+ private final int[] docs;+ private final int[] leafOrds;+ private int head;+ private int size;++ PrefetchRing(int capacity) {+ this.docs = new int[capacity];+ this.leafOrds = new int[capacity];+ }++ boolean isFull() {+ return size == docs.length;+ }++ boolean isEmpty() {+ return size == 0;+ }++ /** Adds a candidate. A window may straddle segments, so the segment travels with the doc id. */+ void append(int leafOrd, int doc) {+ assert isFull() == false : "ring is full";+ int tail = (head + size) % docs.length;+ docs[tail] = doc;+ leafOrds[tail] = leafOrd;+ size++;+ }++ int oldestDoc() {+ assert isEmpty() == false : "ring is empty";+ return docs[head];+ }++ int oldestLeaf() {+ assert isEmpty() == false : "ring is empty";+ return leafOrds[head];+ }++ void removeOldest() {+ assert isEmpty() == false : "ring is empty";+ head = (head + 1) % docs.length;+ size--;+ }+ }++ private static void rescoreInto(HitQueue queue, DoubleValues values, int docBase, int docId)+ throws IOException {+ if (values.advanceExact(docId)) {+ queue.insertWithOverflow(new ScoreDoc(docBase + docId, (float) values.doubleValue()));+ } else {+ queue.insertWithOverflow(new ScoreDoc(docBase + docId, 0f));}- TopDocs topDocs =- new TopDocs(new TotalHits(originalCount, TotalHits.Relation.EQUAL_TO), scoreDocs);- return DocAndScoreQuery.createDocAndScoreQuery(reader, topDocs, 0);}private DoubleValues getDoubleValues(Scorer innerScorer) {diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.javanew file mode 100644index 00000000000..faed0bf63b5--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene104/TestQuantizedVectorValuesPrefetch.java@@ -0,0 +1,135 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.codecs.lucene104;++import java.io.IOException;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedFloat16VectorValues;+import org.apache.lucene.codecs.lucene104.Lucene104ScalarQuantizedVectorsReader.ScalarQuantizedVectorValues;+import org.apache.lucene.index.Float16VectorValues;+import org.apache.lucene.index.FloatVectorValues;+import org.apache.lucene.index.KnnVectorValues;+import org.apache.lucene.tests.util.LuceneTestCase;++/**+ * The {@code FloatVectorValues} handed out for a quantized field wraps the raw full-precision+ * values. Any of these wrappers that forwards {@code vectorValue()} but not {@link+ * KnnVectorValues#prefetch(int, int)} inherits the no-op default instead, which silently leaves+ * full-precision rescoring at one outstanding read per candidate. Nothing else fails, so these+ * assertions are the only thing that catches a dropped override.+ */+public class TestQuantizedVectorValuesPrefetch extends LuceneTestCase {++ public void testScalarQuantizedValuesPrefetchRawVectors() throws IOException {+ RecordingFloatVectorValues raw = new RecordingFloatVectorValues();+ // The quantized side is unused by prefetch, which reads the raw vectors.+ ScalarQuantizedVectorValues values = new ScalarQuantizedVectorValues(raw, null);++ assertTrue(values.prefetch(7, 3));+ assertEquals(1, raw.calls);+ assertEquals(7, raw.lastOrd);+ assertEquals(3, raw.lastCount);+ }++ public void testScalarQuantizedFloat16ValuesPrefetchRawVectors() throws IOException {+ RecordingFloat16VectorValues raw = new RecordingFloat16VectorValues();+ ScalarQuantizedFloat16VectorValues values = new ScalarQuantizedFloat16VectorValues(raw, null);++ assertTrue(values.prefetch(2, 5));+ assertEquals(1, raw.calls);+ assertEquals(2, raw.lastOrd);+ assertEquals(5, raw.lastCount);+ }++ public void testNormalizedValuesPrefetchUnderlyingVectors() throws IOException {+ RecordingFloatVectorValues delegate = new RecordingFloatVectorValues();+ NormalizedFloatVectorValues values = new NormalizedFloatVectorValues(delegate);++ assertTrue(values.prefetch(4, 2));+ assertEquals(1, delegate.calls);+ assertEquals(4, delegate.lastOrd);+ assertEquals(2, delegate.lastCount);+ }++ /** Records what a wrapper forwards, and claims the prefetch was issued. */+ private static class RecordingFloatVectorValues extends FloatVectorValues {+ int calls;+ int lastOrd = -1;+ int lastCount = -1;++ @Override+ public boolean prefetch(int ord, int count) {+ calls++;+ lastOrd = ord;+ lastCount = count;+ return true;+ }++ @Override+ public float[] vectorValue(int ord) {+ return new float[dimension()];+ }++ @Override+ public FloatVectorValues copy() {+ return this;+ }++ @Override+ public int dimension() {+ return 4;+ }++ @Override+ public int size() {+ return 16;+ }+ }++ private static class RecordingFloat16VectorValues extends Float16VectorValues {+ int calls;+ int lastOrd = -1;+ int lastCount = -1;++ @Override+ public boolean prefetch(int ord, int count) {+ calls++;+ lastOrd = ord;+ lastCount = count;+ return true;+ }++ @Override+ public short[] vectorValue(int ord) {+ return new short[dimension()];+ }++ @Override+ public Float16VectorValues copy() {+ return this;+ }++ @Override+ public int dimension() {+ return 4;+ }++ @Override+ public int size() {+ return 16;+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.javaindex abb6017ad07..fec29554f22 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestOffHeapVectorValues.java@@ -26,20 +26,81 @@ import org.apache.lucene.tests.util.LuceneTestCase;public class TestOffHeapVectorValues extends LuceneTestCase {public void testPrefetchVectorValuesWithMoreThanOneOrds() throws IOException {+ // {1, 2} are consecutive, so they are prefetched with a single read.CountingIndexInput byteIndexInput = new CountingIndexInput();OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);values.prefetch(new int[] {1, 2, 5}, 2);- assertEquals(2, byteIndexInput.getPrefetchCount());+ assertEquals(1, byteIndexInput.getPrefetchCount());CountingIndexInput floatIndexInput = new CountingIndexInput();OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);floatValues.prefetch(new int[] {1, 2, 5}, 2);- assertEquals(2, floatIndexInput.getPrefetchCount());+ assertEquals(1, floatIndexInput.getPrefetchCount());CountingIndexInput float16IndexInput = new CountingIndexInput();OffHeapFloat16VectorValues float16Values = createTestFloat16VectorValues(float16IndexInput);float16Values.prefetch(new int[] {1, 2, 5}, 2);- assertEquals(2, float16IndexInput.getPrefetchCount());+ assertEquals(1, float16IndexInput.getPrefetchCount());+ }++ public void testPrefetchCoalescesConsecutiveOrdsOnly() throws IOException {+ // One read per run of consecutive ords: {1, 2, 3}, {7}, {9, 10}.+ CountingIndexInput byteIndexInput = new CountingIndexInput();+ OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);+ values.prefetch(new int[] {1, 2, 3, 7, 9, 10}, 6);+ assertEquals(3, byteIndexInput.getPrefetchCount());++ // Ords are not sorted here, so no run forms and every ord costs a read.+ CountingIndexInput floatIndexInput = new CountingIndexInput();+ OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);+ floatValues.prefetch(new int[] {10, 3, 2, 8}, 4);+ assertEquals(4, floatIndexInput.getPrefetchCount());+ }++ public void testPrefetchContiguousRun() throws IOException {+ // byteSize is 1 here, so a run of n ords is a single n-byte read at offset ord.+ CountingIndexInput byteIndexInput = new CountingIndexInput();+ OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);+ assertTrue(values.prefetch(10, 4));+ assertEquals(1, byteIndexInput.getPrefetchCount());+ assertEquals(10, byteIndexInput.lastOffset);+ assertEquals(4, byteIndexInput.lastLength);++ // A run reaching past the last ord is clamped to the vectors that exist.+ assertTrue(values.prefetch(98, 50));+ assertEquals(2, byteIndexInput.getPrefetchCount());+ assertEquals(2, byteIndexInput.lastLength);++ // Nothing to prefetch: out of range ord, or a non-positive count.+ assertFalse(values.prefetch(100, 1));+ assertFalse(values.prefetch(-1, 1));+ assertFalse(values.prefetch(1, 0));+ assertEquals(2, byteIndexInput.getPrefetchCount());+ }++ public void testPrefetchFloat16VectorValues() throws IOException {+ // fp16 stores vectors contiguously by ordinal just like fp32 and byte, so it prefetches the+ // same way: one read per run of consecutive ords.+ CountingIndexInput indexInput = new CountingIndexInput();+ OffHeapFloat16VectorValues values = createTestFloat16VectorValues(indexInput);+ assertTrue(values.prefetch(3, 2));+ assertEquals(1, indexInput.getPrefetchCount());+ assertEquals(3 * 2, indexInput.lastOffset);+ assertEquals(2 * 2, indexInput.lastLength);++ values.prefetch(new int[] {1, 2, 3, 7}, 4);+ assertEquals(3, indexInput.getPrefetchCount());++ assertFalse(values.prefetch(100, 1));+ assertEquals(3, indexInput.getPrefetchCount());+ }++ public void testPrefetchReportsNothingPrefetchedWhenInputDeclinesIt() throws IOException {+ // The default IndexInput#prefetch is a no-op returning false: callers gain nothing by+ // deferring.+ OffHeapFloatVectorValues floatValues =+ createTestFloatVectorValues(new NoopPrefetchIndexInput());+ assertFalse(floatValues.prefetch(1, 4));}public void testPrefetchVectorValuesWithLessThanOneOrds() throws IOException {@@ -70,18 +131,21 @@ public class TestOffHeapVectorValues extends LuceneTestCase {// array length so it only prefetches the available ords instead of running past the array end.CountingIndexInput byteIndexInput = new CountingIndexInput();OffHeapByteVectorValues values = createTestByteVectorValues(byteIndexInput);- values.prefetch(new int[] {1, 2}, 5);+ values.prefetch(new int[] {1, 3}, 5);assertEquals(2, byteIndexInput.getPrefetchCount());CountingIndexInput floatIndexInput = new CountingIndexInput();OffHeapFloatVectorValues floatValues = createTestFloatVectorValues(floatIndexInput);- floatValues.prefetch(new int[] {1, 2}, 5);+ floatValues.prefetch(new int[] {1, 3}, 5);assertEquals(2, floatIndexInput.getPrefetchCount());CountingIndexInput float16IndexInput = new CountingIndexInput();OffHeapFloat16VectorValues float16Values = createTestFloat16VectorValues(float16IndexInput);+ // Bounding is independent of coalescing: {1, 2} are consecutive, so the two in-bounds ords form+ // one run and cost a single read, where the ords above use {1, 3} and cost two. Either way the+ // trailing three slots of the length-2 array are never touched.float16Values.prefetch(new int[] {1, 2}, 5);- assertEquals(2, float16IndexInput.getPrefetchCount());+ assertEquals(1, float16IndexInput.getPrefetchCount());}private OffHeapByteVectorValues createTestByteVectorValues(IndexInput indexInput)@@ -102,9 +166,20 @@ public class TestOffHeapVectorValues extends LuceneTestCase {100, 100, indexInput, 2, null, VectorSimilarityFunction.EUCLIDEAN);}- private static final class CountingIndexInput extends IndexInput {+ /** An input that declines to prefetch, like the default {@link IndexInput#prefetch}. */+ private static final class NoopPrefetchIndexInput extends CountingIndexInput {++ @Override+ public boolean prefetch(long offset, long length) throws IOException {+ return false;+ }+ }++ private static class CountingIndexInput extends IndexInput {AtomicInteger counter;+ long lastOffset = -1;+ long lastLength = -1;public CountingIndexInput() {super("closing index input");@@ -114,6 +189,8 @@ public class TestOffHeapVectorValues extends LuceneTestCase {@Overridepublic boolean prefetch(long offset, long length) throws IOException {counter.incrementAndGet();+ lastOffset = offset;+ lastLength = length;return true;}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.javanew file mode 100644index 00000000000..955baeeff92--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterReadersForwardVectorPrefetch.java@@ -0,0 +1,193 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.io.IOException;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.NumericDocValuesField;+import org.apache.lucene.search.IndexSearcher;+import org.apache.lucene.search.KnnFloatVectorQuery;+import org.apache.lucene.search.Query;+import org.apache.lucene.search.RescoreTopNQuery;+import org.apache.lucene.search.Sort;+import org.apache.lucene.search.SortField;+import org.apache.lucene.search.TopDocs;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;++/**+ * Filter readers that wrap {@link KnnVectorValues} have to forward {@link+ * KnnVectorValues#prefetch(int, int)} along with everything else. A wrapper that forwards {@code+ * vectorValue()} but not {@code prefetch()} inherits the no-op default, so prefetching silently+ * stops reaching the store and reads go back to one at a time. Nothing else changes, which is why+ * this needs asserting rather than eyeballing.+ */+public class TestFilterReadersForwardVectorPrefetch extends LuceneTestCase {++ private static final String FIELD = "vector";+ private static final String SORT_FIELD = "sort";+ private static final int DIMS = 4;++ public void testExitableDirectoryReaderForwardsPrefetch() throws Exception {+ try (PrefetchCountingDirectory dir = newIndex()) {+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ // A timeout that never fires: the wrapping, not the timeout, is what is under test.+ DirectoryReader exitable = new ExitableDirectoryReader(reader, () -> false);+ LeafReader leaf = getOnlyLeafReader(exitable);+ assertPrefetchReachesStore("ExitableDirectoryReader", dir, leaf);+ }+ }+ }++ public void testSortingCodecReaderForwardsPrefetch() throws Exception {+ try (PrefetchCountingDirectory dir = newIndex()) {+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ Sort sort = new Sort(new SortField(SORT_FIELD, SortField.Type.LONG, true));+ CodecReader sorting =+ SortingCodecReader.wrap(SlowCodecReaderWrapper.wrap(getOnlyLeafReader(reader)), sort);+ assertPrefetchReachesStore("SortingCodecReader", dir, sorting);+ }+ }+ }++ public void testFullPrecisionRescoringWorksThroughExitableReader() throws Exception {+ // Prefetching runs over a second copy() of the vector values, so a wrapper whose copy() throws+ // breaks rescoring outright rather than just losing the prefetch. A search carrying a timeout+ // goes through ExitableDirectoryReader, so this is the common case, not an exotic one.+ try (PrefetchCountingDirectory dir = newIndex()) {+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ DirectoryReader exitable = new ExitableDirectoryReader(reader, () -> false);+ IndexSearcher searcher = new IndexSearcher(exitable);+ float[] target = new float[DIMS];+ target[0] = 1f;+ Query knn = new KnnFloatVectorQuery(FIELD, target, 8);+ TopDocs topDocs =+ searcher.search(+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knn, target, FIELD, 4), 4);+ assertEquals(4, topDocs.scoreDocs.length);+ }+ }+ }++ public void testUnwrappedReaderPrefetchesForComparison() throws Exception {+ // Guards the test itself: if the plain reader stopped prefetching, the assertions above would+ // pass or fail for reasons that have nothing to do with the wrappers.+ try (PrefetchCountingDirectory dir = newIndex()) {+ try (DirectoryReader reader = DirectoryReader.open(dir)) {+ assertPrefetchReachesStore("unwrapped reader", dir, getOnlyLeafReader(reader));+ }+ }+ }++ private static void assertPrefetchReachesStore(+ String what, PrefetchCountingDirectory dir, LeafReader leaf) throws IOException {+ FloatVectorValues values = leaf.getFloatVectorValues(FIELD);+ assertNotNull(what + " has no vector values", values);+ int before = dir.prefetchCount(".vec");+ values.prefetch(0, 2);+ assertTrue(+ what+ + " dropped prefetch: no read was started on the raw vector file. Counted prefetches "+ + "per extension: "+ + dir.counts,+ dir.prefetchCount(".vec") > before);+ }++ private PrefetchCountingDirectory newIndex() throws IOException {+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());+ IndexWriterConfig config = new IndexWriterConfig();+ config.setCodec(+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));+ // Attribute prefetches to the vector file itself rather than to an enclosing .cfs, and keep+ // everything in one segment so getOnlyLeafReader() applies.+ config.setUseCompoundFile(false);+ config.setMaxBufferedDocs(64);+ try (IndexWriter writer = new IndexWriter(dir, config)) {+ for (int i = 0; i < 16; i++) {+ Document doc = new Document();+ float[] vector = new float[DIMS];+ vector[i % DIMS] = 1f;+ doc.add(new KnnFloatVectorField(FIELD, vector, VectorSimilarityFunction.DOT_PRODUCT));+ doc.add(new NumericDocValuesField(SORT_FIELD, i));+ writer.addDocument(doc);+ }+ writer.forceMerge(1);+ }+ return dir;+ }++ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */+ private static class PrefetchCountingDirectory extends FilterDirectory {+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();++ PrefetchCountingDirectory(Directory in) {+ super(in);+ }++ int prefetchCount(String extension) {+ AtomicInteger counter = counts.get(extension);+ return counter == null ? 0 : counter.get();+ }++ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ IndexInput in = super.openInput(name, context);+ int dot = name.lastIndexOf('.');+ String extension = dot < 0 ? name : name.substring(dot);+ AtomicInteger counter = counts.computeIfAbsent(extension, _ -> new AtomicInteger());+ return new CountingIndexInput(in, counter);+ }+ }++ private static class CountingIndexInput extends FilterIndexInput {+ private final AtomicInteger counter;++ CountingIndexInput(IndexInput in, AtomicInteger counter) {+ super("CountingIndexInput(" + in + ")", in);+ this.counter = counter;+ }++ @Override+ public boolean prefetch(long offset, long length) throws IOException {+ counter.incrementAndGet();+ // The count, not the return value, is what the assertions look at: ByteBuffersDirectory does+ // not implement prefetch, so the delegate reports false even though the call arrived.+ return in.prefetch(offset, length);+ }++ @Override+ public IndexInput clone() {+ return new CountingIndexInput(in.clone(), counter);+ }++ @Override+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.javanew file mode 100644index 00000000000..35d92564960--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryFloat16Prefetch.java@@ -0,0 +1,184 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.HashMap;+import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.IntField;+import org.apache.lucene.document.KnnFloat16VectorField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.apache.lucene.util.TestVectorUtil;++/**+ * The float16 counterpart of {@link TestRescoreTopNQueryPrefetch}: verifies that full-precision+ * rescoring of a float16 field also reaches {@link IndexInput#prefetch} on the raw vector file.+ *+ * <p>Both float32 and float16 rescoring inherit {@code DoubleValues#prefetch} from {@code+ * AbstractFullPrecisionVectorSimilarityValuesSource}, so one test passing while the other fails+ * would mean the shared implementation is reached on only one of the two paths. A missing override+ * anywhere along it silently degrades rescoring to a single outstanding read per candidate, which+ * no correctness test can detect.+ */+public class TestRescoreTopNQueryFloat16Prefetch extends LuceneTestCase {++ private static final String FIELD = "vector";+ private static final int DIMS = 8;+ private static final VectorSimilarityFunction SIMILARITY = VectorSimilarityFunction.DOT_PRODUCT;++ public void testFloat16RescoringPrefetchesRawVectors() throws Exception {+ Map<Integer, short[]> vectors = new HashMap<>();+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());+ try {+ IndexWriterConfig config = new IndexWriterConfig();+ config.setCodec(+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));+ // Keep files uncompounded so prefetches can be attributed to the raw vector file itself+ // rather than to the enclosing .cfs.+ config.setUseCompoundFile(false);+ int perSegment = 60;+ int segments = 3;+ try (IndexWriter writer = new IndexWriter(dir, config)) {+ for (int s = 0; s < segments; s++) {+ for (int i = 0; i < perSegment; i++) {+ int id = s * perSegment + i;+ short[] vector = TestVectorUtil.randomVectorFloat16(DIMS);+ Document doc = new Document();+ doc.add(new IntField("id", id, Field.Store.YES));+ doc.add(new KnnFloat16VectorField(FIELD, vector, SIMILARITY));+ writer.addDocument(doc);+ vectors.put(id, vector);+ }+ writer.flush();+ }+ }++ try (IndexReader reader = DirectoryReader.open(dir)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ short[] target = TestVectorUtil.randomVectorFloat16(DIMS);+ int k = 10;+ int shortlist = k * 5;++ // Baseline: the first-phase query alone must not touch the raw vectors at all.+ KnnFloat16VectorQuery knnQuery = new KnnFloat16VectorQuery(FIELD, target, shortlist);+ searcher.search(knnQuery, shortlist);+ int afterFirstPhase = dir.prefetchCount(".vec");++ Query rescored =+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, target, FIELD, k);+ TopDocs topDocs = searcher.search(rescored, k);++ int prefetched = dir.prefetchCount(".vec") - afterFirstPhase;+ assertTrue(+ "float16 rescoring issued no prefetch on the raw vector file; counted prefetches per "+ + "extension: "+ + dir.counts+ + ". A missing prefetch override on the float16 rescore path leaves the store at "+ + "one outstanding read per candidate.",+ prefetched > 0);++ // Rescoring reads one raw vector per candidate, so prefetches should scale with the+ // shortlist rather than being a token few.+ assertTrue(+ "expected prefetches to scale with the rescored candidate count, got " + prefetched,+ prefetched >= topDocs.scoreDocs.length);++ // Deferring the scoring behind the prefetch window must not change the scores.+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ Document doc = searcher.storedFields().document(scoreDoc.doc);+ int id = doc.getField("id").numericValue().intValue();+ short[] docVector = vectors.get(id);+ assertNotNull("vector for id " + id + " not found", docVector);+ assertEquals(+ "score does not match full-precision similarity for id " + id,+ SIMILARITY.compare(target, docVector),+ scoreDoc.score,+ 1e-5);+ }+ }+ } finally {+ dir.close();+ }+ }++ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */+ private static class PrefetchCountingDirectory extends FilterDirectory {+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();++ PrefetchCountingDirectory(Directory in) {+ super(in);+ }++ int prefetchCount(String extension) {+ AtomicInteger counter = counts.get(extension);+ return counter == null ? 0 : counter.get();+ }++ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ IndexInput in = super.openInput(name, context);+ int dot = name.lastIndexOf('.');+ if (dot < 0) {+ return in;+ }+ AtomicInteger counter = counts.computeIfAbsent(name.substring(dot), _ -> new AtomicInteger());+ return new CountingIndexInput(in, counter);+ }+ }++ private static class CountingIndexInput extends FilterIndexInput {+ private final AtomicInteger counter;++ CountingIndexInput(IndexInput in, AtomicInteger counter) {+ super("CountingIndexInput(" + in + ")", in);+ this.counter = counter;+ }++ @Override+ public boolean prefetch(long offset, long length) throws IOException {+ counter.incrementAndGet();+ return in.prefetch(offset, length);+ }++ @Override+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);+ }++ @Override+ public IndexInput clone() {+ return new CountingIndexInput(in.clone(), counter);+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.javanew file mode 100644index 00000000000..d3f2c36bfea--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQueryPrefetch.java@@ -0,0 +1,208 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.HashMap;+import java.util.Map;+import java.util.Random;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.lucene.codecs.lucene104.Lucene104HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.IntField;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.store.FilterDirectory;+import org.apache.lucene.store.FilterIndexInput;+import org.apache.lucene.store.IOContext;+import org.apache.lucene.store.IndexInput;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;++/**+ * Verifies that full-precision rescoring actually reaches {@link IndexInput#prefetch} on the raw+ * vector file. The prefetch has to travel from {@link RescoreTopNQuery} through {@link+ * DoubleValues} and the quantized {@code FloatVectorValues} wrapper down to the store; a missing+ * override anywhere on that path silently degrades rescoring to one outstanding read at a time,+ * which is invisible in correctness tests.+ */+public class TestRescoreTopNQueryPrefetch extends LuceneTestCase {++ private static final String FIELD = "vector";+ private static final int DIMS = 8;+ private static final VectorSimilarityFunction SIMILARITY = VectorSimilarityFunction.DOT_PRODUCT;++ public void testRescoringPrefetchesRawVectors() throws Exception {+ Random random = random();+ Map<Integer, float[]> vectors = new HashMap<>();+ PrefetchCountingDirectory dir = new PrefetchCountingDirectory(new ByteBuffersDirectory());+ try {+ IndexWriterConfig config = new IndexWriterConfig();+ config.setCodec(+ TestUtil.alwaysKnnVectorsFormat(new Lucene104HnswScalarQuantizedVectorsFormat()));+ // Keep files uncompounded so prefetches can be attributed to the raw vector file itself+ // rather than to the enclosing .cfs.+ config.setUseCompoundFile(false);+ int perSegment = 60;+ int segments = 3;+ try (IndexWriter writer = new IndexWriter(dir, config)) {+ for (int s = 0; s < segments; s++) {+ for (int i = 0; i < perSegment; i++) {+ int id = s * perSegment + i;+ float[] vector = randomVector(random);+ Document doc = new Document();+ doc.add(new IntField("id", id, Field.Store.YES));+ doc.add(new KnnFloatVectorField(FIELD, vector, SIMILARITY));+ writer.addDocument(doc);+ vectors.put(id, vector);+ }+ writer.flush();+ }+ }++ try (IndexReader reader = DirectoryReader.open(dir)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ float[] target = randomVector(random);+ int k = 10;+ int shortlist = k * 5;++ // Baseline: the first-phase query alone must not touch the raw vectors at all.+ KnnFloatVectorQuery knnQuery = new KnnFloatVectorQuery(FIELD, target, shortlist);+ searcher.search(knnQuery, shortlist);+ int afterFirstPhase = dir.prefetchCount(".vec");++ Query rescored =+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, target, FIELD, k);+ TopDocs topDocs = searcher.search(rescored, k);++ int prefetched = dir.prefetchCount(".vec") - afterFirstPhase;+ assertTrue(+ "rescoring issued no prefetch on the raw vector file; counted prefetches per extension: "+ + dir.counts+ + ". A missing prefetch override on the rescore path leaves the store at one "+ + "outstanding read per candidate.",+ prefetched > 0);++ // Rescoring reads one raw vector per candidate, so prefetches should scale with the+ // shortlist rather than being a token few.+ assertTrue(+ "expected prefetches to scale with the rescored candidate count, got " + prefetched,+ prefetched >= topDocs.scoreDocs.length);++ // Deferring the scoring behind the prefetch window must not change the scores.+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ Document doc = searcher.storedFields().document(scoreDoc.doc);+ int id = doc.getField("id").numericValue().intValue();+ float[] docVector = vectors.get(id);+ assertNotNull("vector for id " + id + " not found", docVector);+ assertEquals(+ "score does not match full-precision similarity for id " + id,+ SIMILARITY.compare(target, docVector),+ scoreDoc.score,+ 1e-5);+ }+ }+ } finally {+ dir.close();+ }+ }++ private static float[] randomVector(Random random) {+ float[] v = new float[DIMS];+ for (int i = 0; i < v.length; i++) {+ v[i] = random.nextFloat() - 0.5f;+ }+ VectorUtilSupport.normalize(v);+ return v;+ }++ /** Normalizes so DOT_PRODUCT stays in range. */+ private static class VectorUtilSupport {+ static void normalize(float[] v) {+ double sum = 0;+ for (float f : v) {+ sum += (double) f * f;+ }+ float norm = (float) Math.sqrt(sum);+ if (norm == 0) {+ v[0] = 1f;+ return;+ }+ for (int i = 0; i < v.length; i++) {+ v[i] /= norm;+ }+ }+ }++ /** Counts {@link IndexInput#prefetch} calls per file extension, including through slices. */+ private static class PrefetchCountingDirectory extends FilterDirectory {+ final Map<String, AtomicInteger> counts = new ConcurrentHashMap<>();++ PrefetchCountingDirectory(Directory in) {+ super(in);+ }++ int prefetchCount(String extension) {+ AtomicInteger counter = counts.get(extension);+ return counter == null ? 0 : counter.get();+ }++ @Override+ public IndexInput openInput(String name, IOContext context) throws IOException {+ IndexInput in = super.openInput(name, context);+ int dot = name.lastIndexOf('.');+ if (dot < 0) {+ return in;+ }+ AtomicInteger counter = counts.computeIfAbsent(name.substring(dot), _ -> new AtomicInteger());+ return new CountingIndexInput(in, counter);+ }+ }++ private static class CountingIndexInput extends FilterIndexInput {+ private final AtomicInteger counter;++ CountingIndexInput(IndexInput in, AtomicInteger counter) {+ super("CountingIndexInput(" + in + ")", in);+ this.counter = counter;+ }++ @Override+ public boolean prefetch(long offset, long length) throws IOException {+ counter.incrementAndGet();+ return in.prefetch(offset, length);+ }++ @Override+ public IndexInput slice(String sliceDescription, long offset, long length) throws IOException {+ return new CountingIndexInput(in.slice(sliceDescription, offset, length), counter);+ }++ @Override+ public IndexInput clone() {+ return new CountingIndexInput(in.clone(), counter);+ }+ }+}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.javaindex b3cacda9f81..b4acc025f82 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupScalarQuantizedVectorValues.java@@ -200,6 +200,11 @@ final class DedupScalarQuantizedVectorValues {groupView.prefetch(scratch, numOrds);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());+ }+@Overridepublic byte[] vectorValue(int ord) throws IOException {return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));@@ -304,8 +309,8 @@ final class DedupScalarQuantizedVectorValues {}@Override- public void prefetch(int[] ordsToPrefetch, int numOrds) throws IOException {- rawValues.prefetch(ordsToPrefetch, numOrds);+ public boolean prefetch(int ord, int count) throws IOException {+ return rawValues.prefetch(ord, count);}@Overridediff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.javaindex 23656f3fcab..70981af05c5 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupUtil.java@@ -23,6 +23,7 @@ import java.io.IOException;import org.apache.lucene.codecs.lucene95.OrdToDocDISIReaderConfiguration;import org.apache.lucene.index.DocsWithFieldSet;import org.apache.lucene.index.FieldInfo;+import org.apache.lucene.index.KnnVectorValues;import org.apache.lucene.index.VectorEncoding;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.sandbox.codecs.dedup.DedupVectorValues.FieldOrdToGroupOrd;@@ -181,6 +182,32 @@ final class DedupUtil {}}+ /**+ * Prefetches a run of {@code count} field ords starting at {@code ord} from the group view. Field+ * ords that are consecutive need not map to consecutive group ords, so the run is remapped and+ * prefetched ord by ord rather than as a single contiguous read.+ *+ * @return true if a prefetch was actually issued for any of them+ */+ static boolean prefetchRemapped(+ KnnVectorValues groupView,+ FieldOrdToGroupOrd fieldOrdToGroupOrd,+ int ord,+ int count,+ int size)+ throws IOException {+ if (ord < 0 || ord >= size || count <= 0) {+ return false;+ }+ final int runLength = Math.min(count, size - ord);+ boolean prefetched = false;+ for (int i = 0; i < runLength; i++) {+ // Not short-circuiting: every ord in the run has to be issued.+ prefetched |= groupView.prefetch(fieldOrdToGroupOrd.get(ord + i), 1);+ }+ return prefetched;+ }+static long hashBytes(byte[] bytes) {return murmurhash3_x64_128(bytes, 0, bytes.length, GOOD_FAST_HASH_SEED)[0];}diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.javaindex 2e79cf65377..fffe388ff6d 100644--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/dedup/DedupVectorValues.java@@ -162,6 +162,11 @@ sealed interface DedupVectorValuesgroupView.prefetch(scratch, numOrds);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());+ }+@Overridepublic byte[] vectorValue(int ord) throws IOException {return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));@@ -287,6 +292,11 @@ sealed interface DedupVectorValuesgroupView.prefetch(scratch, numOrds);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());+ }+@Overridepublic float[] vectorValue(int ord) throws IOException {return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));@@ -412,6 +422,11 @@ sealed interface DedupVectorValuesgroupView.prefetch(scratch, numOrds);}+ @Override+ public boolean prefetch(int ord, int count) throws IOException {+ return DedupUtil.prefetchRemapped(groupView, fieldOrdToGroupOrd, ord, count, size());+ }+@Overridepublic short[] vectorValue(int ord) throws IOException {return groupView.vectorValue(fieldOrdToGroupOrd.get(ord));