Apache Lucene open-source search software
Add Query for reranking KnnFloatVectorQuery with full-precision vectors (#14009)
Dzung Bui3404496b3d77
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex ae7f9dbb36c..3d76fabe19c 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -115,6 +115,9 @@ New Features* GITHUB#14784: Make pack methods public for BigIntegerPoint and HalfFloatPoint. (Prudhvi Godithi)+* GITHUB#14009: Add a new Query that can rescore other Query based on a generic DoubleValueSource+ and trim the results down to top N (Anh Dung Bui)+Improvements---------------------* GITHUB#14458: Add an IndexDeletion policy that retains the last N commits. (Owais Kazi)diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.javaindex 5590643535f..510e5cf72e5 100644--- a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java@@ -16,12 +16,8 @@*/package org.apache.lucene.search;-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;-import java.io.IOException;import java.util.ArrayList;-import java.util.Arrays;-import java.util.Comparator;import java.util.HashMap;import java.util.Iterator;import java.util.List;@@ -142,7 +138,7 @@ abstract class AbstractKnnVectorQuery extends Query {if (topK.scoreDocs.length == 0) {return new MatchNoDocsQuery();}- return createRewrittenQuery(reader, topK, reentryCount);+ return DocAndScoreQuery.createDocAndScoreQuery(reader, topK, reentryCount);}private TopDocs runSearchTasks(@@ -398,46 +394,6 @@ abstract class AbstractKnnVectorQuery extends Query {}}- protected Query createRewrittenQuery(IndexReader reader, TopDocs topK, int reentryCount) {- int len = topK.scoreDocs.length;- assert len > 0;- float maxScore = topK.scoreDocs[0].score;- Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc));- int[] docs = new int[len];- float[] scores = new float[len];- for (int i = 0; i < len; i++) {- docs[i] = topK.scoreDocs[i].doc;- scores[i] = topK.scoreDocs[i].score;- }- int[] segmentStarts = findSegmentStarts(reader.leaves(), docs);- return new DocAndScoreQuery(- docs,- scores,- maxScore,- segmentStarts,- topK.totalHits.value(),- reader.getContext().id(),- reentryCount);- }-- static int[] findSegmentStarts(List<LeafReaderContext> leaves, int[] docs) {- int[] starts = new int[leaves.size() + 1];- starts[starts.length - 1] = docs.length;- if (starts.length == 2) {- return starts;- }- int resultIndex = 0;- for (int i = 1; i < starts.length - 1; i++) {- int upper = leaves.get(i).docBase;- resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper);- if (resultIndex < 0) {- resultIndex = -1 - resultIndex;- }- starts[i] = resultIndex;- }- return starts;- }-@Overridepublic void visit(QueryVisitor visitor) {if (visitor.acceptField(field)) {@@ -483,199 +439,6 @@ abstract class AbstractKnnVectorQuery extends Query {return filter;}- /** Caches the results of a KnnVector search: a list of docs and their scores */- static class DocAndScoreQuery extends Query {-- private final int[] docs;- private final float[] scores;- private final float maxScore;- private final int[] segmentStarts;- private final long visited;- private final Object contextIdentity;- private final int reentryCount;-- /**- * Constructor- *- * @param docs the global docids of documents that match, in ascending order- * @param scores the scores of the matching documents- * @param maxScore the max of those scores? why do we need to pass in?- * @param segmentStarts the indexes in docs and scores corresponding to the first matching- * document in each segment. If a segment has no matching documents, it should be assigned- * the index of the next segment that does. There should be a final entry that is always- * docs.length-1.- * @param visited the number of graph nodes that were visited, and for which vector distance- * scores were evaluated.- * @param contextIdentity an object identifying the reader context that was used to build this- * query- */- DocAndScoreQuery(- int[] docs,- float[] scores,- float maxScore,- int[] segmentStarts,- long visited,- Object contextIdentity,- int reentryCount) {- this.docs = docs;- this.scores = scores;- this.maxScore = maxScore;- this.segmentStarts = segmentStarts;- this.visited = visited;- this.contextIdentity = contextIdentity;- this.reentryCount = reentryCount;- }-- /*- DocAndScoreQuery(DocAndScoreQuery other) {- this.docs = other.docs;- this.scores = other.scores;- this.maxScore = other.maxScore;- this.segmentStarts = other.segmentStarts;- this.visited = other.visited;- this.contextIdentity = other.contextIdentity;- this.reentryCount = other.reentryCount;- }- */-- int reentryCount() {- return reentryCount;- }-- @Override- public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)- throws IOException {- if (searcher.getIndexReader().getContext().id() != contextIdentity) {- throw new IllegalStateException("This DocAndScore query was created by a different reader");- }- return new Weight(this) {- @Override- public Explanation explain(LeafReaderContext context, int doc) {- int found = Arrays.binarySearch(docs, doc + context.docBase);- if (found < 0) {- return Explanation.noMatch("not in top " + docs.length + " docs");- }- return Explanation.match(scores[found] * boost, "within top " + docs.length + " docs");- }-- @Override- public int count(LeafReaderContext context) {- return segmentStarts[context.ord + 1] - segmentStarts[context.ord];- }-- @Override- public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {- if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {- return null;- }- final var scorer =- new Scorer() {- final int lower = segmentStarts[context.ord];- final int upper = segmentStarts[context.ord + 1];- int upTo = -1;-- @Override- public DocIdSetIterator iterator() {- return new DocIdSetIterator() {- @Override- public int docID() {- return docIdNoShadow();- }-- @Override- public int nextDoc() {- if (upTo == -1) {- upTo = lower;- } else {- ++upTo;- }- return docIdNoShadow();- }-- @Override- public int advance(int target) throws IOException {- return slowAdvance(target);- }-- @Override- public long cost() {- return upper - lower;- }- };- }-- @Override- public float getMaxScore(int docId) {- return maxScore * boost;- }-- @Override- public float score() {- return scores[upTo] * boost;- }-- /**- * move the implementation of docID() into a differently-named method so we can call- * it from DocIDSetIterator.docID() even though this class is anonymous- *- * @return the current docid- */- private int docIdNoShadow() {- if (upTo == -1) {- return -1;- }- if (upTo >= upper) {- return NO_MORE_DOCS;- }- return docs[upTo] - context.docBase;- }-- @Override- public int docID() {- return docIdNoShadow();- }- };- return new DefaultScorerSupplier(scorer);- }-- @Override- public boolean isCacheable(LeafReaderContext ctx) {- return true;- }- };- }-- @Override- public String toString(String field) {- return "DocAndScoreQuery[" + docs[0] + ",...][" + scores[0] + ",...]," + maxScore;- }-- @Override- public void visit(QueryVisitor visitor) {- visitor.visitLeaf(this);- }-- public long visited() {- return visited;- }-- @Override- public boolean equals(Object obj) {- if (sameClassAs(obj) == false) {- return false;- }- return contextIdentity == ((DocAndScoreQuery) obj).contextIdentity- && Arrays.equals(docs, ((DocAndScoreQuery) obj).docs)- && Arrays.equals(scores, ((DocAndScoreQuery) obj).scores);- }-- @Override- public int hashCode() {- return Objects.hash(- classHash(), contextIdentity, Arrays.hashCode(docs), Arrays.hashCode(scores));- }- }-public KnnSearchStrategy getSearchStrategy() {return searchStrategy;}diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreQuery.java b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreQuery.javanew file mode 100644index 00000000000..cf71ca8d8c2--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreQuery.java@@ -0,0 +1,248 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;++import java.io.IOException;+import java.util.Arrays;+import java.util.Comparator;+import java.util.List;+import java.util.Objects;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.LeafReaderContext;++/** A query that wraps precomputed documents and scores */+class DocAndScoreQuery extends Query {++ private final int[] docs;+ private final float[] scores;+ private final float maxScore;+ private final int[] segmentStarts;+ private final long visited;+ private final Object contextIdentity;+ private final int reentryCount;++ /**+ * Constructor+ *+ * @param docs the global docids of documents that match, in ascending order+ * @param scores the scores of the matching documents+ * @param maxScore the max of those scores? why do we need to pass in?+ * @param segmentStarts the indexes in docs and scores corresponding to the first matching+ * document in each segment. If a segment has no matching documents, it should be assigned the+ * index of the next segment that does. There should be a final entry that is always+ * docs.length-1.+ * @param visited the number of graph nodes that were visited, and for which vector distance+ * scores were evaluated.+ * @param contextIdentity an object identifying the reader context that was used to build this+ * query+ */+ DocAndScoreQuery(+ int[] docs,+ float[] scores,+ float maxScore,+ int[] segmentStarts,+ long visited,+ Object contextIdentity,+ int reentryCount) {+ this.docs = docs;+ this.scores = scores;+ this.maxScore = maxScore;+ this.segmentStarts = segmentStarts;+ this.visited = visited;+ this.contextIdentity = contextIdentity;+ this.reentryCount = reentryCount;+ }++ static Query createDocAndScoreQuery(IndexReader reader, TopDocs topK, int reentryCount) {+ int len = topK.scoreDocs.length;+ assert len > 0;+ float maxScore = topK.scoreDocs[0].score;+ Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc));+ int[] docs = new int[len];+ float[] scores = new float[len];+ for (int i = 0; i < len; i++) {+ docs[i] = topK.scoreDocs[i].doc;+ scores[i] = topK.scoreDocs[i].score;+ }+ int[] segmentStarts = findSegmentStarts(reader.leaves(), docs);+ return new DocAndScoreQuery(+ docs,+ scores,+ maxScore,+ segmentStarts,+ topK.totalHits.value(),+ reader.getContext().id(),+ reentryCount);+ }++ static int[] findSegmentStarts(List<LeafReaderContext> leaves, int[] docs) {+ int[] starts = new int[leaves.size() + 1];+ starts[starts.length - 1] = docs.length;+ if (starts.length == 2) {+ return starts;+ }+ int resultIndex = 0;+ for (int i = 1; i < starts.length - 1; i++) {+ int upper = leaves.get(i).docBase;+ resultIndex = Arrays.binarySearch(docs, resultIndex, docs.length, upper);+ if (resultIndex < 0) {+ resultIndex = -1 - resultIndex;+ }+ starts[i] = resultIndex;+ }+ return starts;+ }++ int reentryCount() {+ return reentryCount;+ }++ @Override+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)+ throws IOException {+ if (searcher.getIndexReader().getContext().id() != contextIdentity) {+ throw new IllegalStateException("This DocAndScore query was created by a different reader");+ }+ return new Weight(this) {+ @Override+ public Explanation explain(LeafReaderContext context, int doc) {+ int found = Arrays.binarySearch(docs, doc + context.docBase);+ if (found < 0) {+ return Explanation.noMatch("not in top " + docs.length + " docs");+ }+ return Explanation.match(scores[found] * boost, "within top " + docs.length + " docs");+ }++ @Override+ public int count(LeafReaderContext context) {+ return segmentStarts[context.ord + 1] - segmentStarts[context.ord];+ }++ @Override+ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {+ if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {+ return null;+ }+ final var scorer =+ new Scorer() {+ final int lower = segmentStarts[context.ord];+ final int upper = segmentStarts[context.ord + 1];+ int upTo = -1;++ @Override+ public DocIdSetIterator iterator() {+ return new DocIdSetIterator() {+ @Override+ public int docID() {+ return docIdNoShadow();+ }++ @Override+ public int nextDoc() {+ if (upTo == -1) {+ upTo = lower;+ } else {+ ++upTo;+ }+ return docIdNoShadow();+ }++ @Override+ public int advance(int target) throws IOException {+ return slowAdvance(target);+ }++ @Override+ public long cost() {+ return upper - lower;+ }+ };+ }++ @Override+ public float getMaxScore(int docId) {+ return maxScore * boost;+ }++ @Override+ public float score() {+ return scores[upTo] * boost;+ }++ /**+ * move the implementation of docID() into a differently-named method so we can call+ * it from DocIDSetIterator.docID() even though this class is anonymous+ *+ * @return the current docid+ */+ private int docIdNoShadow() {+ if (upTo == -1) {+ return -1;+ }+ if (upTo >= upper) {+ return NO_MORE_DOCS;+ }+ return docs[upTo] - context.docBase;+ }++ @Override+ public int docID() {+ return docIdNoShadow();+ }+ };+ return new DefaultScorerSupplier(scorer);+ }++ @Override+ public boolean isCacheable(LeafReaderContext ctx) {+ return true;+ }+ };+ }++ @Override+ public String toString(String field) {+ return "DocAndScoreQuery[" + docs[0] + ",...][" + scores[0] + ",...]," + maxScore;+ }++ @Override+ public void visit(QueryVisitor visitor) {+ visitor.visitLeaf(this);+ }++ public long visited() {+ return visited;+ }++ @Override+ public boolean equals(Object obj) {+ if (sameClassAs(obj) == false) {+ return false;+ }+ return contextIdentity == ((DocAndScoreQuery) obj).contextIdentity+ && Arrays.equals(docs, ((DocAndScoreQuery) obj).docs)+ && Arrays.equals(scores, ((DocAndScoreQuery) obj).scores);+ }++ @Override+ public int hashCode() {+ return Objects.hash(+ classHash(), contextIdentity, Arrays.hashCode(docs), Arrays.hashCode(scores));+ }+}diff --git a/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.javanew file mode 100644index 00000000000..3b9a8fa045c--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/search/RescoreTopNQuery.java@@ -0,0 +1,151 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.Objects;+import org.apache.lucene.index.IndexReader;++/**+ * A Query that re-scores another Query with a {@link DoubleValuesSource} function and cut-off the+ * results at top N. Unlike {@link Rescorer} which does rescoring at post-collection phase, this+ * Query does the rescoring at rewrite() phase. The reason it operates in rewrite phase is to be+ * compatible with KNN vector query, where the results are collected upfront, but it can work with+ * any type of Query. Unlike or <code>FunctionScoreQuery</code>, this Query will work even with the+ * no-scoring {@link ScoreMode}.+ *+ * @lucene.experimental+ */+public class RescoreTopNQuery extends Query {++ private final int n;+ private final Query query;+ private final DoubleValuesSource valuesSource;++ /**+ * Execute the inner Query, re-score using a customizable DoubleValueSource and trim down the+ * result to k+ *+ * @param query the query to execute as initial phase+ * @param valuesSource the double value source to re-score+ * @param n the number of documents to find+ * @throws IllegalArgumentException if <code>n</code> is less than 1+ */+ public RescoreTopNQuery(Query query, DoubleValuesSource valuesSource, int n) {+ if (n < 1) {+ throw new IllegalArgumentException("n must be >= 1");+ }+ this.query = query;+ this.valuesSource = valuesSource;+ this.n = n;+ }++ @Override+ public Query rewrite(IndexSearcher indexSearcher) throws IOException {+ DoubleValuesSource rewrittenValueSource = valuesSource.rewrite(indexSearcher);+ IndexReader reader = indexSearcher.getIndexReader();+ Query rewritten = indexSearcher.rewrite(query);+ Weight weight = indexSearcher.createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1.0f);+ HitQueue queue = new HitQueue(n, false);+ int originalCount = 0;+ for (var leaf : reader.leaves()) {+ Scorer innerScorer = weight.scorer(leaf);+ if (innerScorer == null) {+ continue;+ }+ DoubleValues rescores = rewrittenValueSource.getValues(leaf, getDoubleValues(innerScorer));+ DocIdSetIterator iterator = innerScorer.iterator();+ while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {+ int docId = iterator.docID();+ if (rescores.advanceExact(docId)) {+ double v = rescores.doubleValue();+ queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, (float) v));+ } else {+ queue.insertWithOverflow(new ScoreDoc(leaf.docBase + docId, 0f));+ }+ originalCount++;+ }+ }+ int i = 0;+ ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()];+ for (ScoreDoc topDoc : queue) {+ scoreDocs[i++] = topDoc;+ }+ TopDocs topDocs =+ new TopDocs(new TotalHits(originalCount, TotalHits.Relation.EQUAL_TO), scoreDocs);+ return DocAndScoreQuery.createDocAndScoreQuery(reader, topDocs, 0);+ }++ private DoubleValues getDoubleValues(Scorer innerScorer) {+ // if the value source doesn't need document score to compute value, return null+ if (valuesSource.needsScores() == false) {+ return null;+ }+ return DoubleValuesSource.fromScorer(innerScorer);+ }++ @Override+ public int hashCode() {+ int result = valuesSource.hashCode();+ result = 31 * result + Objects.hash(query, n);+ return result;+ }++ @Override+ public boolean equals(Object o) {+ if (this == o) return true;+ if (o == null || getClass() != o.getClass()) return false;+ RescoreTopNQuery that = (RescoreTopNQuery) o;+ return Objects.equals(query, that.query)+ && Objects.equals(valuesSource, that.valuesSource)+ && n == that.n;+ }++ @Override+ public void visit(QueryVisitor visitor) {+ query.visit(visitor);+ }++ @Override+ public String toString(String field) {+ return getClass().getSimpleName()+ + ":"+ + query.toString(field)+ + ":"+ + valuesSource.toString()+ + "["+ + n+ + "]";+ }++ /**+ * Utility method to create a new RescoreTopNQuery which uses full-precision vectors for+ * rescoring.+ *+ * @param in the inner Query to rescore+ * @param targetVector the target vector to compute score+ * @param field the vector field to compute score+ * @param n the number of results to keep+ * @return the RescoreTopNQuery+ */+ public static Query createFullPrecisionRescorerQuery(+ Query in, float[] targetVector, String field, int n) {+ DoubleValuesSource valuaSource =+ new FullPrecisionFloatVectorSimilarityValuesSource(targetVector, field);+ return new RescoreTopNQuery(in, valuaSource, n);+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.javaindex 5278beb5d97..48ae27ce0b0 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java@@ -232,10 +232,10 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {maxScore = Math.max(maxScore, scores[i]);}IndexReader indexReader = searcher.getIndexReader();- int[] segments = AbstractKnnVectorQuery.findSegmentStarts(indexReader.leaves(), docs);+ int[] segments = DocAndScoreQuery.findSegmentStarts(indexReader.leaves(), docs);- AbstractKnnVectorQuery.DocAndScoreQuery query =- new AbstractKnnVectorQuery.DocAndScoreQuery(+ DocAndScoreQuery query =+ new DocAndScoreQuery(docs,scores,maxScore,diff --git a/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.javanew file mode 100644index 00000000000..e1be2c81822--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/search/TestRescoreTopNQuery.java@@ -0,0 +1,166 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.search;++import java.io.IOException;+import java.util.HashMap;+import java.util.Map;+import java.util.Random;+import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.document.IntField;+import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.index.DirectoryReader;+import org.apache.lucene.index.IndexReader;+import org.apache.lucene.index.IndexWriter;+import org.apache.lucene.index.IndexWriterConfig;+import org.apache.lucene.index.Term;+import org.apache.lucene.index.VectorSimilarityFunction;+import org.apache.lucene.store.ByteBuffersDirectory;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;+import org.apache.lucene.tests.util.TestUtil;+import org.junit.Assert;+import org.junit.Before;+import org.junit.Test;++public class TestRescoreTopNQuery extends LuceneTestCase {++ private static final String FIELD = "vector";+ private static final VectorSimilarityFunction VECTOR_SIMILARITY_FUNCTION =+ VectorSimilarityFunction.COSINE;+ private static final int NUM_VECTORS = 1000;+ private static final int VECTOR_DIMENSION = 128;++ private Directory directory;+ private IndexWriterConfig config;++ @Before+ @Override+ public void setUp() throws Exception {+ super.setUp();+ directory = new ByteBuffersDirectory();++ // Set up the IndexWriterConfig to use quantized vector storage+ config = new IndexWriterConfig();+ config.setCodec(+ TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswScalarQuantizedVectorsFormat()));+ }++ @Test+ public void testInvalidN() {+ expectThrows(+ IllegalArgumentException.class,+ () ->+ new RescoreTopNQuery(+ new TermQuery(new Term("test")), DoubleValuesSource.constant(0), 0));+ }++ @Test+ public void testRescoreField() throws Exception {+ Map<Integer, float[]> vectors = new HashMap<>();++ Random random = random();++ int numVectors = atLeast(NUM_VECTORS);+ int numSegments = random.nextInt(2, 10);++ // Step 1: Index random vectors in quantized format+ try (IndexWriter writer = new IndexWriter(directory, config)) {+ for (int j = 0; j < numSegments; j++) {+ for (int i = 0; i < numVectors; i++) {+ float[] vector = randomFloatVector(VECTOR_DIMENSION, random);+ Document doc = new Document();+ int id = j * numVectors + i;+ doc.add(new IntField("id", id, Field.Store.YES));+ doc.add(new KnnFloatVectorField(FIELD, vector, VECTOR_SIMILARITY_FUNCTION));+ writer.addDocument(doc);+ vectors.put(id, vector);++ writer.flush();+ }+ }+ }++ // Step 2: Run TwoPhaseKnnVectorQuery with a random target vector+ try (IndexReader reader = DirectoryReader.open(directory)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ float[] targetVector = randomFloatVector(VECTOR_DIMENSION, random);+ int k = 10;+ double oversample = random.nextFloat(1.5f, 3.0f);++ KnnFloatVectorQuery knnQuery =+ new KnnFloatVectorQuery(FIELD, targetVector, k + (int) (k * oversample));++ Query query =+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, targetVector, FIELD, k);+ TopDocs topDocs = searcher.search(query, k);++ // Step 3: Verify that TopDocs scores match similarity with unquantized vectors+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ Document retrievedDoc = searcher.storedFields().document(scoreDoc.doc);+ int id = retrievedDoc.getField("id").numericValue().intValue();+ float[] docVector = vectors.get(id);+ assert docVector != null : "Vector for id " + id + " not found";+ float expectedScore = VECTOR_SIMILARITY_FUNCTION.compare(targetVector, docVector);+ Assert.assertEquals(+ "Score does not match expected similarity for doc ord: " + scoreDoc.doc + ", id: " + id,+ expectedScore,+ scoreDoc.score,+ 1e-5);+ }+ }+ }++ public void testMissingDoubleValues() throws IOException {+ Random random = random();++ try (IndexWriter writer = new IndexWriter(directory, config)) {+ float[] vector = randomFloatVector(VECTOR_DIMENSION, random);+ Document doc = new Document();+ doc.add(new KnnFloatVectorField(FIELD, vector, VECTOR_SIMILARITY_FUNCTION));+ writer.addDocument(doc);+ }++ // Step 2: Run TwoPhaseKnnVectorQuery with a random target vector+ try (IndexReader reader = DirectoryReader.open(directory)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ float[] targetVector = randomFloatVector(VECTOR_DIMENSION, random);+ int k = 1;++ KnnFloatVectorQuery knnQuery = new KnnFloatVectorQuery(FIELD, targetVector, k);++ Query query =+ RescoreTopNQuery.createFullPrecisionRescorerQuery(knnQuery, targetVector, "field-1", k);+ TopDocs topDocs = searcher.search(query, k);++ // Step 3: The rescoring field is invalid, so the score should be 0+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {+ Assert.assertEquals("Score must be 0 for missing DoubleValues", 0, scoreDoc.score, 1e-5);+ }+ }+ }++ private float[] randomFloatVector(int dimension, Random random) {+ float[] vector = new float[dimension];+ for (int i = 0; i < dimension; i++) {+ vector[i] = random.nextFloat();+ }+ return vector;+ }+}