Apache Lucene open-source search software
Simplify max score for kNN vector queries (#12146)
Julie Tibshirani8340b01c3cc2
diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.javaindex 6e348fcc5ee..f2e4b125f98 100644--- a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java@@ -189,6 +189,10 @@ abstract class AbstractKnnVectorQuery extends Query {private Query createRewrittenQuery(IndexReader reader, TopDocs topK) {int len = topK.scoreDocs.length;++ assert len > 0;+ float maxScore = topK.scoreDocs[0].score;+Arrays.sort(topK.scoreDocs, Comparator.comparingInt(a -> a.doc));int[] docs = new int[len];float[] scores = new float[len];@@ -197,7 +201,7 @@ abstract class AbstractKnnVectorQuery extends Query {scores[i] = topK.scoreDocs[i].score;}int[] segmentStarts = findSegmentStarts(reader, docs);- return new DocAndScoreQuery(docs, scores, segmentStarts, reader.getContext().id());+ return new DocAndScoreQuery(docs, scores, maxScore, segmentStarts, reader.getContext().id());}private int[] findSegmentStarts(IndexReader reader, int[] docs) {@@ -265,6 +269,7 @@ abstract class AbstractKnnVectorQuery extends Query {private final int[] docs;private final float[] scores;+ private final float maxScore;private final int[] segmentStarts;private final Object contextIdentity;@@ -280,9 +285,11 @@ abstract class AbstractKnnVectorQuery extends Query {* @param contextIdentity an object identifying the reader context that was used to build this* query*/- DocAndScoreQuery(int[] docs, float[] scores, int[] segmentStarts, Object contextIdentity) {+ DocAndScoreQuery(+ int[] docs, float[] scores, float maxScore, int[] segmentStarts, Object contextIdentity) {this.docs = docs;this.scores = scores;+ this.maxScore = maxScore;this.segmentStarts = segmentStarts;this.contextIdentity = contextIdentity;}@@ -343,11 +350,6 @@ abstract class AbstractKnnVectorQuery extends Query {@Overridepublic float getMaxScore(int docId) {- docId += context.docBase;- float maxScore = 0;- for (int idx = Math.max(0, upTo); idx < upper && docs[idx] <= docId; idx++) {- maxScore = Math.max(maxScore, scores[idx]);- }return maxScore * boost;}@@ -356,19 +358,6 @@ abstract class AbstractKnnVectorQuery extends Query {return scores[upTo] * boost;}- @Override- public int advanceShallow(int docid) {- int start = Math.max(upTo, lower);- int docidIndex = Arrays.binarySearch(docs, start, upper, docid + context.docBase);- if (docidIndex < 0) {- docidIndex = -1 - docidIndex;- }- if (docidIndex >= upper) {- return NO_MORE_DOCS;- }- return docs[docidIndex];- }-/*** move the implementation of docID() into a differently-named method so we can call it* from DocIDSetIterator.docID() even though this class is anonymousdiff --git a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.javaindex 88d551619a4..dbb13d9f058 100644--- a/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java+++ b/lucene/core/src/test/org/apache/lucene/search/BaseKnnVectorQueryTestCase.java@@ -244,36 +244,6 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {}}- public void testAdvanceShallow() throws IOException {- try (Directory d = newDirectory()) {- try (IndexWriter w = new IndexWriter(d, new IndexWriterConfig())) {- for (int j = 0; j < 5; j++) {- Document doc = new Document();- doc.add(getKnnVectorField("field", new float[] {j, j}));- w.addDocument(doc);- }- }- try (IndexReader reader = DirectoryReader.open(d)) {- IndexSearcher searcher = new IndexSearcher(reader);- AbstractKnnVectorQuery query = getKnnVectorQuery("field", new float[] {2, 3}, 3);- Query dasq = query.rewrite(searcher);- Scorer scorer =- dasq.createWeight(searcher, ScoreMode.COMPLETE, 1).scorer(reader.leaves().get(0));- // before advancing the iterator- assertEquals(1, scorer.advanceShallow(0));- assertEquals(1, scorer.advanceShallow(1));- assertEquals(NO_MORE_DOCS, scorer.advanceShallow(10));-- // after advancing the iterator- scorer.iterator().advance(2);- assertEquals(2, scorer.advanceShallow(0));- assertEquals(2, scorer.advanceShallow(2));- assertEquals(3, scorer.advanceShallow(3));- assertEquals(NO_MORE_DOCS, scorer.advanceShallow(10));- }- }- }-public void testScoreEuclidean() throws IOException {float[][] vectors = new float[5][];for (int j = 0; j < 5; j++) {@@ -291,9 +261,6 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {assertEquals(-1, scorer.docID());expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);- // test getMaxScore- assertEquals(0, scorer.getMaxScore(-1), 0);- assertEquals(0, scorer.getMaxScore(0), 0);// This is 1 / ((l2distance((2,3), (2, 2)) = 1) + 1) = 0.5assertEquals(1 / 2f, scorer.getMaxScore(2), 0);assertEquals(1 / 2f, scorer.getMaxScore(Integer.MAX_VALUE), 0);@@ -304,6 +271,7 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {assertEquals(1 / 6f, scorer.score(), 0);assertEquals(3, it.advance(3));assertEquals(1 / 2f, scorer.score(), 0);+assertEquals(NO_MORE_DOCS, it.advance(4));expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);}@@ -330,32 +298,30 @@ abstract class BaseKnnVectorQueryTestCase extends LuceneTestCase {assertEquals(-1, scorer.docID());expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);- // test getMaxScore- assertEquals(0, scorer.getMaxScore(-1), 0);- /* maxAtZero = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then+ /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then* normalized by (1 + x) /2.*/- float maxAtZero =+ float score0 =(float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);- assertEquals(maxAtZero, scorer.getMaxScore(0), 0.001);- /* max at 2 is actually the score for doc 1 which is the highest (since doc 1 vector (2, 4)- * is the closest to (2, 3)). This is ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then+ /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then* normalized by (1 + x) /2*/- float expected =+ float score1 =(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);- assertEquals(expected, scorer.getMaxScore(2), 0);- assertEquals(expected, scorer.getMaxScore(Integer.MAX_VALUE), 0);++ // doc 1 happens to have the maximum score+ assertEquals(score1, scorer.getMaxScore(2), 0.0001);+ assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);DocIdSetIterator it = scorer.iterator();assertEquals(3, it.cost());assertEquals(0, it.nextDoc());// doc 0 has (1, 1)- assertEquals(maxAtZero, scorer.score(), 0.0001);+ assertEquals(score0, scorer.score(), 0.0001);assertEquals(1, it.advance(1));- assertEquals(expected, scorer.score(), 0);- assertEquals(2, it.nextDoc());+ assertEquals(score1, scorer.score(), 0.0001);+// since topK was 3assertEquals(NO_MORE_DOCS, it.advance(4));expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.javaindex c4f10f874be..04f5a53b246 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java@@ -133,32 +133,30 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {assertEquals(-1, scorer.docID());expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);- // test getMaxScore- assertEquals(0, scorer.getMaxScore(-1), 0);- /* maxAtZero = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then+ /* score0 = ((2,3) * (1, 1) = 5) / (||2, 3|| * ||1, 1|| = sqrt(26)), then* normalized by (1 + x) /2.*/- float maxAtZero =+ float score0 =(float) ((1 + (2 * 1 + 3 * 1) / Math.sqrt((2 * 2 + 3 * 3) * (1 * 1 + 1 * 1))) / 2);- assertEquals(maxAtZero, scorer.getMaxScore(0), 0.001);- /* max at 2 is actually the score for doc 1 which is the highest (since doc 1 vector (2, 4)- * is the closest to (2, 3)). This is ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then+ /* score1 = ((2,3) * (2, 4) = 16) / (||2, 3|| * ||2, 4|| = sqrt(260)), then* normalized by (1 + x) /2*/- float expected =+ float score1 =(float) ((1 + (2 * 2 + 3 * 4) / Math.sqrt((2 * 2 + 3 * 3) * (2 * 2 + 4 * 4))) / 2);- assertEquals(expected, scorer.getMaxScore(2), 0);- assertEquals(expected, scorer.getMaxScore(Integer.MAX_VALUE), 0);++ // doc 1 happens to have the max score+ assertEquals(score1, scorer.getMaxScore(2), 0.0001);+ assertEquals(score1, scorer.getMaxScore(Integer.MAX_VALUE), 0.0001);DocIdSetIterator it = scorer.iterator();assertEquals(3, it.cost());assertEquals(0, it.nextDoc());// doc 0 has (1, 1)- assertEquals(maxAtZero, scorer.score(), 0.0001);+ assertEquals(score0, scorer.score(), 0.0001);assertEquals(1, it.advance(1));- assertEquals(expected, scorer.score(), 0);- assertEquals(2, it.nextDoc());+ assertEquals(score1, scorer.score(), 0.0001);+// since topK was 3assertEquals(NO_MORE_DOCS, it.advance(4));expectThrows(ArrayIndexOutOfBoundsException.class, scorer::score);