Apache Lucene open-source search software
Minor vector search matching doc optimizations (#12152)
Benjamin Trentdbfca9a62b83
diff --git a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.javaindex f2e4b125f98..39c2d34ffd2 100644--- a/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java+++ b/lucene/core/src/java/org/apache/lucene/search/AbstractKnnVectorQuery.java@@ -204,7 +204,7 @@ abstract class AbstractKnnVectorQuery extends Query {return new DocAndScoreQuery(docs, scores, maxScore, segmentStarts, reader.getContext().id());}- private int[] findSegmentStarts(IndexReader reader, int[] docs) {+ static int[] findSegmentStarts(IndexReader reader, int[] docs) {int[] starts = new int[reader.leaves().size() + 1];starts[starts.length - 1] = docs.length;if (starts.length == 2) {@@ -311,8 +311,15 @@ abstract class AbstractKnnVectorQuery extends Query {}@Override- public Scorer scorer(LeafReaderContext context) {+ public int count(LeafReaderContext context) {+ return segmentStarts[context.ord + 1] - segmentStarts[context.ord];+ }+ @Override+ public Scorer scorer(LeafReaderContext context) {+ if (segmentStarts[context.ord] == segmentStarts[context.ord + 1]) {+ return null;+ }return new Scorer(this) {final int lower = segmentStarts[context.ord];final int upper = segmentStarts[context.ord + 1];diff --git a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.javaindex 04f5a53b246..f55d552ffc8 100644--- a/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java+++ b/lucene/core/src/test/org/apache/lucene/search/TestKnnFloatVectorQuery.java@@ -16,13 +16,19 @@*/package org.apache.lucene.search;+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomFloat;import static org.apache.lucene.index.VectorSimilarityFunction.DOT_PRODUCT;import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;import java.io.IOException;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Comparator;+import java.util.List;import org.apache.lucene.document.Document;import org.apache.lucene.document.Field;import org.apache.lucene.document.KnnFloatVectorField;+import org.apache.lucene.document.StringField;import org.apache.lucene.index.DirectoryReader;import org.apache.lucene.index.IndexReader;import org.apache.lucene.index.IndexWriter;@@ -30,6 +36,8 @@ import org.apache.lucene.index.IndexWriterConfig;import org.apache.lucene.index.LeafReaderContext;import org.apache.lucene.index.VectorSimilarityFunction;import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.index.RandomIndexWriter;+import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.util.TestVectorUtil;import org.apache.lucene.util.VectorUtil;@@ -164,6 +172,71 @@ public class TestKnnFloatVectorQuery extends BaseKnnVectorQueryTestCase {}}+ public void testDocAndScoreQueryBasics() throws IOException {+ try (Directory directory = newDirectory()) {+ final DirectoryReader reader;+ try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {+ for (int i = 0; i < 50; i++) {+ Document doc = new Document();+ doc.add(new StringField("field", "value" + i, Field.Store.NO));+ iw.addDocument(doc);+ if (i % 10 == 0) {+ iw.flush();+ }+ }+ reader = iw.getReader();+ }+ try (reader) {+ IndexSearcher searcher = LuceneTestCase.newSearcher(reader);+ List<ScoreDoc> scoreDocsList = new ArrayList<>();+ for (int doc = 0; doc < 30; doc += 1 + random().nextInt(5)) {+ scoreDocsList.add(new ScoreDoc(doc, randomFloat()));+ }+ ScoreDoc[] scoreDocs = scoreDocsList.toArray(new ScoreDoc[0]);+ int[] docs = new int[scoreDocs.length];+ float[] scores = new float[scoreDocs.length];+ float maxScore = Float.MIN_VALUE;+ for (int i = 0; i < scoreDocs.length; i++) {+ docs[i] = scoreDocs[i].doc;+ scores[i] = scoreDocs[i].score;+ maxScore = Math.max(maxScore, scores[i]);+ }+ int[] segments = AbstractKnnVectorQuery.findSegmentStarts(reader, docs);++ AbstractKnnVectorQuery.DocAndScoreQuery query =+ new AbstractKnnVectorQuery.DocAndScoreQuery(+ docs, scores, maxScore, segments, reader.getContext().id());+ final Weight w = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0f);+ TopDocs topDocs = searcher.search(query, 100);+ assertEquals(scoreDocs.length, topDocs.totalHits.value);+ assertEquals(TotalHits.Relation.EQUAL_TO, topDocs.totalHits.relation);+ Arrays.sort(topDocs.scoreDocs, Comparator.comparingInt(scoreDoc -> scoreDoc.doc));+ assertEquals(scoreDocs.length, topDocs.scoreDocs.length);+ for (int i = 0; i < scoreDocs.length; i++) {+ assertEquals(scoreDocs[i].doc, topDocs.scoreDocs[i].doc);+ assertEquals(scoreDocs[i].score, topDocs.scoreDocs[i].score, 0.0001f);+ assertTrue(searcher.explain(query, scoreDocs[i].doc).isMatch());+ }++ for (LeafReaderContext leafReaderContext : searcher.getLeafContexts()) {+ final Scorer scorer = w.scorer(leafReaderContext);+ final int count = w.count(leafReaderContext);+ if (scorer == null) {+ assertEquals(0, count);+ } else {+ assertTrue(leafReaderContext.toString(), scorer.getMaxScore(NO_MORE_DOCS) > 0.0f);+ assertTrue(count > 0);+ int iteratorCount = 0;+ while (scorer.iterator().nextDoc() != NO_MORE_DOCS) {+ iteratorCount++;+ }+ assertEquals(iteratorCount, count);+ }+ }+ }+ }+ }+private static class ThrowingKnnVectorQuery extends KnnFloatVectorQuery {public ThrowingKnnVectorQuery(String field, float[] target, int k, Query filter) {