Apache Lucene open-source search software
Adjust equivalent min similarity HNSW exploration logic (#14366)
Benjamin Trent11c2ebd09b2f
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5b84225699f..26a984534e3 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -206,7 +206,7 @@ Bug Fixes* GITHUB#14126: Avoid overflow in index input slices invariant checks(Chris Hegarty)-* GITHUB#14215: Fix degenerate case in HNSW where all vectors have the same score. (Ben Trent)+* GITHUB#14215, GITHUB#14249, GITHUB#14366: Fix degenerate case in HNSW where all vectors have the same score. (Ben Trent, Ignacio Vera)* GITHUB#14320: Fix DirectIOIndexInput seek to not read when position is within buffer (Chris Hegarty)diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.javaindex 4c7075dbd9f..cee873446dc 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java@@ -266,11 +266,21 @@ public class HnswGraphSearcher extends AbstractHnswGraphSearcher {// A bound that holds the minimum similarity to the query vector that a candidate vector must// have to be considered.float minAcceptedSimilarity = Math.nextUp(results.minCompetitiveSimilarity());+ // We should allow exploring equivalent minAcceptedSimilarity values at least once+ boolean shouldExploreMinSim = true;while (candidates.size() > 0 && results.earlyTerminated() == false) {// get the best candidate (closest or best scoring)float topCandidateSimilarity = candidates.topScore();if (topCandidateSimilarity < minAcceptedSimilarity) {- break;+ // if the similarity is equivalent to the minAcceptedSimilarity,+ // we should explore one candidate+ // however, running into many duplicates can be expensive,+ // so we should stop exploring if equivalent minimum scores are found+ if (shouldExploreMinSim && Math.nextUp(topCandidateSimilarity) == minAcceptedSimilarity) {+ shouldExploreMinSim = false;+ } else {+ break;+ }}int topCandidateNode = candidates.pop();@@ -291,7 +301,13 @@ public class HnswGraphSearcher extends AbstractHnswGraphSearcher {candidates.add(friendOrd, friendSimilarity);if (acceptOrds == null || acceptOrds.get(friendOrd)) {if (results.collect(friendOrd, friendSimilarity)) {+ float oldMinAcceptedSimilarity = minAcceptedSimilarity;minAcceptedSimilarity = Math.nextUp(results.minCompetitiveSimilarity());+ if (minAcceptedSimilarity > oldMinAcceptedSimilarity) {+ // we adjusted our minAcceptedSimilarity, so we should explore the next equivalent+ // if necessary+ shouldExploreMinSim = true;+ }}}}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.javaindex 858629c357c..cd50705ac73 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java@@ -49,7 +49,6 @@ import org.apache.lucene.search.TopDocs;import org.apache.lucene.store.Directory;import org.apache.lucene.tests.util.LuceneTestCase;import org.apache.lucene.tests.util.TestUtil;-import org.apache.lucene.util.Bits;import org.apache.lucene.util.BytesRef;import org.apache.lucene.util.IOUtils;import org.apache.lucene.util.VectorUtil;@@ -243,10 +242,18 @@ public class TestKnnGraph extends LuceneTestCase {* orientation of the various priority queues, the scoring function, but not so much the* approximate KNN search algorithm*/- assertGraphSearch(new int[] {0, 15, 3, 18, 5}, new float[] {0f, 0.1f}, dr);+ assertGraphSearch(+ new int[] {0, 15, 3, 18, 5},+ new float[] {0.99f, 0.55f, 0.50f, 0.36f, 0.22f},+ new float[] {0f, 0.1f},+ dr);// Tiebreaking by docid must be done after search.// assertGraphSearch(new int[]{11, 1, 8, 14, 21}, new float[]{2, 2}, dr);- assertGraphSearch(new int[] {15, 18, 0, 3, 5}, new float[] {0.3f, 0.8f}, dr);+ assertGraphSearch(+ new int[] {15, 18, 0, 3, 5},+ new float[] {0.88f, 0.65f, 0.58f, 0.47f, 0.40f},+ new float[] {0.3f, 0.8f},+ dr);}}}@@ -297,15 +304,11 @@ public class TestKnnGraph extends LuceneTestCase {latch.await();IndexSearcher searcher = manager.acquire();try {- KnnFloatVectorQuery query =- new KnnFloatVectorQuery("vector", new float[] {0f, 0.1f}, 5);- TopDocs results = searcher.search(query, 5);- StoredFields storedFields = searcher.storedFields();- for (ScoreDoc doc : results.scoreDocs) {- // map docId to insertion id- doc.doc = Integer.parseInt(storedFields.document(doc.doc).get("id"));- }- assertResults(new int[] {0, 15, 3, 18, 5}, results);+ assertGraphSearch(+ new int[] {0, 15, 3, 18, 5},+ new float[] {0.99f, 0.55f, 0.50f, 0.36f, 0.22f},+ new float[] {0f, 0.1f},+ searcher.getIndexReader());} finally {manager.release(searcher);}@@ -323,7 +326,8 @@ public class TestKnnGraph extends LuceneTestCase {IOUtils.close(manager, iw, dir);}- private void assertGraphSearch(int[] expected, float[] vector, IndexReader reader)+ private void assertGraphSearch(+ int[] expected, float[] expectedScores, float[] vector, IndexReader reader)throws IOException {TopDocs results = doKnnSearch(reader, vector, 5);StoredFields storedFields = reader.storedFields();@@ -331,29 +335,26 @@ public class TestKnnGraph extends LuceneTestCase {// map docId to insertion iddoc.doc = Integer.parseInt(storedFields.document(doc.doc).get("id"));}- assertResults(expected, results);+ assertResults(expected, expectedScores, results);}private static TopDocs doKnnSearch(IndexReader reader, float[] vector, int k) throws IOException {- TopDocs[] results = new TopDocs[reader.leaves().size()];- for (LeafReaderContext ctx : reader.leaves()) {- Bits liveDocs = ctx.reader().getLiveDocs();- results[ctx.ord] =- ctx.reader()- .searchNearestVectors(KNN_GRAPH_FIELD, vector, k, liveDocs, Integer.MAX_VALUE);- if (ctx.docBase > 0) {- for (ScoreDoc doc : results[ctx.ord].scoreDocs) {- doc.doc += ctx.docBase;- }- }- }- return TopDocs.merge(k, results);+ IndexSearcher searcher = new IndexSearcher(reader);+ return searcher.search(new KnnFloatVectorQuery(KNN_GRAPH_FIELD, vector, k), k);}- private void assertResults(int[] expected, TopDocs results) {+ private void assertResults(int[] expected, float[] expectedScores, TopDocs results) {assertEquals(results.toString(), expected.length, results.scoreDocs.length);for (int i = expected.length - 1; i >= 0; i--) {- assertEquals(Arrays.toString(results.scoreDocs), expected[i], results.scoreDocs[i].doc);+ assertEquals(+ "doc mismatch at idx: " + i + " results: " + Arrays.toString(results.scoreDocs),+ expected[i],+ results.scoreDocs[i].doc);+ assertEquals(+ "score mismatch at idx: " + i + " results: " + Arrays.toString(results.scoreDocs),+ expectedScores[i],+ results.scoreDocs[i].score,+ 0.01f);}}