Apache Lucene open-source search software
Fix HNSW graph visitation limit bug (#12413)
Benjamin Trent861153020a16
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 74d81c5f2de..c65b584beb1 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -155,6 +155,8 @@ Bug Fixes* GITHUB#12388: JoinUtil queries were ignoring boosts. (Alan Woodward)+* GITHUB#12413: Fix HNSW graph search bug that potentially leaked unapproved docs (Ben Trent).+Other---------------------diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.javaindex 1cd5183a993..ab792ca3bd0 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphSearcher.java@@ -204,26 +204,26 @@ public class HnswGraphSearcher<T> {if (initialEp == -1) {return new NeighborQueue(1, true);}- NeighborQueue results;- results = new NeighborQueue(1, false);- int[] eps = new int[] {graph.entryNode()};- int numVisited = 0;- for (int level = graph.numLevels() - 1; level >= 1; level--) {- results.clear();- graphSearcher.searchLevel(results, query, 1, level, eps, vectors, graph, null, visitedLimit);-- numVisited += results.visitedCount();- visitedLimit -= results.visitedCount();-- if (results.incomplete()) {- results.setVisitedCount(numVisited);- return results;- }- eps[0] = results.pop();+ int[] epAndVisited = graphSearcher.findBestEntryPoint(query, vectors, graph, visitedLimit);+ int numVisited = epAndVisited[1];+ int ep = epAndVisited[0];+ if (ep == -1) {+ NeighborQueue results = new NeighborQueue(1, false);+ results.setVisitedCount(numVisited);+ results.markIncomplete();+ return results;}- results = new NeighborQueue(topK, false);+ NeighborQueue results = new NeighborQueue(topK, false);graphSearcher.searchLevel(- results, query, topK, 0, eps, vectors, graph, acceptOrds, visitedLimit);+ results,+ query,+ topK,+ 0,+ new int[] {ep},+ vectors,+ graph,+ acceptOrds,+ visitedLimit - numVisited);results.setVisitedCount(results.visitedCount() + numVisited);return results;}@@ -256,6 +256,56 @@ public class HnswGraphSearcher<T> {return results;}+ /**+ * Function to find the best entry point from which to search the zeroth graph layer.+ *+ * @param query vector query with which to search+ * @param vectors random access vector values+ * @param graph the HNSWGraph+ * @param visitLimit How many vectors are allowed to be visited+ * @return An integer array whose first element is the best entry point, and second is the number+ * of candidates visited. Entry point of `-1` indicates visitation limit exceed+ * @throws IOException When accessing the vector fails+ */+ private int[] findBestEntryPoint(+ T query, RandomAccessVectorValues<T> vectors, HnswGraph graph, int visitLimit)+ throws IOException {+ int size = graph.size();+ int visitedCount = 1;+ prepareScratchState(vectors.size());+ int currentEp = graph.entryNode();+ float currentScore = compare(query, vectors, currentEp);+ boolean foundBetter;+ for (int level = graph.numLevels() - 1; level >= 1; level--) {+ foundBetter = true;+ visited.set(currentEp);+ // Keep searching the given level until we stop finding a better candidate entry point+ while (foundBetter) {+ foundBetter = false;+ graphSeek(graph, level, currentEp);+ int friendOrd;+ while ((friendOrd = graphNextNeighbor(graph)) != NO_MORE_DOCS) {+ assert friendOrd < size : "friendOrd=" + friendOrd + "; size=" + size;+ if (visited.getAndSet(friendOrd)) {+ continue;+ }+ if (visitedCount >= visitLimit) {+ return new int[] {-1, visitedCount};+ }+ float friendSimilarity = compare(query, vectors, friendOrd);+ visitedCount++;+ if (friendSimilarity > currentScore+ || (friendSimilarity == currentScore && friendOrd < currentEp)) {+ currentScore = friendSimilarity;+ currentEp = friendOrd;+ foundBetter = true;+ }+ }+ }+ }+ return new int[] {currentEp, visitedCount};+ }+/*** Add the closest neighbors found to a priority queue (heap). These are returned in REVERSE* proximity order -- the most distant neighbor of the topK found, i.e. the one with the lowest