Apache Lucene open-source search software
HNSW - don't add duplicates when repairing graph links (#16728)
Simon Cooper4eb488d0f9e8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex bbae22bf84e..c403a8bdb8f 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -632,6 +632,9 @@ Bug Fixes* GITHUB#16577: Fix incorrect position increment generated by FixedShingleFilter. (Kishor Kharbas)+* GITHUB#16728: Handle edge cases with inaccurate scorers that can cause duplicate neighbor links+ in the HNSW graph. (Simon Cooper)+* GITHUB#16139: Fix IBSimilarity with DistributionSPL producing negative scores due tofloating-point rounding errors. (Prithvi S)diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.javaindex d37d980537b..7e05884f086 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java@@ -504,6 +504,12 @@ public class HnswGraphBuilder implements HnswBuilder {// here we don't need to lock, because there's no incoming link so no others is able to// discover this node such that no others will modify this neighbor array as wellif (isLinkRepair) {+ // there's a very small chance this is trying to add a duplicate node,+ // if the scoring function is estimated and the highest score is NOT+ // the identity function (which would already be filtered out by diversityCheck)+ if (contains(neighbors, cNode)) {+ continue;+ }neighbors.addOutOfOrder(cNode, cScore);} else {neighbors.addInOrder(cNode, cScore);@@ -513,6 +519,13 @@ public class HnswGraphBuilder implements HnswBuilder {return mask;}+ private static boolean contains(NeighborArray array, int node) {+ for (int i = 0; i < array.size(); i++) {+ if (array.nodes()[i] == node) return true;+ }+ return false;+ }+static void popToScratch(GraphBuilderKnnCollector candidates, NeighborArray scratch) {scratch.clear();int candidateCount = candidates.size();