Skip to content

Apache Lucene open-source search software

HNSW - don't add duplicates when repairing graph links (#16728)

Simon Cooper4eb488d0f9e8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index bbae22bf84e..c403a8bdb8f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -632,6 +632,9 @@ Bug Fixes
* GITHUB#16577: Fix incorrect position increment generated by FixedShingleFilter. (Kishor Kharbas)
+* GITHUB#16728: Handle edge cases with inaccurate scorers that can cause duplicate neighbor links
+ in the HNSW graph. (Simon Cooper)
+
* GITHUB#16139: Fix IBSimilarity with DistributionSPL producing negative scores due to
floating-point rounding errors. (Prithvi S)
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
index d37d980537b..7e05884f086 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
@@ -504,6 +504,12 @@ public class HnswGraphBuilder implements HnswBuilder {
// here we don't need to lock, because there's no incoming link so no others is able to
// discover this node such that no others will modify this neighbor array as well
if (isLinkRepair) {
+ // there's a very small chance this is trying to add a duplicate node,
+ // if the scoring function is estimated and the highest score is NOT
+ // the identity function (which would already be filtered out by diversityCheck)
+ if (contains(neighbors, cNode)) {
+ continue;
+ }
neighbors.addOutOfOrder(cNode, cScore);
} else {
neighbors.addInOrder(cNode, cScore);
@@ -513,6 +519,13 @@ public class HnswGraphBuilder implements HnswBuilder {
return mask;
}
+ private static boolean contains(NeighborArray array, int node) {
+ for (int i = 0; i < array.size(); i++) {
+ if (array.nodes()[i] == node) return true;
+ }
+ return false;
+ }
+
static void popToScratch(GraphBuilderKnnCollector candidates, NeighborArray scratch) {
scratch.clear();
int candidateCount = candidates.size();