Skip to content

Apache Lucene open-source search software

Fix join set degeneration on low-degree nodes (#16435)

Mayya Sharipova5fa57e79659e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 44924919b4f..90d5a039325 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -398,6 +398,11 @@ Optimizations
Bug Fixes
---------------------
+* GITHUB#16251: Fix UpdateGraphsUtils#computeJoinSet to not request more coverage for a node than
+ the node's degree. Previously nodes with a degree of 1 were always forced into the join set,
+ which made the join set degenerate to the whole graph on hub-and-spoke shaped HNSW graphs.
+ (Mayya Sharipova)
+
* GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward)
* GITHUB#16378: Accumulate join Total/Avg scores in double precision in
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java
index ce103952d50..4a551a38f3f 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/UpdateGraphsUtils.java
@@ -49,7 +49,7 @@ public class UpdateGraphsUtils {
for (int v = 0; v < size; v++) {
graph.seek(0, v);
int degree = graph.neighborCount();
- k = degree < 9 ? 2 : Math.ceilDiv(degree, 4);
+ k = coverage(degree);
gExit += k;
int gain = k + degree;
heap.push(encode(gain, v));
@@ -67,7 +67,7 @@ public class UpdateGraphsUtils {
for (int u = graph.nextNeighbor(); u != NO_MORE_DOCS; u = graph.nextNeighbor()) {
ns[i++] = u;
}
- k = degree < 9 ? 2 : Math.ceilDiv(degree, 4);
+ k = coverage(degree);
if (stale[v]) { // if stale, recalculate gain
int newGain = Math.max(0, k - counts[v]);
for (int u : ns) {
@@ -101,6 +101,22 @@ public class UpdateGraphsUtils {
return j;
}
+ /**
+ * How many times a node needs to be covered by the nodes of the join set before we consider it
+ * covered.
+ *
+ * <p>A node's neighbours are the only nodes that can cover it, so a node can never be covered
+ * more than {@code degree} times. Requesting a bigger coverage than that would make the coverage
+ * unsatisfiable, and the node would be forced to join the join set itself. This happens on
+ * degenerate graphs (e.g. hub-and-spoke shapes produced by highly duplicated or low dimensional
+ * vectors) where most of the nodes have a degree of 1: without clamping, every leaf ends up in
+ * the join set, and the join set degenerates to the whole graph.
+ */
+ private static int coverage(int degree) {
+ int k = degree < 9 ? 2 : Math.ceilDiv(degree, 4);
+ return Math.min(k, degree);
+ }
+
private static long encode(int value1, int value2) {
return (((long) -value1) << 32) | (value2 & 0xFFFFFFFFL);
}