Skip to content

Apache Lucene open-source search software

Fix race and improve performance of HNSW graph building (#15590)

Viliam Durina3b8a0cdbcc6e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4cf189855e0..272dd0210c6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -238,6 +238,8 @@ Bug Fixes
* GITHUB#15668: Fix UnsupportedOperationException in WeightedSpanTermExtractor by
implementing getFieldInfos in DelegatingLeafReader. (Simranjeet Singh)
+* GITHUB#15590: Fix race and improve performance of HNSW graph building (Viliam Durina)
+
* GITHUB#15702: Fix Potential NullPointException in BytesRefHash initialization#15702 (tyronecai)
* GITHUB#15731: Fix tessellator failure when a hole is touching another hole in the middle of a segment. In this case
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
index f2ceda091be..c8d73723746 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/OnHeapHnswGraph.java
@@ -21,6 +21,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
import org.apache.lucene.internal.hppc.IntArrayList;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.ArrayUtil;
@@ -63,7 +64,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
private int upto;
private NeighborArray cur;
- private volatile long graphRamBytesUsed;
+ private final LongAdder graphRamBytesUsed = new LongAdder();
/**
* ctor
@@ -83,7 +84,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
numNodes = INIT_SIZE;
}
this.graph = new NeighborArray[numNodes][];
- this.graphRamBytesUsed = RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(graph);
+ this.graphRamBytesUsed.add(RAM_BYTES_USED + RamUsageEstimator.shallowSizeOf(graph));
}
/**
@@ -160,27 +161,14 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
graph[node] = ArrayUtil.growExact(graph[node], level + 1);
}
- if (level == 0) {
- graph[node][level] =
- new NeighborArray(
- nsize0,
- true,
- l -> {
- assert l > 0;
- long bytesUsed = graphRamBytesUsed;
- graphRamBytesUsed = bytesUsed + l;
- });
- } else {
- graph[node][level] =
- new NeighborArray(
- nsize,
- true,
- l -> {
- assert l > 0;
- long bytesUsed = graphRamBytesUsed;
- graphRamBytesUsed = bytesUsed + l;
- });
- }
+ graph[node][level] =
+ new NeighborArray(
+ level == 0 ? nsize0 : nsize,
+ true,
+ l -> {
+ assert l > 0;
+ graphRamBytesUsed.add(l);
+ });
maxNodeId.accumulateAndGet(node, Math::max);
}
@@ -289,7 +277,6 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
}
}
- @SuppressWarnings({"unchecked", "rawtypes"})
private void generateLevelToNodes() {
if (lastFreezeSize == size()) {
return;
@@ -323,7 +310,7 @@ public final class OnHeapHnswGraph extends HnswGraph implements Accountable {
*/
@Override
public long ramBytesUsed() {
- return graphRamBytesUsed;
+ return graphRamBytesUsed.sum();
}
@Override