Apache Lucene open-source search software
Fix HNSW InfoStream duplicate times and add per-chunk completion logging (#15978)
Prithvi6d7bf0320d3a
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 8898787ba37..6045f128242 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -156,6 +156,10 @@ OptimizationsBug Fixes---------------------+* GITHUB#15967: Fix IndexWriter's HNSW InfoStream progress during concurrent merges to show elapsed+ time since merge start instead of duplicate incremental/total times. Add per-chunk completion+ logging with sub-millisecond precision to aid merge concurrency debugging. (Prithvi S)+* GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zerowhen all values are identical. (Mike Sokolov)diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.javaindex 9fbf69558e5..f1a4666fd7e 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswConcurrentMergeBuilder.java@@ -22,8 +22,10 @@ import static org.apache.lucene.util.hnsw.HnswGraphBuilder.HNSW_COMPONENT;import java.io.IOException;import java.util.ArrayList;import java.util.List;+import java.util.Locale;import java.util.concurrent.Callable;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;import java.util.concurrent.locks.Lock;import org.apache.lucene.internal.hppc.IntHashSet;import org.apache.lucene.search.TaskExecutor;@@ -76,11 +78,17 @@ public class HnswConcurrentMergeBuilder implements HnswBuilder {if (frozen) {throw new IllegalStateException("graph has already been built");}+ long mergeStartTimeNs = System.nanoTime();if (infoStream.isEnabled(HNSW_COMPONENT)) {infoStream.message(HNSW_COMPONENT,"build graph from " + maxOrd + " vectors, with " + workers.length + " workers");}+ AtomicLong cumulativeWorkTimeNs = new AtomicLong();+ for (ConcurrentMergeWorker worker : workers) {+ worker.setMergeStartTimeNs(mergeStartTimeNs);+ worker.setCumulativeWorkTimeNs(cumulativeWorkTimeNs);+ }List<Callable<Void>> futures = new ArrayList<>();for (int i = 0; i < workers.length; i++) {int finalI = i;@@ -91,6 +99,20 @@ public class HnswConcurrentMergeBuilder implements HnswBuilder {});}taskExecutor.invokeAll(futures);+ if (infoStream.isEnabled(HNSW_COMPONENT)) {+ double wallClockMs = (System.nanoTime() - mergeStartTimeNs) / 1_000_000.0;+ double totalWorkerMs = cumulativeWorkTimeNs.get() / 1_000_000.0;+ double effectiveConcurrency = wallClockMs > 0 ? totalWorkerMs / wallClockMs : 0;+ infoStream.message(+ HNSW_COMPONENT,+ String.format(+ Locale.ROOT,+ "merge completed: %d vectors, %.2f ms wall clock, %.2f ms cumulative worker time, %.2fx effective concurrency",+ maxOrd,+ wallClockMs,+ totalWorkerMs,+ effectiveConcurrency));+ }return getCompletedGraph();}diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.javaindex c5fba5c88d4..e42b69ddb27 100644--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java@@ -26,7 +26,7 @@ import java.util.List;import java.util.Locale;import java.util.Objects;import java.util.SplittableRandom;-import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicLong;import java.util.concurrent.locks.Lock;import org.apache.lucene.internal.hppc.IntHashSet;import org.apache.lucene.search.KnnCollector;@@ -85,6 +85,29 @@ public class HnswGraphBuilder implements HnswBuilder {protected InfoStream infoStream = InfoStream.getDefault();protected boolean frozen;+ /**+ * Merge-level start time in nanoseconds. When set, the periodic progress prints (every 10K+ * vectors) show elapsed time since the overall merge began rather than since the current chunk+ * began. A value of -1 means not set (non-concurrent path).+ */+ private long mergeStartTimeNs = -1;++ /**+ * Shared accumulator for total worker time across all concurrent merge workers. Each chunk's+ * elapsed time is added here so that effective concurrency can be computed at merge end.+ */+ private AtomicLong cumulativeWorkTimeNs;++ /** Set the merge-level start time so progress prints show time since merge began. */+ void setMergeStartTimeNs(long mergeStartTimeNs) {+ this.mergeStartTimeNs = mergeStartTimeNs;+ }++ /** Set the shared accumulator for tracking cumulative worker time across concurrent chunks. */+ void setCumulativeWorkTimeNs(AtomicLong cumulativeWorkTimeNs) {+ this.cumulativeWorkTimeNs = cumulativeWorkTimeNs;+ }+public static HnswGraphBuilder create(RandomVectorScorerSupplier scorerSupplier, int M, int beamWidth, long seed)throws IOException {@@ -204,16 +227,30 @@ public class HnswGraphBuilder implements HnswBuilder {if (frozen) {throw new IllegalStateException("This HnswGraphBuilder is frozen and cannot be updated");}- long start = System.nanoTime(), t = start;- if (infoStream.isEnabled(HNSW_COMPONENT)) {- infoStream.message(HNSW_COMPONENT, "addVectors [" + minOrd + " " + maxOrd + ")");- }+ long startNs = System.nanoTime();+ long progressStartNs = mergeStartTimeNs != -1 ? mergeStartTimeNs : startNs;for (int node = minOrd; node < maxOrd; node++) {addGraphNode(node);if ((node % 10000 == 0) && infoStream.isEnabled(HNSW_COMPONENT)) {- t = printGraphBuildStatus(node, start, t);+ printGraphBuildStatus(node, progressStartNs);}}+ long chunkedElapsedNs = System.nanoTime() - startNs;+ if (cumulativeWorkTimeNs != null) {+ cumulativeWorkTimeNs.addAndGet(chunkedElapsedNs);+ }+ if (infoStream.isEnabled(HNSW_COMPONENT)) {+ double elapsedMs = chunkedElapsedNs / 1_000_000.0;+ infoStream.message(+ HNSW_COMPONENT,+ String.format(+ Locale.ROOT,+ "addVectors [%d %d): %d vectors in %.2f ms",+ minOrd,+ maxOrd,+ maxOrd - minOrd,+ elapsedMs));+ }}private void addVectors(int maxOrd) throws IOException {@@ -339,17 +376,10 @@ public class HnswGraphBuilder implements HnswBuilder {addGraphNodeInternal(node, scorer, eps0);}- private long printGraphBuildStatus(int node, long start, long t) {- long now = System.nanoTime();+ private void printGraphBuildStatus(int node, long startNs) {+ double elapsedMs = (System.nanoTime() - startNs) / 1_000_000.0;infoStream.message(- HNSW_COMPONENT,- String.format(- Locale.ROOT,- "built %d in %d/%d ms",- node,- TimeUnit.NANOSECONDS.toMillis(now - t),- TimeUnit.NANOSECONDS.toMillis(now - start)));- return now;+ HNSW_COMPONENT, String.format(Locale.ROOT, "built %d in %.2f ms", node, elapsedMs));}void addDiverseNeighbors(