Skip to content

Apache Lucene open-source search software

Increase the default number of merge threads. (#13294)

Adrien Grande19238a7bde1
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 74bfa5f4917..1b68f73b8f6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -180,6 +180,9 @@ Changes in Runtime Behavior
* GITHUB13293: Auto I/O throttling is now disabled by default on ConcurrentMergeScheduler.
(Adrien Grand)
+* GITHUB#13293: ConcurrentMergeScheduler now allows up to 50% of the threads of the host to be used
+ for merging. (Adrien Grand)
+
Changes in Backwards Compatibility Policy
-----------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
index f27308f2aca..d0e79375ea6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
@@ -181,7 +181,14 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
Throwable ignored) {
}
- maxThreadCount = Math.max(1, Math.min(4, coreCount / 2));
+ // If you are indexing at full throttle, how many merge threads do you need to keep up? It
+ // depends: for most data structures, merging is cheaper than indexing/flushing, but for knn
+ // vectors, merges can require about as much work as the initial indexing/flushing. Plus
+ // documents are indexed/flushed only once, but may be merged multiple times.
+ // Here, we assume an intermediate scenario where merging requires about as much work as
+ // indexing/flushing overall, so we give half the core count to merges.
+
+ maxThreadCount = Math.max(1, coreCount / 2);
maxMergeCount = maxThreadCount + 5;
}
}