Skip to content

Apache Lucene open-source search software

Allow `LogMergePolicy` to merge more than `mergeFactor` segments together when the merge is below the min merge size. (#14166)

Adrien Granddd76dc4973fd
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5b7c49860da..6eb52ac0b9b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -71,6 +71,10 @@ Improvements
individual and bulk data retrieval overloads; avoid double buffering with
slices. (Chris Hegarty)
+* GITHUB#14166: Log(ByteSize|Doc)MergePolicy now allow merging more than
+ mergeFactor segments together when the merge is below the min merge size.
+ (Adrien Grand)
+
Optimizations
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
index 881ae099d5f..b6dc9848c9d 100644
--- a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
@@ -636,6 +636,31 @@ public abstract class LogMergePolicy extends MergePolicy {
mergeDocs += segmentDocs;
}
+ if (end - start >= mergeFactor
+ && minMergeSize < maxMergeSize
+ && mergeSize < minMergeSize
+ && anyMerging == false) {
+ // If the merge has mergeFactor segments but is still smaller than the min merged segment
+ // size, keep packing candidate segments.
+ while (end < 1 + upto) {
+ final SegmentInfoAndLevel segLevel = levels.get(end);
+ final SegmentCommitInfo info = segLevel.info;
+ if (mergingSegments.contains(info)) {
+ anyMerging = true;
+ break;
+ }
+ long segmentSize = size(info, mergeContext);
+ long segmentDocs = sizeDocs(info, mergeContext);
+ if (mergeSize + segmentSize > minMergeSize || mergeDocs + segmentDocs > maxMergeDocs) {
+ break;
+ }
+
+ mergeSize += segmentSize;
+ mergeDocs += segmentDocs;
+ end++;
+ }
+ }
+
if (anyMerging || end - start <= 1) {
// skip: there is an ongoing merge at the current level or the computed merge has a single
// segment and this merge policy doesn't do singleton merges
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
index ea60f9b1e09..8c121c1f6a0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java
@@ -50,8 +50,13 @@ public class TestLogMergePolicy extends BaseMergePolicyTestCase {
@Override
protected void assertMerge(MergePolicy policy, MergeSpecification merge) throws IOException {
LogMergePolicy lmp = (LogMergePolicy) policy;
+ MergeContext mockMergeContext = new MockMergeContext(SegmentCommitInfo::getDelCount);
for (OneMerge oneMerge : merge.merges) {
- assertTrue(oneMerge.segments.size() <= lmp.getMergeFactor());
+ long mergeSize = 0;
+ for (SegmentCommitInfo info : oneMerge.segments) {
+ mergeSize += lmp.size(info, mockMergeContext);
+ }
+ assertTrue(mergeSize < lmp.minMergeSize || oneMerge.segments.size() <= lmp.getMergeFactor());
}
}