Apache Lucene open-source search software
Allow `LogMergePolicy` to merge more than `mergeFactor` segments together when the merge is below the min merge size. (#14166)
Adrien Granddd76dc4973fd
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 5b7c49860da..6eb52ac0b9b 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -71,6 +71,10 @@ Improvementsindividual and bulk data retrieval overloads; avoid double buffering withslices. (Chris Hegarty)+* GITHUB#14166: Log(ByteSize|Doc)MergePolicy now allow merging more than+ mergeFactor segments together when the merge is below the min merge size.+ (Adrien Grand)+Optimizations---------------------diff --git a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.javaindex 881ae099d5f..b6dc9848c9d 100644--- a/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java+++ b/lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java@@ -636,6 +636,31 @@ public abstract class LogMergePolicy extends MergePolicy {mergeDocs += segmentDocs;}+ if (end - start >= mergeFactor+ && minMergeSize < maxMergeSize+ && mergeSize < minMergeSize+ && anyMerging == false) {+ // If the merge has mergeFactor segments but is still smaller than the min merged segment+ // size, keep packing candidate segments.+ while (end < 1 + upto) {+ final SegmentInfoAndLevel segLevel = levels.get(end);+ final SegmentCommitInfo info = segLevel.info;+ if (mergingSegments.contains(info)) {+ anyMerging = true;+ break;+ }+ long segmentSize = size(info, mergeContext);+ long segmentDocs = sizeDocs(info, mergeContext);+ if (mergeSize + segmentSize > minMergeSize || mergeDocs + segmentDocs > maxMergeDocs) {+ break;+ }++ mergeSize += segmentSize;+ mergeDocs += segmentDocs;+ end++;+ }+ }+if (anyMerging || end - start <= 1) {// skip: there is an ongoing merge at the current level or the computed merge has a single// segment and this merge policy doesn't do singleton mergesdiff --git a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.javaindex ea60f9b1e09..8c121c1f6a0 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestLogMergePolicy.java@@ -50,8 +50,13 @@ public class TestLogMergePolicy extends BaseMergePolicyTestCase {@Overrideprotected void assertMerge(MergePolicy policy, MergeSpecification merge) throws IOException {LogMergePolicy lmp = (LogMergePolicy) policy;+ MergeContext mockMergeContext = new MockMergeContext(SegmentCommitInfo::getDelCount);for (OneMerge oneMerge : merge.merges) {- assertTrue(oneMerge.segments.size() <= lmp.getMergeFactor());+ long mergeSize = 0;+ for (SegmentCommitInfo info : oneMerge.segments) {+ mergeSize += lmp.size(info, mockMergeContext);+ }+ assertTrue(mergeSize < lmp.minMergeSize || oneMerge.segments.size() <= lmp.getMergeFactor());}}