Apache Lucene open-source search software
Fix flakiness issues with TestTieredMergePolicy. (#13881)
Adrien Grandd9707609d3ed
diff --git a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.javaindex 702df660c44..2fb0c0783a2 100644--- a/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java+++ b/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java@@ -438,7 +438,10 @@ public class TieredMergePolicy extends MergePolicy {}// allowedSegCount may occasionally be less than segsPerTier// if segment sizes are below the floor size- allowedSegCount = Math.max(allowedSegCount, Math.max(segsPerTier, targetSearchConcurrency));+ allowedSegCount = Math.max(allowedSegCount, segsPerTier);+ // No need to merge if the total number of segments (including too big segments) is less than or+ // equal to the target search concurrency.+ allowedSegCount = Math.max(allowedSegCount, targetSearchConcurrency - tooBigCount);int allowedDocCount = getMaxAllowedDocs(totalMaxDoc, totalDelDocs);if (verbose(mergeContext) && tooBigCount > 0) {diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.javaindex eb24d964702..a2d678a3ec0 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java@@ -19,6 +19,7 @@ package org.apache.lucene.index;import java.io.IOException;import java.util.ArrayList;import java.util.Collections;+import java.util.Comparator;import java.util.HashMap;import java.util.List;import java.util.Locale;@@ -39,6 +40,8 @@ import org.apache.lucene.util.Version;public class TestTieredMergePolicy extends BaseMergePolicyTestCase {+ private record DocCountAndSizeInBytes(int docCount, long sizeInBytes) {}+@Overridepublic TieredMergePolicy mergePolicy() {return newTieredMergePolicy();@@ -54,7 +57,7 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {int totalDelCount = 0;int totalMaxDoc = 0;long totalBytes = 0;- List<Long> segmentSizes = new ArrayList<>();+ List<DocCountAndSizeInBytes> segmentSizes = new ArrayList<>();for (SegmentCommitInfo sci : infos) {totalDelCount += sci.getDelCount();totalMaxDoc += sci.info.maxDoc();@@ -62,10 +65,11 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {double liveRatio = 1 - (double) sci.getDelCount() / sci.info.maxDoc();long weightedByteSize = (long) (liveRatio * byteSize);totalBytes += weightedByteSize;- segmentSizes.add(weightedByteSize);+ segmentSizes.add(+ new DocCountAndSizeInBytes(sci.info.maxDoc() - sci.getDelCount(), weightedByteSize));minSegmentBytes = Math.min(minSegmentBytes, weightedByteSize);}- Collections.sort(segmentSizes);+ Collections.sort(segmentSizes, Comparator.comparingLong(DocCountAndSizeInBytes::sizeInBytes));final double delPercentage = 100.0 * totalDelCount / totalMaxDoc;assertTrue(@@ -78,7 +82,7 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {long levelSizeBytes = Math.max(minSegmentBytes, (long) (tmp.getFloorSegmentMB() * 1024 * 1024));long bytesLeft = totalBytes;double allowedSegCount = 0;- List<Long> biggestSegments = segmentSizes;+ List<DocCountAndSizeInBytes> biggestSegments = segmentSizes;if (biggestSegments.size() > tmp.getTargetSearchConcurrency() - 1) {biggestSegments =biggestSegments.subList(@@ -86,11 +90,18 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {biggestSegments.size());}// Allow whole segments for the targetSearchConcurrency-1 biggest segments- for (long size : biggestSegments) {- bytesLeft -= size;+ for (DocCountAndSizeInBytes size : biggestSegments) {+ bytesLeft -= size.sizeInBytes();allowedSegCount++;}+ int tooBigCount = 0;+ for (DocCountAndSizeInBytes size : segmentSizes) {+ if (size.sizeInBytes() >= maxMergedSegmentBytes / 2) {+ tooBigCount++;+ }+ }+// below we make the assumption that segments that reached the max segment// size divided by 2 don't need merging anymoreint mergeFactor = (int) Math.min(tmp.getSegmentsPerTier(), tmp.getMaxMergeAtOnce());@@ -105,39 +116,31 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {bytesLeft -= tmp.getSegmentsPerTier() * levelSizeBytes;levelSizeBytes = Math.min(levelSizeBytes * mergeFactor, maxMergedSegmentBytes / 2);}- allowedSegCount = Math.max(allowedSegCount, tmp.getSegmentsPerTier());+ // Allow at least a full tier in addition of the too big segments.+ allowedSegCount = Math.max(allowedSegCount, tooBigCount + tmp.getSegmentsPerTier());+ // Allow at least `targetSearchConcurrency` segments.+ allowedSegCount = Math.max(allowedSegCount, tmp.getTargetSearchConcurrency());- // It's ok to be over the allowed segment count if none of the most balanced merges are balanced- // enough- boolean hasBalancedMerges = false;- for (int i = 0; i < segmentSizes.size() - mergeFactor; ++i) {- long maxMergeSegmentSize = segmentSizes.get(i + mergeFactor - 1);- if (maxMergeSegmentSize >= maxMergedSegmentBytes / 2) {- break;- }- long totalMergeSize = 0;- for (int j = 0; j < i + mergeFactor; ++j) {- totalMergeSize += segmentSizes.get(j);- }- if (maxMergedSegmentBytes * 1.5 <= totalMergeSize) {- hasBalancedMerges = true;+ // It's ok to be over the allowed segment count if none of the merges are legal, because they+ // are either not balanced or because they exceed the max merged segment doc count.+ // We only check pairwise merges instead of every possible merge to keep things simple. If none+ // of the pairwise merges are legal, chances are high that no merge is legal.+ int maxDocsPerSegment = tmp.getMaxAllowedDocs(infos.totalMaxDoc(), totalDelCount);+ boolean hasLegalMerges = false;+ for (int i = 0; i < segmentSizes.size() - 1; ++i) {+ DocCountAndSizeInBytes size1 = segmentSizes.get(i);+ DocCountAndSizeInBytes size2 = segmentSizes.get(i + 1);+ long mergedSegmentSizeInBytes = size1.sizeInBytes() + size2.sizeInBytes();+ int mergedSegmentDocCount = size1.docCount() + size2.docCount();++ if (mergedSegmentSizeInBytes <= maxMergedSegmentBytes+ && size2.sizeInBytes() * 1.5 <= mergedSegmentSizeInBytes+ && mergedSegmentDocCount <= maxDocsPerSegment) {+ hasLegalMerges = true;break;}}- // There can be more segments if we can't merge docs because they are balanced between segments.- // At least the- // 2 smallest segments should be mergeable.- // should be 2 segments to merge- int maxDocsPerSegment = tmp.getMaxAllowedDocs(infos.totalMaxDoc(), totalDelCount);- List<Integer> segmentDocs =- infos.asList().stream()- .map(info -> info.info.maxDoc() - info.getDelCount())- .sorted()- .toList();- boolean eligibleDocsMerge =- segmentDocs.size() >= 2 && segmentDocs.get(0) + segmentDocs.get(1) < maxDocsPerSegment;-int numSegments = infos.asList().size();assertTrue(String.format(@@ -154,7 +157,7 @@ public class TestTieredMergePolicy extends BaseMergePolicyTestCase {delPercentage,tmp.getDeletesPctAllowed(),tmp.getTargetSearchConcurrency()),- numSegments <= allowedSegCount || hasBalancedMerges == false || eligibleDocsMerge == false);+ numSegments <= allowedSegCount || hasLegalMerges == false);}@Override