Apache Lucene open-source search software
Add estimatedByteSizes to merges kicked off by IndexWriter.addIndexes(CodecReader[]) (#15120)
Craig Perkinsbfdbac385322
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 6fd5516742f..99feba7a0f5 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -79,6 +79,8 @@ Bug Fixes* GITHUB#14847: Allow Faiss vector format to index >2GB of vectors per-field per-segment by using MemorySegment APIs(instead of ByteBuffer) to copy bytes to native memory. (Kaival Parikh)+* GITHUB#15120: Add estimatedByteSizes to merges kicked off by IndexWriter.addIndexes(CodecReader[]) (Craig Perkins)+Changes in Runtime Behavior---------------------* GITHUB#14187: The query cache is now disabled by default. (Adrien Grand)diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.javaindex 1224bebc9d0..a8c63c3000e 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java@@ -21,6 +21,7 @@ import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;import java.io.Closeable;import java.io.IOException;+import java.io.UncheckedIOException;import java.time.Instant;import java.util.ArrayDeque;import java.util.ArrayList;@@ -3285,6 +3286,11 @@ public class IndexWriter}public void registerMerge(MergePolicy.OneMerge merge) {+ try {+ addEstimatedBytesToMerge(merge);+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }synchronized (IndexWriter.this) {pendingAddIndexesMerges.add(merge);}@@ -4777,6 +4783,21 @@ public class IndexWritercloseMergeReaders(merge, true, false);}+ /** Compute {@code estimatedMergeBytes} and {@code totalMergeBytes} for a merge. */+ void addEstimatedBytesToMerge(MergePolicy.OneMerge merge) throws IOException {+ assert merge.estimatedMergeBytes == 0;+ assert merge.totalMergeBytes == 0;+ for (SegmentCommitInfo info : merge.segments) {+ if (info.info.maxDoc() > 0) {+ final int delCount = numDeletedDocs(info);+ assert delCount <= info.info.maxDoc();+ final double delRatio = ((double) delCount) / info.info.maxDoc();+ merge.estimatedMergeBytes += (long) (info.sizeInBytes() * (1.0 - delRatio));+ merge.totalMergeBytes += info.sizeInBytes();+ }+ }+ }+/*** Checks whether this merge involves any segments already participating in a merge. If not, this* merge is "registered", meaning we record that its segments are now participating in a merge,@@ -4868,17 +4889,7 @@ public class IndexWritermergingSegments.add(info);}- assert merge.estimatedMergeBytes == 0;- assert merge.totalMergeBytes == 0;- for (SegmentCommitInfo info : merge.segments) {- if (info.info.maxDoc() > 0) {- final int delCount = numDeletedDocs(info);- assert delCount <= info.info.maxDoc();- final double delRatio = ((double) delCount) / info.info.maxDoc();- merge.estimatedMergeBytes += (long) (info.sizeInBytes() * (1.0 - delRatio));- merge.totalMergeBytes += info.sizeInBytes();- }- }+ addEstimatedBytesToMerge(merge);// Merge is now registeredmerge.registerDone = true;diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.javaindex e3f47be432a..a1c887e93b2 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMerging.java@@ -461,4 +461,35 @@ public class TestIndexWriterMerging extends LuceneTestCase {directory.close();}++ public void testAddEstimatedBytesToMerge() throws IOException {+ try (Directory dir = newDirectory();+ IndexWriter writer =+ new IndexWriter(+ dir,+ newIndexWriterConfig(new MockAnalyzer(random()))+ .setMergePolicy(NoMergePolicy.INSTANCE))) {++ Document doc = new Document();+ doc.add(newTextField("field", "content", Field.Store.YES));++ for (int i = 0; i < 10; i++) {++ writer.addDocument(doc);+ }+ writer.flush();++ // Create a merge with the segments+ SegmentInfos segmentInfos = writer.cloneSegmentInfos();+ MergePolicy.OneMerge merge = new MergePolicy.OneMerge(segmentInfos.asList());++ writer.addEstimatedBytesToMerge(merge);++ assertTrue(merge.estimatedMergeBytes > 0);++ assertTrue(merge.totalMergeBytes > 0);++ assertTrue(merge.estimatedMergeBytes <= merge.totalMergeBytes);+ }+ }}