Skip to content

Apache Lucene open-source search software

Run merge-on-full-flush even though no changes got flushed. (#12549)

Adrien Grand3d4082ca3dcf
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index aef273348fd..4e235ae7d8e 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -3680,7 +3680,7 @@ public class IndexWriter
// merge completes which would otherwise have
// removed the files we are now syncing.
deleter.incRef(toCommit.files(false));
- if (anyChanges && maxCommitMergeWaitMillis > 0) {
+ if (maxCommitMergeWaitMillis > 0) {
// we can safely call preparePointInTimeMerge since writeReaderPool(true) above
// wrote all
// necessary files to disk and checkpointed them.
@@ -4232,7 +4232,6 @@ public class IndexWriter
flushSuccess = true;
} finally {
assert Thread.holdsLock(fullFlushLock);
- ;
docWriter.finishFullFlush(flushSuccess);
processEvents(false);
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 8221d5399ab..6e993a17473 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -654,7 +654,8 @@ public class TestDirectoryReaderReopen extends LuceneTestCase {
public void testOverDecRefDuringReopen() throws Exception {
MockDirectoryWrapper dir = newMockDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ IndexWriterConfig iwc =
+ new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setCodec(TestUtil.getDefaultCodec());
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index 4b8ce1f70e2..bd52a84f7e6 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -519,11 +519,10 @@ public class TestIndexWriter extends LuceneTestCase {
doc.add(newField("field", "aaa", customType));
for (int i = 0; i < 19; i++) writer.addDocument(doc);
writer.flush(false, true);
- writer.close();
- SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
// Since we flushed w/o allowing merging we should now
// have 10 segments
- assertEquals(10, sis.size());
+ assertEquals(10, writer.getSegmentCount());
+ writer.close();
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index 58f844247d6..468c91fc6ec 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -1314,7 +1314,7 @@ public class TestIndexWriterDelete extends LuceneTestCase {
w.addDocument(doc);
w.close();
- iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE);
iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
w = new IndexWriter(d, iwc);
IndexReader r = DirectoryReader.open(w, false, false);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
index 8eabef5dae5..727f9ce3bd5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
@@ -394,14 +394,13 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
.setMaxFullFlushMergeWaitMillis(Integer.MAX_VALUE);
IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
- writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
+ // No changes. Refresh doesn't trigger a merge.
DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
assertEquals(5, unmergedReader.leaves().size());
unmergedReader.close();
- TestIndexWriter.addDoc(writerWithMergePolicy);
- writerWithMergePolicy.commit(); // Doc added, do merge on commit.
+ writerWithMergePolicy.commit(); // Do merge on commit.
assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
DirectoryReader mergedReader = DirectoryReader.open(writerWithMergePolicy);
@@ -410,8 +409,8 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
try (IndexReader reader = DirectoryReader.open(writerWithMergePolicy)) {
IndexSearcher searcher = new IndexSearcher(reader);
- assertEquals(6, reader.numDocs());
- assertEquals(6, searcher.count(new MatchAllDocsQuery()));
+ assertEquals(5, reader.numDocs());
+ assertEquals(5, searcher.count(new MatchAllDocsQuery()));
}
writerWithMergePolicy.close();
@@ -445,16 +444,14 @@ public class TestIndexWriterMergePolicy extends LuceneTestCase {
.setIndexWriterEventListener(eventListener);
IndexWriter writerWithMergePolicy = new IndexWriter(dir, iwc);
- writerWithMergePolicy.commit(); // No changes. Commit doesn't trigger a merge.
+ // No changes. Refresh doesn't trigger a merge.
DirectoryReader unmergedReader = DirectoryReader.open(writerWithMergePolicy);
assertEquals(5, unmergedReader.leaves().size());
unmergedReader.close();
- TestIndexWriter.addDoc(writerWithMergePolicy);
-
assertFalse(eventListener.isEventsRecorded());
- writerWithMergePolicy.commit(); // Doc added, do merge on commit.
+ writerWithMergePolicy.commit(); // Do merge on commit.
assertEquals(1, writerWithMergePolicy.getSegmentCount()); //
assertTrue(eventListener.isEventsRecorded());
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
index 8554dd3f40a..02600f5d59d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPendingSoftDeletes.java
@@ -263,6 +263,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
newIndexWriterConfig()
.setSoftDeletesField("_soft_deletes")
.setMaxBufferedDocs(3) // make sure we write one segment
+ .setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Document doc = new Document();
doc.add(new StringField("id", "1", Field.Store.YES));
@@ -331,6 +332,7 @@ public class TestPendingSoftDeletes extends TestPendingDeletes {
newIndexWriterConfig()
.setSoftDeletesField("_soft_deletes")
.setMaxBufferedDocs(3) // make sure we write one segment
+ .setMergePolicy(NoMergePolicy.INSTANCE) // prevent deletes from triggering merges
.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Document doc = new Document();
doc.add(new StringField("id", "1", Field.Store.YES));
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
index bc1f95e0b58..a0782159d87 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/MockDirectoryWrapper.java
@@ -46,6 +46,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.NoDeletionPolicy;
import org.apache.lucene.index.SegmentInfos;
+import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
@@ -970,7 +971,12 @@ public class MockDirectoryWrapper extends BaseDirectoryWrapper {
DirectoryReader ir1 = DirectoryReader.open(this);
int numDocs1 = ir1.numDocs();
ir1.close();
- new IndexWriter(this, new IndexWriterConfig(null)).close();
+ // Use a serial merge scheduler, otherwise merges may be scheduled on a different thread
+ // and will deadlock since the lock on `this` is already taken by close(), so
+ // createOutput() won't be able to take it.
+ new IndexWriter(
+ this, new IndexWriterConfig(null).setMergeScheduler(new SerialMergeScheduler()))
+ .close();
DirectoryReader ir2 = DirectoryReader.open(this);
int numDocs2 = ir2.numDocs();
ir2.close();