Apache Lucene open-source search software
Check if merge is aborted before executing file integrity checks (#16264)
Tanguy Lerouxa34cefd33cbd
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex ad7b71d0e74..fa21530f244 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -410,6 +410,9 @@ ImprovementsDiversifiedTopDocsCollector, removing usages of the deprecated IndexSearcher#search(Query, Collector).(Luca Cavanna)+* GITHUB#16264: Check if merge is aborted before executing file integrity checks to avoid+ costly full-file checksums on segments when the merge has already been cancelled. (Tanguy Leroux)+Optimizations---------------------* GITHUB#16111: Optimize FieldExistsQuery.count() when all docs have the field. (Prithvi S)diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.javaindex 0c26af89152..b75cce9b1a5 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java@@ -132,6 +132,7 @@ public abstract class DocValuesConsumer implements Closeable {public void merge(MergeState mergeState) throws IOException {for (DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {if (docValuesProducer != null) {+ mergeState.checkAborted();docValuesProducer.checkIntegrity();}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.javaindex 5443585ab97..537410f8569 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java@@ -80,6 +80,7 @@ public abstract class FieldsConsumer implements Closeable {final int maxDoc = mergeState.maxDocs[readerIndex];if (f != null) {+ mergeState.checkAborted();f.checkIntegrity();slices.add(new ReaderSlice(docBase, maxDoc, readerIndex));fields.add(f);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.javaindex d2215e99808..d1e2b2ee1c4 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/KnnVectorsWriter.java@@ -113,6 +113,7 @@ public abstract class KnnVectorsWriter implements Accountable, Closeable {KnnVectorsReader reader = mergeState.knnVectorsReaders[i];assert reader != null || mergeState.fieldInfos[i].hasVectorValues() == false;if (reader != null) {+ mergeState.checkAborted();reader.checkIntegrity();}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.javaindex 7147e6b3327..fedd9584db3 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java@@ -66,6 +66,7 @@ public abstract class NormsConsumer implements Closeable {public void merge(MergeState mergeState) throws IOException {for (NormsProducer normsProducer : mergeState.normsProducers) {if (normsProducer != null) {+ mergeState.checkAborted();normsProducer.checkIntegrity();}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.javaindex 9d756e36d6f..4b4f2a01773 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/PointsWriter.java@@ -222,6 +222,7 @@ public abstract class PointsWriter implements Closeable {// check each incoming readerfor (PointsReader reader : mergeState.pointsReaders) {if (reader != null) {+ mergeState.checkAborted();reader.checkIntegrity();}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.javaindex a9f9ce464a3..f5eb9e4c113 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java@@ -130,6 +130,7 @@ public abstract class StoredFieldsWriter implements Closeable, Accountable {List<StoredFieldsMergeSub> subs = new ArrayList<>();for (int i = 0; i < mergeState.storedFieldsReaders.length; i++) {StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[i];+ mergeState.checkAborted();storedFieldsReader.checkIntegrity();subs.add(new StoredFieldsMergeSub(diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.javaindex 45dcc504186..8c9b11dd352 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java@@ -200,6 +200,7 @@ public abstract class TermVectorsWriter implements Closeable, Accountable {for (int i = 0; i < mergeState.termVectorsReaders.length; i++) {TermVectorsReader reader = mergeState.termVectorsReaders[i];if (reader != null) {+ mergeState.checkAborted();reader.checkIntegrity();}subs.add(new TermVectorsMergeSub(mergeState.docMaps[i], reader, mergeState.maxDocs[i]));diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.javaindex 7f56d855473..eaea9d45bf5 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90PointsWriter.java@@ -204,6 +204,7 @@ public class Lucene90PointsWriter extends PointsWriter {}for (PointsReader reader : mergeState.pointsReaders) {if (reader != null) {+ mergeState.checkAborted();reader.checkIntegrity();}}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.javaindex 39009b1eb56..4d2b2f25a14 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingStoredFieldsWriter.java@@ -601,6 +601,7 @@ public final class Lucene90CompressingStoredFieldsWriter extends StoredFieldsWrinew ArrayList<>(mergeState.storedFieldsReaders.length);for (int i = 0; i < mergeState.storedFieldsReaders.length; i++) {final StoredFieldsReader reader = mergeState.storedFieldsReaders[i];+ mergeState.checkAborted();reader.checkIntegrity();MergeStrategy mergeStrategy = getMergeStrategy(mergeState, matchingReaders, i);if (mergeStrategy == MergeStrategy.VISITOR) {diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.javaindex 00eddc03ee8..3fe9f2a47ef 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/compressing/Lucene90CompressingTermVectorsWriter.java@@ -898,6 +898,7 @@ public final class Lucene90CompressingTermVectorsWriter extends TermVectorsWritefor (int i = 0; i < numReaders; i++) {final TermVectorsReader reader = mergeState.termVectorsReaders[i];if (reader != null) {+ mergeState.checkAborted();reader.checkIntegrity();}final boolean bulkMerge = canPerformBulkMerge(mergeState, matchingReaders, i);diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.javaindex bb67ec618fd..85df2414aa3 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldMergeState.java@@ -69,7 +69,8 @@ final class PerFieldMergeState {in.maxDocs,in.infoStream,in.intraMergeTaskExecutor,- in.needsIndexSort);+ in.needsIndexSort,+ in.oneMerge);}private static class FilterFieldInfos extends FieldInfos {diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.javaindex bf806dd08c0..68ddd1611b9 100644--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java@@ -3521,7 +3521,8 @@ public class IndexWritertrackingDir,globalFieldNumberMap,context,- intraMergeExecutor);+ intraMergeExecutor,+ merge);try {if (!merger.shouldMerge()) {return;@@ -5322,7 +5323,8 @@ public class IndexWriterdirWrapper,globalFieldNumberMap,context,- intraMergeExecutor);+ intraMergeExecutor,+ merge);MergeState mergeState = merger.mergeState;MergeState.DocMap[] docMaps;try {diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.javaindex 838699215f0..fafdafbb0a4 100644--- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java+++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java@@ -91,14 +91,24 @@ public class MergeState {/** Indicates if the index needs to be sorted * */public boolean needsIndexSort;+ /**+ * The merge that this state is associated with, or {@code null} if this merge state is not+ * associated with an {@link IndexWriter} merge (e.g. for addIndexes).+ *+ * @lucene.internal+ */+ public final MergePolicy.OneMerge oneMerge;+/** Sole constructor. */MergeState(List<CodecReader> readers,SegmentInfo segmentInfo,InfoStream infoStream,- Executor intraMergeTaskExecutor)+ Executor intraMergeTaskExecutor,+ MergePolicy.OneMerge oneMerge)throws IOException {verifyIndexSort(readers, segmentInfo);+ this.oneMerge = oneMerge;this.infoStream = infoStream;int numReaders = readers.size();this.intraMergeTaskExecutor = intraMergeTaskExecutor;@@ -230,6 +240,16 @@ public class MergeState {}}+ /**+ * Checks if the merge has been aborted, throwing {@link MergePolicy.MergeAbortedException} if so.+ * This is a no-op if this merge state is not associated with an {@link IndexWriter} merge.+ */+ public void checkAborted() throws MergePolicy.MergeAbortedException {+ if (oneMerge != null) {+ oneMerge.checkAborted();+ }+ }+private static void verifyIndexSort(List<CodecReader> readers, SegmentInfo segmentInfo) {Sort indexSort = segmentInfo.getIndexSort();if (indexSort == null) {@@ -284,7 +304,8 @@ public class MergeState {int[] maxDocs,InfoStream infoStream,Executor intraMergeTaskExecutor,- boolean needsIndexSort) {+ boolean needsIndexSort,+ MergePolicy.OneMerge oneMerge) {this.docMaps = docMaps;this.segmentInfo = segmentInfo;this.mergeFieldInfos = mergeFieldInfos;@@ -301,5 +322,6 @@ public class MergeState {this.infoStream = infoStream;this.intraMergeTaskExecutor = intraMergeTaskExecutor;this.needsIndexSort = needsIndexSort;+ this.oneMerge = oneMerge;}}diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.javaindex 8dab44c1c8e..321780406ad 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java@@ -60,18 +60,20 @@ final class SegmentMerger {Directory dir,FieldInfos.FieldNumbers fieldNumbers,IOContext context,- Executor intraMergeTaskExecutor)+ Executor intraMergeTaskExecutor,+ MergePolicy.OneMerge oneMerge)throws IOException {if (context.context() != IOContext.Context.MERGE) {throw new IllegalArgumentException("IOContext.context should be MERGE; got: " + context.context());}- mergeState = new MergeState(readers, segmentInfo, infoStream, intraMergeTaskExecutor);mergeStateCreationThread = Thread.currentThread();directory = dir;this.codec = segmentInfo.getCodec();this.context = context;this.fieldInfosBuilder = new FieldInfos.Builder(fieldNumbers);+ this.mergeState =+ new MergeState(readers, segmentInfo, infoStream, intraMergeTaskExecutor, oneMerge);Version minVersion = Version.LATEST;for (CodecReader reader : readers) {Version leafMinVersion = reader.getMetaData().minVersion();diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.javaindex 5f1fc6a2267..e00a1d8c96a 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestMergedVectorValues.java@@ -40,7 +40,7 @@ public class TestMergedVectorValues extends LuceneTestCase {MergeState state =new MergeState(null, null, null, null, null, null, null, null, null, null, null, null, null, null,- null, false);+ null, false, null);// Run the testByteVectorValues values =@@ -68,7 +68,7 @@ public class TestMergedVectorValues extends LuceneTestCase {MergeState state =new MergeState(null, null, null, null, null, null, null, null, null, null, null, null, null, null,- null, false);+ null, false, null);// Run the testFloatVectorValues values =diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.javaindex 13384fcb892..2151b24157c 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java@@ -239,7 +239,8 @@ public class TestDoc extends LuceneTestCase {trackingDir,new FieldInfos.FieldNumbers(null, null),context,- new SameThreadExecutorService());+ new SameThreadExecutorService(),+ null);merger.merge();merger.cleanupMerge();diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.javaindex 762e934a636..56dd6ed2e2e 100644--- a/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java+++ b/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java@@ -107,7 +107,8 @@ public class TestSegmentMerger extends LuceneTestCase {mergedDir,new FieldInfos.FieldNumbers(null, null),newIOContext(random(), IOContext.merge(new MergeInfo(-1, -1, false, -1))),- new SameThreadExecutorService());+ new SameThreadExecutorService(),+ null);MergeState mergeState = merger.merge();merger.cleanupMerge();int docsMerged = mergeState.segmentInfo.maxDoc();