Skip to content

Apache Lucene open-source search software

Disable intra-merge parallelism for all structures but kNN vectors (#13799)

Benjamin Trent6d987e1ce1c3
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index fba7e1144c2..bcbc1e33b83 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -470,6 +470,8 @@ Bug Fixes
`IndexWriter.forceMerge` or
`IndexWriter.addIndexes(CodecReader...)`, or reindexing entirely.
+* GITHUB#13799: Disable intra-merge parallelism for all structures but kNN vectors. (Ben Trent)
+
Build
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
index dcf9923feb3..838699215f0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java
@@ -41,7 +41,7 @@ import org.apache.lucene.util.packed.PackedLongValues;
*
* @lucene.experimental
*/
-public class MergeState implements Cloneable {
+public class MergeState {
/** Maps document IDs from old segments to document IDs in the new segment */
public final DocMap[] docMaps;
@@ -302,55 +302,4 @@ public class MergeState implements Cloneable {
this.intraMergeTaskExecutor = intraMergeTaskExecutor;
this.needsIndexSort = needsIndexSort;
}
-
- @Override
- public MergeState clone() {
- StoredFieldsReader[] storedFieldsReaders = this.storedFieldsReaders.clone();
- TermVectorsReader[] termVectorsReaders = this.termVectorsReaders.clone();
- NormsProducer[] normsProducers = this.normsProducers.clone();
- DocValuesProducer[] docValuesProducers = this.docValuesProducers.clone();
- FieldsProducer[] fieldsProducers = this.fieldsProducers.clone();
- PointsReader[] pointsReaders = this.pointsReaders.clone();
- KnnVectorsReader[] knnVectorsReaders = this.knnVectorsReaders.clone();
- for (int i = 0; i < storedFieldsReaders.length; ++i) {
- if (storedFieldsReaders[i] != null) {
- storedFieldsReaders[i] = storedFieldsReaders[i].getMergeInstance();
- }
- if (termVectorsReaders[i] != null) {
- termVectorsReaders[i] = termVectorsReaders[i].getMergeInstance();
- }
- if (normsProducers[i] != null) {
- normsProducers[i] = normsProducers[i].getMergeInstance();
- }
- if (docValuesProducers[i] != null) {
- docValuesProducers[i] = docValuesProducers[i].getMergeInstance();
- }
- if (fieldsProducers[i] != null) {
- fieldsProducers[i] = fieldsProducers[i].getMergeInstance();
- }
- if (pointsReaders[i] != null) {
- pointsReaders[i] = pointsReaders[i].getMergeInstance();
- }
- if (knnVectorsReaders[i] != null) {
- knnVectorsReaders[i] = knnVectorsReaders[i].getMergeInstance();
- }
- }
- return new MergeState(
- docMaps,
- segmentInfo,
- mergeFieldInfos,
- storedFieldsReaders,
- termVectorsReaders,
- normsProducers,
- docValuesProducers,
- fieldInfos,
- liveDocs,
- fieldsProducers,
- pointsReaders,
- knnVectorsReaders,
- maxDocs,
- infoStream,
- intraMergeTaskExecutor,
- needsIndexSort);
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
index b9f14b4e39c..5e336c7fef0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
@@ -17,9 +17,7 @@
package org.apache.lucene.index;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.codecs.Codec;
@@ -31,7 +29,6 @@ import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PointsWriter;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.search.TaskExecutor;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.InfoStream;
@@ -102,12 +99,7 @@ final class SegmentMerger {
}
private MergeState mergeState() {
- MergeState mergeState = this.mergeState;
- if (Thread.currentThread() != mergeStateCreationThread) {
- // Most merges, e.g. small merges, run in the same thread, so save the cost of pulling a clone
- // in that case.
- mergeState = mergeState.clone();
- }
+ assert Thread.currentThread() == mergeStateCreationThread;
return mergeState;
}
@@ -147,8 +139,6 @@ final class SegmentMerger {
IOContext.DEFAULT,
segmentWriteState.segmentSuffix);
- TaskExecutor taskExecutor = new TaskExecutor(mergeState.intraMergeTaskExecutor);
- List<Callable<Void>> mergingTasks = new ArrayList<>();
if (mergeState.mergeFieldInfos.hasNorms()) {
mergeWithLogging(this::mergeNorms, segmentWriteState, segmentReadState, "norms", numMerged);
}
@@ -161,12 +151,7 @@ final class SegmentMerger {
}
if (mergeState.mergeFieldInfos.hasPointValues()) {
- mergingTasks.add(
- () -> {
- mergeWithLogging(
- this::mergePoints, segmentWriteState, segmentReadState, "points", numMerged);
- return null;
- });
+ mergeWithLogging(this::mergePoints, segmentWriteState, segmentReadState, "points", numMerged);
}
if (mergeState.mergeFieldInfos.hasVectorValues()) {
@@ -179,14 +164,9 @@ final class SegmentMerger {
}
if (mergeState.mergeFieldInfos.hasTermVectors()) {
- mergingTasks.add(
- () -> {
- mergeWithLogging(this::mergeTermVectors, "term vectors");
- return null;
- });
+ mergeWithLogging(this::mergeTermVectors, "term vectors");
}
- taskExecutor.invokeAll(mergingTasks);
// write the merged infos
mergeWithLogging(
this::mergeFieldInfos, segmentWriteState, segmentReadState, "field infos", numMerged);