Apache Lucene open-source search software
Disable intra-merge parallelism for all structures but kNN vectors (#13799)
Benjamin Trent6d987e1ce1c3
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex fba7e1144c2..bcbc1e33b83 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -470,6 +470,8 @@ Bug Fixes`IndexWriter.forceMerge` or`IndexWriter.addIndexes(CodecReader...)`, or reindexing entirely.+* GITHUB#13799: Disable intra-merge parallelism for all structures but kNN vectors. (Ben Trent)+Build---------------------diff --git a/lucene/core/src/java/org/apache/lucene/index/MergeState.java b/lucene/core/src/java/org/apache/lucene/index/MergeState.javaindex dcf9923feb3..838699215f0 100644--- a/lucene/core/src/java/org/apache/lucene/index/MergeState.java+++ b/lucene/core/src/java/org/apache/lucene/index/MergeState.java@@ -41,7 +41,7 @@ import org.apache.lucene.util.packed.PackedLongValues;** @lucene.experimental*/-public class MergeState implements Cloneable {+public class MergeState {/** Maps document IDs from old segments to document IDs in the new segment */public final DocMap[] docMaps;@@ -302,55 +302,4 @@ public class MergeState implements Cloneable {this.intraMergeTaskExecutor = intraMergeTaskExecutor;this.needsIndexSort = needsIndexSort;}-- @Override- public MergeState clone() {- StoredFieldsReader[] storedFieldsReaders = this.storedFieldsReaders.clone();- TermVectorsReader[] termVectorsReaders = this.termVectorsReaders.clone();- NormsProducer[] normsProducers = this.normsProducers.clone();- DocValuesProducer[] docValuesProducers = this.docValuesProducers.clone();- FieldsProducer[] fieldsProducers = this.fieldsProducers.clone();- PointsReader[] pointsReaders = this.pointsReaders.clone();- KnnVectorsReader[] knnVectorsReaders = this.knnVectorsReaders.clone();- for (int i = 0; i < storedFieldsReaders.length; ++i) {- if (storedFieldsReaders[i] != null) {- storedFieldsReaders[i] = storedFieldsReaders[i].getMergeInstance();- }- if (termVectorsReaders[i] != null) {- termVectorsReaders[i] = termVectorsReaders[i].getMergeInstance();- }- if (normsProducers[i] != null) {- normsProducers[i] = normsProducers[i].getMergeInstance();- }- if (docValuesProducers[i] != null) {- docValuesProducers[i] = docValuesProducers[i].getMergeInstance();- }- if (fieldsProducers[i] != null) {- fieldsProducers[i] = fieldsProducers[i].getMergeInstance();- }- if (pointsReaders[i] != null) {- pointsReaders[i] = pointsReaders[i].getMergeInstance();- }- if (knnVectorsReaders[i] != null) {- knnVectorsReaders[i] = knnVectorsReaders[i].getMergeInstance();- }- }- return new MergeState(- docMaps,- segmentInfo,- mergeFieldInfos,- storedFieldsReaders,- termVectorsReaders,- normsProducers,- docValuesProducers,- fieldInfos,- liveDocs,- fieldsProducers,- pointsReaders,- knnVectorsReaders,- maxDocs,- infoStream,- intraMergeTaskExecutor,- needsIndexSort);- }}diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.javaindex b9f14b4e39c..5e336c7fef0 100644--- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java@@ -17,9 +17,7 @@package org.apache.lucene.index;import java.io.IOException;-import java.util.ArrayList;import java.util.List;-import java.util.concurrent.Callable;import java.util.concurrent.Executor;import java.util.concurrent.TimeUnit;import org.apache.lucene.codecs.Codec;@@ -31,7 +29,6 @@ import org.apache.lucene.codecs.NormsProducer;import org.apache.lucene.codecs.PointsWriter;import org.apache.lucene.codecs.StoredFieldsWriter;import org.apache.lucene.codecs.TermVectorsWriter;-import org.apache.lucene.search.TaskExecutor;import org.apache.lucene.store.Directory;import org.apache.lucene.store.IOContext;import org.apache.lucene.util.InfoStream;@@ -102,12 +99,7 @@ final class SegmentMerger {}private MergeState mergeState() {- MergeState mergeState = this.mergeState;- if (Thread.currentThread() != mergeStateCreationThread) {- // Most merges, e.g. small merges, run in the same thread, so save the cost of pulling a clone- // in that case.- mergeState = mergeState.clone();- }+ assert Thread.currentThread() == mergeStateCreationThread;return mergeState;}@@ -147,8 +139,6 @@ final class SegmentMerger {IOContext.DEFAULT,segmentWriteState.segmentSuffix);- TaskExecutor taskExecutor = new TaskExecutor(mergeState.intraMergeTaskExecutor);- List<Callable<Void>> mergingTasks = new ArrayList<>();if (mergeState.mergeFieldInfos.hasNorms()) {mergeWithLogging(this::mergeNorms, segmentWriteState, segmentReadState, "norms", numMerged);}@@ -161,12 +151,7 @@ final class SegmentMerger {}if (mergeState.mergeFieldInfos.hasPointValues()) {- mergingTasks.add(- () -> {- mergeWithLogging(- this::mergePoints, segmentWriteState, segmentReadState, "points", numMerged);- return null;- });+ mergeWithLogging(this::mergePoints, segmentWriteState, segmentReadState, "points", numMerged);}if (mergeState.mergeFieldInfos.hasVectorValues()) {@@ -179,14 +164,9 @@ final class SegmentMerger {}if (mergeState.mergeFieldInfos.hasTermVectors()) {- mergingTasks.add(- () -> {- mergeWithLogging(this::mergeTermVectors, "term vectors");- return null;- });+ mergeWithLogging(this::mergeTermVectors, "term vectors");}- taskExecutor.invokeAll(mergingTasks);// write the merged infosmergeWithLogging(this::mergeFieldInfos, segmentWriteState, segmentReadState, "field infos", numMerged);