Skip to content

Apache Lucene open-source search software

MultiIndexMergeScheduler: a production multi-tenant merge scheduler (#15015)

Shawn Yarbroughcfb407bb0080
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a784be2c590..d1939e303f6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -123,6 +123,8 @@ API Changes
New Features
---------------------
+* GITHUB#15015: MultiIndexMergeScheduler: a production multi-tenant merge scheduler (Shawn Yarbrough)
+
* GITHUB#14404: Introducing DocValuesMultiRangeQuery.SortedNumericStabbingBuilder into sandbox.
(Mikhail Khludnev)
diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
index e2d5ee569b1..8df3760a1e4 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
@@ -583,7 +583,6 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
// OK to spawn a new merge thread to handle this
// merge:
final MergeThread newMergeThread = getMergeThread(mergeSource, merge);
- mergeThreads.add(newMergeThread);
updateIOThrottle(newMergeThread.merge, newMergeThread.rateLimiter);
@@ -671,6 +670,7 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
final MergeThread thread = new MergeThread(mergeSource, merge);
thread.setDaemon(true);
thread.setName("Lucene Merge Thread #" + mergeThreadCounter++);
+ mergeThreads.add(thread);
return thread;
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.java
new file mode 100644
index 00000000000..84b88aafbc9
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.io.IOException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.InfoStream;
+
+/**
+ * Multi-index or multi-tenant merge scheduling.
+ *
+ * <p>MultiIndexMergeScheduler builds on existing functionality in ConcurrentMergeScheduler by
+ * automatically tracking merge sources and merge threads and the index Directory that each applies
+ * to, and then shunting all of them into a single ConcurrentMergeScheduler instance.
+ *
+ * <p>The multi-tenant merge scheduling can be used easily by creating a MultiIndexMergeScheduler
+ * instance for each index and then using each instance normally, the same way you would use a lone
+ * ConcurrentMergeScheduler.
+ *
+ * @lucene.experimental
+ */
+class MultiIndexMergeScheduler extends MergeScheduler {
+ private final Directory directory;
+ private final CombinedMergeScheduler combinedMergeScheduler;
+ private final boolean manageSingleton;
+
+ /** The main MultiIndexMergeScheduler constructor -- use this one. */
+ public MultiIndexMergeScheduler(Directory directory) {
+ this.directory = directory;
+ this.combinedMergeScheduler = CombinedMergeScheduler.acquireSingleton();
+ this.manageSingleton = true;
+ }
+
+ /** Alternate MultiIndexMergeScheduler constructor for unit testing. Does not close() the CMS. */
+ MultiIndexMergeScheduler(Directory directory, CombinedMergeScheduler combinedMergeScheduler) {
+ this.directory = directory;
+ this.combinedMergeScheduler = combinedMergeScheduler;
+ this.manageSingleton = false;
+ }
+
+ public Directory getDirectory() {
+ return directory;
+ }
+
+ public CombinedMergeScheduler getCombinedMergeScheduler() {
+ return combinedMergeScheduler;
+ }
+
+ @Override
+ public void merge(MergeSource mergeSource, MergeTrigger trigger) throws IOException {
+ var taggedMergeSource =
+ new CombinedMergeScheduler.TaggedMergeSource(mergeSource, this.directory);
+ this.combinedMergeScheduler.merge(taggedMergeSource, trigger);
+ }
+
+ @Override
+ public Directory wrapForMerge(MergePolicy.OneMerge merge, Directory in) {
+ return this.combinedMergeScheduler.wrapForMerge(merge, in);
+ }
+
+ /** Close this scheduler for one directory/index. Called automatically by IndexWriter. */
+ @Override
+ public void close() throws IOException {
+ this.combinedMergeScheduler.sync(this.directory);
+ if (this.manageSingleton) {
+ CombinedMergeScheduler.releaseSingleton();
+ }
+ }
+
+ // We created this method because we cannot easily override the initialize() method
+ // in ConcurrentMergeScheduler. We don't need the initDynamicDefaults() part in the
+ // initialize() method, and only need the setInfoStream().
+ public void setInfoStream(InfoStream infoStream) {
+ this.combinedMergeScheduler.setInfoStream(infoStream);
+ }
+
+ /**
+ * CombinedMergeScheduler is used internally by MultiIndexMergeScheduler to balance resources
+ * across multiple indices. Normally you don't need to use this.
+ *
+ * <p>For testing purposes, or if partitioning of tenants into groups is needed for some reason, a
+ * CombinedMergeScheduler can be provided to the MultiIndexMergeScheduler constructor.
+ *
+ * <p>CombinedMergeScheduler should <b><i>not</i></b> be passed directly to IndexWriter.
+ */
+ static class CombinedMergeScheduler extends ConcurrentMergeScheduler {
+ @SuppressWarnings("NonFinalStaticField")
+ private static CombinedMergeScheduler singleton = null;
+
+ @SuppressWarnings("NonFinalStaticField")
+ private static int singletonRefCount = 0;
+
+ private static synchronized CombinedMergeScheduler acquireSingleton() {
+ if (singleton == null) {
+ singleton = new CombinedMergeScheduler();
+ }
+ singletonRefCount++;
+ return singleton;
+ }
+
+ private static synchronized void releaseSingleton() throws IOException {
+ if (singletonRefCount < 1) {
+ throw new IllegalStateException("decrementSingletonReference() called too many times");
+ }
+ singletonRefCount--;
+ if (singletonRefCount == 0) {
+ singleton.close();
+ singleton = null;
+ }
+ }
+
+ public static synchronized CombinedMergeScheduler peekSingleton() {
+ return singleton;
+ }
+
+ // A filter pattern
+ static class TaggedMergeSource implements MergeScheduler.MergeSource {
+ private final MergeScheduler.MergeSource in;
+ private final Directory directory;
+
+ TaggedMergeSource(MergeScheduler.MergeSource in, Directory directory) {
+ this.in = in;
+ this.directory = directory;
+ }
+
+ public Directory getDirectory() {
+ return this.directory;
+ }
+
+ @Override
+ public MergePolicy.OneMerge getNextMerge() {
+ return this.in.getNextMerge();
+ }
+
+ @Override
+ public void onMergeFinished(MergePolicy.OneMerge merge) {
+ this.in.onMergeFinished(merge);
+ }
+
+ @Override
+ public boolean hasPendingMerges() {
+ return this.in.hasPendingMerges();
+ }
+
+ @Override
+ public void merge(MergePolicy.OneMerge merge) throws IOException {
+ this.in.merge(merge);
+ }
+ }
+
+ public void setInfoStream(InfoStream infoStream) {
+ this.infoStream = infoStream;
+ }
+
+ public void sync(Directory directory) {
+ boolean interrupted = false;
+ try {
+ while (true) {
+ MergeThread toSync = null;
+ synchronized (this) {
+ for (MergeThread t : this.mergeThreads) {
+ // In case a merge thread is calling us, don't try to sync on
+ // itself, since that will never finish!
+ if (t.isAlive()
+ && t != Thread.currentThread()
+ // Only wait for merge threads for the current index to finish
+ && t.mergeSource instanceof TaggedMergeSource
+ && ((TaggedMergeSource) t.mergeSource).getDirectory().equals(directory)) {
+ toSync = t;
+ break;
+ }
+ }
+ }
+ if (toSync != null) {
+ try {
+ toSync.join();
+ } catch (
+ @SuppressWarnings("unused")
+ InterruptedException ie) {
+ // ignore this Exception, we will retry until all threads are dead
+ interrupted = true;
+ }
+ } else {
+ break;
+ }
+ }
+ } finally {
+ // finally, restore interrupt status:
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.java
new file mode 100644
index 00000000000..961edfdc351
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.index;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestMultiIndexMergeScheduler extends LuceneTestCase {
+
+ public void testCloseSingle() throws Exception {
+ Directory directory = newDirectory();
+ HashSet<Directory> directoriesToBeClosed = new HashSet<>();
+ directoriesToBeClosed.add(directory);
+
+ MultiIndexMergeScheduler scheduler =
+ new MultiIndexMergeScheduler(
+ directory,
+ new MultiIndexMergeScheduler.CombinedMergeScheduler() {
+ @Override
+ public void sync(Directory directory) {
+ super.sync(directory);
+ assertTrue(directoriesToBeClosed.remove(directory));
+ }
+ });
+ scheduler.getCombinedMergeScheduler().setDefaultMaxMergesAndThreads(false);
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setMergeScheduler(scheduler);
+ iwc.setMaxBufferedDocs(2);
+ IndexWriter writer = new IndexWriter(directory, iwc);
+
+ for (int i = 0; i < 5; i++) {
+ Document doc = new Document();
+ Field idField = newStringField("id", "", Field.Store.YES);
+ doc.add(idField);
+ writer.addDocument(doc);
+ }
+ writer.commit();
+
+ writer.forceMerge(1);
+ writer.close();
+
+ directory.close();
+ assertTrue(directoriesToBeClosed.isEmpty());
+ }
+
+ public void testCloseMultiple() throws Exception {
+ // Write to many indexes and do merges on them.
+ // The unit test is randomized but reproducible.
+ int DIRECTORY_COUNT = 10;
+ int DOCUMENT_COUNT = 50;
+
+ // Create a new Directory for each index.
+ var directories = ConcurrentHashMap.newKeySet();
+ ArrayList<Directory> directoriesToBeClosed = new ArrayList<>();
+ for (int i = 0; i < DIRECTORY_COUNT; i++) {
+ Directory directory = newDirectory();
+ directories.add(directory);
+ directoriesToBeClosed.add(directory);
+ }
+
+ // This CombinedMergeScheduler setup code is only for unit testing.
+ // Normally, the MultiIndexMergeScheduler would handle this setup automatically and silently.
+ var combinedScheduler =
+ new MultiIndexMergeScheduler.CombinedMergeScheduler() {
+ @Override
+ public void sync(Directory directory) {
+ // Blocks and waits for one MultiIndexMergeScheduler's merges to finish.
+ super.sync(directory);
+ // Record success for this Directory for the unit test.
+ assertTrue(directories.remove(directory));
+ }
+ };
+
+ // Create a MultiIndexMergeScheduler and an IndexWriter for each Directory.
+ ArrayList<MultiIndexMergeScheduler> schedulers = new ArrayList<>();
+ ArrayList<IndexWriter> writers = new ArrayList<>();
+ int[] documentCounts = new int[DIRECTORY_COUNT];
+ int directoryIndex = 0;
+ for (Directory directory : directoriesToBeClosed) {
+ var scheduler = new MultiIndexMergeScheduler(directory, combinedScheduler);
+ schedulers.add(scheduler);
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setMergeScheduler(scheduler);
+ iwc.setMaxBufferedDocs(2);
+
+ IndexWriter writer = new IndexWriter(scheduler.getDirectory(), iwc);
+ writers.add(writer);
+
+ documentCounts[directoryIndex] = 0;
+ ++directoryIndex;
+ }
+
+ assertTrue(schedulers.size() == DIRECTORY_COUNT);
+ assertTrue(writers.size() == DIRECTORY_COUNT);
+
+ // More unit test setup.
+ schedulers.get(0).getCombinedMergeScheduler().setDefaultMaxMergesAndThreads(false);
+
+ // Randomly spread the documents across the indexes.
+ // The merge ordering is calculated beforehand to make this unit test be reproducible.
+ ConcurrentLinkedQueue<Integer> nextDirectoryToWrite = new ConcurrentLinkedQueue<>();
+ for (int i = 0; i < DOCUMENT_COUNT; ++i) {
+ int r = random().nextInt(DIRECTORY_COUNT);
+ nextDirectoryToWrite.add(r);
+ ++documentCounts[r];
+ }
+
+ // Write documents to each index and force some merges.
+ final ArrayList<Thread> threads = new ArrayList<>();
+ final Object lock = new Object();
+ for (int schedulerIndex = 0; schedulerIndex < schedulers.size(); schedulerIndex++) {
+ int threadID = schedulerIndex;
+
+ Thread thread =
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ IndexWriter writer = writers.get(threadID);
+ while (true) {
+ synchronized (lock) {
+ if (!nextDirectoryToWrite.isEmpty()
+ && nextDirectoryToWrite.peek() != threadID) {
+ // sleep
+ while (!nextDirectoryToWrite.isEmpty()
+ && nextDirectoryToWrite.peek() != threadID) {
+ try {
+ lock.wait();
+ } catch (
+ @SuppressWarnings("unused")
+ InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ // awaken
+ continue;
+ }
+
+ // write a doc
+ Document doc = new Document();
+ Field idField = newStringField("id", "", Field.Store.NO);
+ doc.add(idField);
+ writer.addDocument(doc);
+ --documentCounts[threadID];
+
+ if (documentCounts[threadID] == 0) {
+ // force a merge
+ writer.commit();
+ writer.forceMerge(1, true);
+ writer.commit();
+ writer.close();
+ writers.set(threadID, null);
+ }
+
+ nextDirectoryToWrite.poll();
+ lock.notifyAll();
+ if (documentCounts[threadID] == 0) break;
+ }
+ }
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ thread.setName("TestMultiIndexMergeScheduler thread-" + threadID);
+ thread.start();
+ threads.add(thread);
+ }
+
+ // Wait for the workers to finish.
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ // Confirm that each IndexWriter wrote the expected number of documents and was closed.
+ for (IndexWriter writer : writers) assertTrue(writer == null);
+
+ // Confirm that each MultiIndexMergeScheduler was closed by its IndexWriter,
+ // and also that the CombinedMergeScheduler waited for that IndexWriter's merges to finish.
+ assertTrue(directories.isEmpty());
+
+ // Clean up the Directory objects.
+ for (Directory directory : directoriesToBeClosed) {
+ directory.close();
+ }
+ }
+
+ public void testReferenceCounting() throws Exception {
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0
+
+ Directory directory1 = newDirectory();
+ MultiIndexMergeScheduler mims1 = new MultiIndexMergeScheduler(directory1);
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1
+
+ Directory directory2 = newDirectory();
+ MultiIndexMergeScheduler mims2 = new MultiIndexMergeScheduler(directory2);
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 2
+
+ mims1.close();
+ directory1.close();
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1
+
+ mims2.close();
+ directory2.close();
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0
+
+ Directory directory3 = newDirectory();
+ MultiIndexMergeScheduler mims3 = new MultiIndexMergeScheduler(directory3);
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1
+
+ mims3.close();
+ directory3.close();
+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0
+ }
+}