Apache Lucene open-source search software
MultiIndexMergeScheduler: a production multi-tenant merge scheduler (#15015)
Shawn Yarbroughcfb407bb0080
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex a784be2c590..d1939e303f6 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -123,6 +123,8 @@ API ChangesNew Features---------------------+* GITHUB#15015: MultiIndexMergeScheduler: a production multi-tenant merge scheduler (Shawn Yarbrough)+* GITHUB#14404: Introducing DocValuesMultiRangeQuery.SortedNumericStabbingBuilder into sandbox.(Mikhail Khludnev)diff --git a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.javaindex e2d5ee569b1..8df3760a1e4 100644--- a/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java+++ b/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java@@ -583,7 +583,6 @@ public class ConcurrentMergeScheduler extends MergeScheduler {// OK to spawn a new merge thread to handle this// merge:final MergeThread newMergeThread = getMergeThread(mergeSource, merge);- mergeThreads.add(newMergeThread);updateIOThrottle(newMergeThread.merge, newMergeThread.rateLimiter);@@ -671,6 +670,7 @@ public class ConcurrentMergeScheduler extends MergeScheduler {final MergeThread thread = new MergeThread(mergeSource, merge);thread.setDaemon(true);thread.setName("Lucene Merge Thread #" + mergeThreadCounter++);+ mergeThreads.add(thread);return thread;}diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.java b/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.javanew file mode 100644index 00000000000..84b88aafbc9--- /dev/null+++ b/lucene/core/src/java/org/apache/lucene/index/MultiIndexMergeScheduler.java@@ -0,0 +1,209 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.io.IOException;+import org.apache.lucene.store.Directory;+import org.apache.lucene.util.InfoStream;++/**+ * Multi-index or multi-tenant merge scheduling.+ *+ * <p>MultiIndexMergeScheduler builds on existing functionality in ConcurrentMergeScheduler by+ * automatically tracking merge sources and merge threads and the index Directory that each applies+ * to, and then shunting all of them into a single ConcurrentMergeScheduler instance.+ *+ * <p>The multi-tenant merge scheduling can be used easily by creating a MultiIndexMergeScheduler+ * instance for each index and then using each instance normally, the same way you would use a lone+ * ConcurrentMergeScheduler.+ *+ * @lucene.experimental+ */+class MultiIndexMergeScheduler extends MergeScheduler {+ private final Directory directory;+ private final CombinedMergeScheduler combinedMergeScheduler;+ private final boolean manageSingleton;++ /** The main MultiIndexMergeScheduler constructor -- use this one. */+ public MultiIndexMergeScheduler(Directory directory) {+ this.directory = directory;+ this.combinedMergeScheduler = CombinedMergeScheduler.acquireSingleton();+ this.manageSingleton = true;+ }++ /** Alternate MultiIndexMergeScheduler constructor for unit testing. Does not close() the CMS. */+ MultiIndexMergeScheduler(Directory directory, CombinedMergeScheduler combinedMergeScheduler) {+ this.directory = directory;+ this.combinedMergeScheduler = combinedMergeScheduler;+ this.manageSingleton = false;+ }++ public Directory getDirectory() {+ return directory;+ }++ public CombinedMergeScheduler getCombinedMergeScheduler() {+ return combinedMergeScheduler;+ }++ @Override+ public void merge(MergeSource mergeSource, MergeTrigger trigger) throws IOException {+ var taggedMergeSource =+ new CombinedMergeScheduler.TaggedMergeSource(mergeSource, this.directory);+ this.combinedMergeScheduler.merge(taggedMergeSource, trigger);+ }++ @Override+ public Directory wrapForMerge(MergePolicy.OneMerge merge, Directory in) {+ return this.combinedMergeScheduler.wrapForMerge(merge, in);+ }++ /** Close this scheduler for one directory/index. Called automatically by IndexWriter. */+ @Override+ public void close() throws IOException {+ this.combinedMergeScheduler.sync(this.directory);+ if (this.manageSingleton) {+ CombinedMergeScheduler.releaseSingleton();+ }+ }++ // We created this method because we cannot easily override the initialize() method+ // in ConcurrentMergeScheduler. We don't need the initDynamicDefaults() part in the+ // initialize() method, and only need the setInfoStream().+ public void setInfoStream(InfoStream infoStream) {+ this.combinedMergeScheduler.setInfoStream(infoStream);+ }++ /**+ * CombinedMergeScheduler is used internally by MultiIndexMergeScheduler to balance resources+ * across multiple indices. Normally you don't need to use this.+ *+ * <p>For testing purposes, or if partitioning of tenants into groups is needed for some reason, a+ * CombinedMergeScheduler can be provided to the MultiIndexMergeScheduler constructor.+ *+ * <p>CombinedMergeScheduler should <b><i>not</i></b> be passed directly to IndexWriter.+ */+ static class CombinedMergeScheduler extends ConcurrentMergeScheduler {+ @SuppressWarnings("NonFinalStaticField")+ private static CombinedMergeScheduler singleton = null;++ @SuppressWarnings("NonFinalStaticField")+ private static int singletonRefCount = 0;++ private static synchronized CombinedMergeScheduler acquireSingleton() {+ if (singleton == null) {+ singleton = new CombinedMergeScheduler();+ }+ singletonRefCount++;+ return singleton;+ }++ private static synchronized void releaseSingleton() throws IOException {+ if (singletonRefCount < 1) {+ throw new IllegalStateException("decrementSingletonReference() called too many times");+ }+ singletonRefCount--;+ if (singletonRefCount == 0) {+ singleton.close();+ singleton = null;+ }+ }++ public static synchronized CombinedMergeScheduler peekSingleton() {+ return singleton;+ }++ // A filter pattern+ static class TaggedMergeSource implements MergeScheduler.MergeSource {+ private final MergeScheduler.MergeSource in;+ private final Directory directory;++ TaggedMergeSource(MergeScheduler.MergeSource in, Directory directory) {+ this.in = in;+ this.directory = directory;+ }++ public Directory getDirectory() {+ return this.directory;+ }++ @Override+ public MergePolicy.OneMerge getNextMerge() {+ return this.in.getNextMerge();+ }++ @Override+ public void onMergeFinished(MergePolicy.OneMerge merge) {+ this.in.onMergeFinished(merge);+ }++ @Override+ public boolean hasPendingMerges() {+ return this.in.hasPendingMerges();+ }++ @Override+ public void merge(MergePolicy.OneMerge merge) throws IOException {+ this.in.merge(merge);+ }+ }++ public void setInfoStream(InfoStream infoStream) {+ this.infoStream = infoStream;+ }++ public void sync(Directory directory) {+ boolean interrupted = false;+ try {+ while (true) {+ MergeThread toSync = null;+ synchronized (this) {+ for (MergeThread t : this.mergeThreads) {+ // In case a merge thread is calling us, don't try to sync on+ // itself, since that will never finish!+ if (t.isAlive()+ && t != Thread.currentThread()+ // Only wait for merge threads for the current index to finish+ && t.mergeSource instanceof TaggedMergeSource+ && ((TaggedMergeSource) t.mergeSource).getDirectory().equals(directory)) {+ toSync = t;+ break;+ }+ }+ }+ if (toSync != null) {+ try {+ toSync.join();+ } catch (+ @SuppressWarnings("unused")+ InterruptedException ie) {+ // ignore this Exception, we will retry until all threads are dead+ interrupted = true;+ }+ } else {+ break;+ }+ }+ } finally {+ // finally, restore interrupt status:+ if (interrupted) {+ Thread.currentThread().interrupt();+ }+ }+ }+ }+}diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.java b/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.javanew file mode 100644index 00000000000..961edfdc351--- /dev/null+++ b/lucene/core/src/test/org/apache/lucene/index/TestMultiIndexMergeScheduler.java@@ -0,0 +1,239 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.index;++import java.util.ArrayList;+import java.util.HashSet;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ConcurrentLinkedQueue;+import org.apache.lucene.document.Document;+import org.apache.lucene.document.Field;+import org.apache.lucene.store.Directory;+import org.apache.lucene.tests.util.LuceneTestCase;++public class TestMultiIndexMergeScheduler extends LuceneTestCase {++ public void testCloseSingle() throws Exception {+ Directory directory = newDirectory();+ HashSet<Directory> directoriesToBeClosed = new HashSet<>();+ directoriesToBeClosed.add(directory);++ MultiIndexMergeScheduler scheduler =+ new MultiIndexMergeScheduler(+ directory,+ new MultiIndexMergeScheduler.CombinedMergeScheduler() {+ @Override+ public void sync(Directory directory) {+ super.sync(directory);+ assertTrue(directoriesToBeClosed.remove(directory));+ }+ });+ scheduler.getCombinedMergeScheduler().setDefaultMaxMergesAndThreads(false);+ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setMergeScheduler(scheduler);+ iwc.setMaxBufferedDocs(2);+ IndexWriter writer = new IndexWriter(directory, iwc);++ for (int i = 0; i < 5; i++) {+ Document doc = new Document();+ Field idField = newStringField("id", "", Field.Store.YES);+ doc.add(idField);+ writer.addDocument(doc);+ }+ writer.commit();++ writer.forceMerge(1);+ writer.close();++ directory.close();+ assertTrue(directoriesToBeClosed.isEmpty());+ }++ public void testCloseMultiple() throws Exception {+ // Write to many indexes and do merges on them.+ // The unit test is randomized but reproducible.+ int DIRECTORY_COUNT = 10;+ int DOCUMENT_COUNT = 50;++ // Create a new Directory for each index.+ var directories = ConcurrentHashMap.newKeySet();+ ArrayList<Directory> directoriesToBeClosed = new ArrayList<>();+ for (int i = 0; i < DIRECTORY_COUNT; i++) {+ Directory directory = newDirectory();+ directories.add(directory);+ directoriesToBeClosed.add(directory);+ }++ // This CombinedMergeScheduler setup code is only for unit testing.+ // Normally, the MultiIndexMergeScheduler would handle this setup automatically and silently.+ var combinedScheduler =+ new MultiIndexMergeScheduler.CombinedMergeScheduler() {+ @Override+ public void sync(Directory directory) {+ // Blocks and waits for one MultiIndexMergeScheduler's merges to finish.+ super.sync(directory);+ // Record success for this Directory for the unit test.+ assertTrue(directories.remove(directory));+ }+ };++ // Create a MultiIndexMergeScheduler and an IndexWriter for each Directory.+ ArrayList<MultiIndexMergeScheduler> schedulers = new ArrayList<>();+ ArrayList<IndexWriter> writers = new ArrayList<>();+ int[] documentCounts = new int[DIRECTORY_COUNT];+ int directoryIndex = 0;+ for (Directory directory : directoriesToBeClosed) {+ var scheduler = new MultiIndexMergeScheduler(directory, combinedScheduler);+ schedulers.add(scheduler);++ IndexWriterConfig iwc = new IndexWriterConfig();+ iwc.setMergeScheduler(scheduler);+ iwc.setMaxBufferedDocs(2);++ IndexWriter writer = new IndexWriter(scheduler.getDirectory(), iwc);+ writers.add(writer);++ documentCounts[directoryIndex] = 0;+ ++directoryIndex;+ }++ assertTrue(schedulers.size() == DIRECTORY_COUNT);+ assertTrue(writers.size() == DIRECTORY_COUNT);++ // More unit test setup.+ schedulers.get(0).getCombinedMergeScheduler().setDefaultMaxMergesAndThreads(false);++ // Randomly spread the documents across the indexes.+ // The merge ordering is calculated beforehand to make this unit test be reproducible.+ ConcurrentLinkedQueue<Integer> nextDirectoryToWrite = new ConcurrentLinkedQueue<>();+ for (int i = 0; i < DOCUMENT_COUNT; ++i) {+ int r = random().nextInt(DIRECTORY_COUNT);+ nextDirectoryToWrite.add(r);+ ++documentCounts[r];+ }++ // Write documents to each index and force some merges.+ final ArrayList<Thread> threads = new ArrayList<>();+ final Object lock = new Object();+ for (int schedulerIndex = 0; schedulerIndex < schedulers.size(); schedulerIndex++) {+ int threadID = schedulerIndex;++ Thread thread =+ new Thread() {+ @Override+ public void run() {+ try {+ IndexWriter writer = writers.get(threadID);+ while (true) {+ synchronized (lock) {+ if (!nextDirectoryToWrite.isEmpty()+ && nextDirectoryToWrite.peek() != threadID) {+ // sleep+ while (!nextDirectoryToWrite.isEmpty()+ && nextDirectoryToWrite.peek() != threadID) {+ try {+ lock.wait();+ } catch (+ @SuppressWarnings("unused")+ InterruptedException e) {+ Thread.currentThread().interrupt();+ return;+ }+ }+ // awaken+ continue;+ }++ // write a doc+ Document doc = new Document();+ Field idField = newStringField("id", "", Field.Store.NO);+ doc.add(idField);+ writer.addDocument(doc);+ --documentCounts[threadID];++ if (documentCounts[threadID] == 0) {+ // force a merge+ writer.commit();+ writer.forceMerge(1, true);+ writer.commit();+ writer.close();+ writers.set(threadID, null);+ }++ nextDirectoryToWrite.poll();+ lock.notifyAll();+ if (documentCounts[threadID] == 0) break;+ }+ }++ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ }+ };++ thread.setName("TestMultiIndexMergeScheduler thread-" + threadID);+ thread.start();+ threads.add(thread);+ }++ // Wait for the workers to finish.+ for (Thread thread : threads) {+ thread.join();+ }++ // Confirm that each IndexWriter wrote the expected number of documents and was closed.+ for (IndexWriter writer : writers) assertTrue(writer == null);++ // Confirm that each MultiIndexMergeScheduler was closed by its IndexWriter,+ // and also that the CombinedMergeScheduler waited for that IndexWriter's merges to finish.+ assertTrue(directories.isEmpty());++ // Clean up the Directory objects.+ for (Directory directory : directoriesToBeClosed) {+ directory.close();+ }+ }++ public void testReferenceCounting() throws Exception {+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0++ Directory directory1 = newDirectory();+ MultiIndexMergeScheduler mims1 = new MultiIndexMergeScheduler(directory1);+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1++ Directory directory2 = newDirectory();+ MultiIndexMergeScheduler mims2 = new MultiIndexMergeScheduler(directory2);+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 2++ mims1.close();+ directory1.close();+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1++ mims2.close();+ directory2.close();+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0++ Directory directory3 = newDirectory();+ MultiIndexMergeScheduler mims3 = new MultiIndexMergeScheduler(directory3);+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() != null); // 1++ mims3.close();+ directory3.close();+ assertTrue(MultiIndexMergeScheduler.CombinedMergeScheduler.peekSingleton() == null); // 0+ }+}