Apache Solr open-source search software
SOLR-17504: CoreContainer calls UpdateHandler.commit. (#2786)
Bruno Roustant5cc733b9fc60
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 05574f64371..80e4820ff25 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -167,6 +167,8 @@ Improvements* SOLR-17390: EmbeddedSolrServer now considers the ResponseParser (David Smiley)+* SOLR-17504: CoreContainer calls UpdateHandler.commit when closing a read-only core (Bruno Roustant)+Optimizations---------------------* SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since thediff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex a92c73ef496..69b6f07fb47 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -59,7 +59,6 @@ import org.apache.http.auth.AuthSchemeProvider;import org.apache.http.client.CredentialsProvider;import org.apache.http.config.Lookup;import org.apache.lucene.index.CorruptIndexException;-import org.apache.lucene.index.IndexWriter;import org.apache.lucene.search.IndexSearcher;import org.apache.lucene.store.Directory;import org.apache.lucene.util.BytesRef;@@ -95,6 +94,7 @@ import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Replica.State;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.CollectionUtil;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.IOUtils;@@ -138,6 +138,8 @@ import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.metrics.SolrMetricProducer;import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.pkg.SolrPackageLoader;+import org.apache.solr.request.LocalSolrQueryRequest;+import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.request.SolrRequestHandler;import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.search.CacheConfig;@@ -153,10 +155,10 @@ import org.apache.solr.security.PKIAuthenticationPlugin;import org.apache.solr.security.PublicKeyHandler;import org.apache.solr.security.SecurityPluginHolder;import org.apache.solr.security.SolrNodeKeyPair;+import org.apache.solr.update.CommitUpdateCommand;import org.apache.solr.update.SolrCoreState;import org.apache.solr.update.UpdateShardHandler;import org.apache.solr.util.OrderedExecutor;-import org.apache.solr.util.RefCounted;import org.apache.solr.util.StartupLoggingUtils;import org.apache.solr.util.stats.MetricUtils;import org.apache.zookeeper.KeeperException;@@ -2058,19 +2060,8 @@ public class CoreContainer {// force commit on old core if the new one is readOnly and prevent any new updatesif (newCore.readOnly) {- RefCounted<IndexWriter> iwRef = core.getSolrCoreState().getIndexWriter(null);- if (iwRef != null) {- IndexWriter iw = iwRef.get();- // switch old core to readOnly- core.readOnly = true;- try {- if (iw != null) {- iw.commit();- }- } finally {- iwRef.decref();- }- }+ SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());+ core.getUpdateHandler().commit(CommitUpdateCommand.closeOnCommit(req, false));}if (docCollection != null) {diff --git a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.javaindex bb18b770e5d..6a749af2ac9 100644--- a/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java+++ b/solr/core/src/java/org/apache/solr/update/CommitUpdateCommand.java@@ -21,9 +21,21 @@ import org.apache.solr.request.SolrQueryRequest;/** A commit index command encapsulated in an object. */public class CommitUpdateCommand extends UpdateCommand {++ // Behavior about opening a searcher after a hard commit.+ // - Open a standard searcher and wait for it. (default)+ // openSearcher == true && waitSearcher == true && closeSearcher == false+ // - Open a standard searcher, but do not wait for it.+ // openSearcher == true && waitSearcher == false && closeSearcher == false+ // - Open a real-time searcher quicker without auto-warming.+ // openSearcher == false && closeSearcher == false+ // - Do not open any searcher and prevent further updates (e.g. when the core is closing).+ // closeSearcher == true+public boolean optimize;public boolean openSearcher = true; // open a new searcher as part of a hard commitpublic boolean waitSearcher = true;+ private boolean closeSearcher = false;public boolean expungeDeletes = false;public boolean softCommit = false;public boolean prepareCommit = false;@@ -47,6 +59,26 @@ public class CommitUpdateCommand extends UpdateCommand {this.optimize = optimize;}+ /**+ * Creates a {@link CommitUpdateCommand} to commit before closing the core and also prevent any+ * new update by setting the core in read-only mode. Does not open a new searcher.+ */+ public static CommitUpdateCommand closeOnCommit(SolrQueryRequest req, boolean optimize) {+ CommitUpdateCommand cmd = new CommitUpdateCommand(req, optimize);+ cmd.openSearcher = false;+ cmd.waitSearcher = false;+ cmd.closeSearcher = true;+ return cmd;+ }++ /**+ * Indicates whether this command is a commit before the core is closed. Any new updates must be+ * prevented after the commit.+ */+ public boolean isClosingOnCommit() {+ return closeSearcher;+ }+@Overridepublic String name() {return "commit";diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.javaindex d730cc137fb..2704ef4e027 100644--- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java+++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java@@ -738,6 +738,9 @@ public class DirectUpdateHandler2 extends UpdateHandlerRefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);try {+ if (cmd.isClosingOnCommit()) {+ core.readOnly = true;+ }IndexWriter writer = iw.get();if (cmd.optimize) {writer.forceMerge(cmd.maxOptimizeSegments);@@ -786,7 +789,7 @@ public class DirectUpdateHandler2 extends UpdateHandlerif (ulog != null) ulog.preSoftCommit(cmd);if (cmd.openSearcher) {core.getSearcher(true, false, waitSearcher);- } else {+ } else if (!cmd.isClosingOnCommit()) {// force open a new realtime searcher so realtime-get and versioning code can see the// latestRefCounted<SolrIndexSearcher> searchHolder = core.openNewSearcher(true, true);@@ -971,8 +974,10 @@ public class DirectUpdateHandler2 extends UpdateHandler// todo: refactor this shared code (or figure out why a real CommitUpdateCommand can't// be used)- SolrIndexWriter.setCommitData(writer, cmd.getVersion(), null);- writer.commit();+ if (shouldCommit(cmd, writer)) {+ SolrIndexWriter.setCommitData(writer, cmd.getVersion(), cmd.commitData);+ writer.commit();+ }synchronized (solrCoreState.getUpdateLock()) {ulog.postCommit(cmd);diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xmlindex 2c06096732c..856079f329e 100644--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml@@ -59,7 +59,7 @@<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>- <updateHandler class="solr.DirectUpdateHandler2">+ <updateHandler class="${solr.updateHandler:solr.DirectUpdateHandler2}"><autoCommit><maxTime>${solr.autoCommit.maxTime:-1}</maxTime>diff --git a/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.java b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.javanew file mode 100644index 00000000000..49a3be19697--- /dev/null+++ b/solr/core/src/test/org/apache/solr/update/DirectUpdateHandlerWithUpdateLogTest.java@@ -0,0 +1,98 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.update;++import java.io.IOException;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import org.apache.lucene.index.IndexWriter;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.util.TimeSource;+import org.apache.solr.core.SolrCore;+import org.apache.solr.util.LogLevel;+import org.apache.solr.util.TimeOut;+import org.junit.BeforeClass;+import org.junit.Test;++/** Tests {@link DirectUpdateHandler2} with update log enabled. */+@LogLevel("org.apache.solr.update=INFO")+public class DirectUpdateHandlerWithUpdateLogTest extends SolrTestCaseJ4 {++ @BeforeClass+ public static void beforeClass() throws Exception {+ System.setProperty("solr.updateHandler", SpyingUpdateHandler.class.getName());+ initCore("solrconfig.xml", "schema11.xml");+ }++ @Test+ public void testShouldCommitHook() throws Exception {+ // Given a core.+ SolrCore core = h.getCore();+ assertNotNull(core);+ SpyingUpdateHandler updater = (SpyingUpdateHandler) core.getUpdateHandler();+ updater.shouldCommitCallCount.set(0);++ // When we add a doc and commit.+ assertU(adoc("id", "1"));+ assertU(commit());+ // Then the shouldCommit hook is called.+ assertEquals(1, updater.shouldCommitCallCount.get());++ // When we add a doc and soft commit.+ assertU(adoc("id", "2"));+ assertU(commit("softCommit", "true"));+ // Then the shouldCommit hook is not called.+ assertEquals(1, updater.shouldCommitCallCount.get());+ // And when we commit.+ assertU(commit());+ // Then the shouldCommit hook is called.+ assertEquals(2, updater.shouldCommitCallCount.get());++ // When we commit with no updates (empty commit).+ assertU(commit());+ // Then the shouldCommit hook is called (may commit only user metadata).+ assertEquals(3, updater.shouldCommitCallCount.get());++ // When we add a doc, do not commit, and close the IndexWriter.+ assertU(adoc("id", "3"));+ h.close();+ // Then the shouldCommit hook is called.+ new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME)+ .waitFor(+ "Timeout waiting for should commit hook",+ () -> updater.shouldCommitCallCount.get() == 4);+ }++ public static class SpyingUpdateHandler extends DirectUpdateHandler2 {++ final AtomicInteger shouldCommitCallCount = new AtomicInteger();++ public SpyingUpdateHandler(SolrCore core) {+ super(core);+ }++ public SpyingUpdateHandler(SolrCore core, UpdateHandler updateHandler) {+ super(core, updateHandler);+ }++ @Override+ protected boolean shouldCommit(CommitUpdateCommand cmd, IndexWriter writer) throws IOException {+ shouldCommitCallCount.incrementAndGet();+ return super.shouldCommit(cmd, writer);+ }+ }+}