Apache Solr open-source search software
Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown
Chris Hostetterd0d71a7b38f8
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 827010c59c1..5501eb95017 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -231,6 +231,8 @@ Improvements* SOLR-17732: Score-based return fields other than "score" can now be returned in distributed queries. (Houston Putman)+* SOLR-17744: Solr now enables Jetty's Graceful Shutdown features to prevent client connections from being abruptly terminated on orderly shutdown (hossman)+Optimizations---------------------* SOLR-17578: Remove ZkController internal core supplier, for slightly faster reconnection after Zookeeper session loss. (Pierre Salagnac)diff --git a/solr/bin/solr b/solr/bin/solrindex 3466cdc6b23..846ff39ce32 100755--- a/solr/bin/solr+++ b/solr/bin/solr@@ -300,6 +300,11 @@ elsefiexport SOLR_URL_SCHEME+# Gracefully wait for existing requests on shutdown+if [ "${SOLR_JETTY_GRACEFUL:-true}" == "true" ]; then+ SOLR_JETTY_CONFIG+=("--module=graceful")+fi+# Requestlog optionsif [ "${SOLR_REQUESTLOG_ENABLED:-true}" == "true" ]; thenSOLR_JETTY_CONFIG+=("--module=requestlog")diff --git a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.javaindex 515664f1150..ba5949b7a13 100644--- a/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/SplitShardTest.java@@ -387,6 +387,7 @@ public class SplitShardTest extends SolrCloudTestCase {(s, replica) -> {if (replica.getNodeName().equals(jetty.getNodeName())&& !replica.isLeader()+ && replica.getState().equals(Replica.State.ACTIVE)&& set.contains(replica.shard)) {set.remove(replica.shard);}diff --git a/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.javanew file mode 100644index 00000000000..44caa313c52--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/TestGracefulJettyShutdown.java@@ -0,0 +1,161 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.List;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Future;+import java.util.concurrent.Semaphore;+import java.util.concurrent.TimeUnit;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.ExecutorUtil;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.SolrCore;+import org.apache.solr.embedded.JettyConfig;+import org.apache.solr.embedded.JettySolrRunner;+import org.apache.solr.handler.component.SearchHandler;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class TestGracefulJettyShutdown extends SolrTestCaseJ4 {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public void testSingleShardInFlightRequestsDuringShutDown() throws Exception {+ final String collection = getSaferTestName();+ final String handler = "/foo";++ final Semaphore handlerGate = new Semaphore(0);+ final Semaphore handlerSignal = new Semaphore(0);++ final ExecutorService exec = ExecutorUtil.newMDCAwareCachedThreadPool("client-requests");+ final MiniSolrCloudCluster cluster =+ new MiniSolrCloudCluster(1, createTempDir(), JettyConfig.builder().build());+ try {+ assertTrue(+ CollectionAdminRequest.createCollection(collection, "_default", 1, 1)+ .process(cluster.getSolrClient())+ .isSuccess());++ final JettySolrRunner nodeToStop = cluster.getJettySolrRunner(0);++ // register our custom handler with "all" (one) of our SolrCores+ for (String coreName : nodeToStop.getCoreContainer().getLoadedCoreNames()) {+ try (SolrCore core = nodeToStop.getCoreContainer().getCore(coreName)) {+ final BlockingSearchHandler h = new BlockingSearchHandler(handlerGate, handlerSignal);+ h.inform(core);+ core.registerRequestHandler(handler, h);+ }+ }++ final CloudSolrClient cloudClient = cluster.getSolrClient();++ // add a few docs...+ cloudClient.add(collection, sdoc("id", "xxx", "foo_s", "aaa"));+ cloudClient.add(collection, sdoc("id", "yyy", "foo_s", "bbb"));+ cloudClient.add(collection, sdoc("id", "zzz", "foo_s", "aaa"));+ cloudClient.commit(collection);++ final List<Future<QueryResponse>> results = new ArrayList<>(13);++ try (SolrClient jettyClient = nodeToStop.newClient()) {+ final QueryRequest req = new QueryRequest(params("q", "foo_s:aaa"));+ req.setPath(handler);++ // check inflight requests using both clients...+ for (SolrClient client : Arrays.asList(cloudClient, jettyClient)) {+ results.add(+ exec.submit(+ () -> {+ return req.process(client, collection);+ }));+ }++ // wait for our handlers to indicate they have recieved the requests and started processing+ log.info("Waiting for signals from both requests");+ assertTrue(handlerSignal.tryAcquire(2, 300, TimeUnit.SECONDS)); // safety valve++ // stop our node (via executor so it doesn't block) and open the gate for our handlers+ log.info("Stopping jetty node");+ final Future<Boolean> stopped =+ exec.submit(+ () -> {+ nodeToStop.stop();+ return true;+ });+ log.info("Releasing gate for requests");+ handlerGate.release(2);+ log.info("Released gate for requests");++ // confirm success of requests+ assertEquals(2, results.size());+ for (Future<QueryResponse> f : results) {+ final QueryResponse rsp = f.get(300, TimeUnit.SECONDS); // safety valve+ assertEquals(2, rsp.getResults().getNumFound());+ }+ assertTrue(stopped.get(300, TimeUnit.SECONDS)); // safety valve+ }++ } finally {+ handlerGate.release(9999999); // safety valve+ handlerSignal.release(9999999); // safety valve+ cluster.shutdown();+ ExecutorUtil.shutdownAndAwaitTermination(exec);+ }+ }++ /**+ * On every request, prior to handling, releases a ticket to it's signal Semaphre, and then blocks+ * until it can acquire a ticket from it's gate semaphore. Has a safety valve that gives up on the+ * gate after 300 seconds+ */+ private static final class BlockingSearchHandler extends SearchHandler {+ private final Semaphore gate;+ private final Semaphore signal;++ public BlockingSearchHandler(final Semaphore gate, final Semaphore signal) {+ super();+ this.gate = gate;+ this.signal = signal;+ super.init(new NamedList<>());+ }++ @Override+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {+ log.info("Starting request");+ signal.release();+ if (gate.tryAcquire(300, TimeUnit.SECONDS)) {+ super.handleRequestBody(req, rsp);+ } else {+ log.error("Gate safety valve timeout");+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Gate timeout");+ }+ log.info("Finishing request");+ }+ }+}diff --git a/solr/server/etc/jetty-graceful.xml b/solr/server/etc/jetty-graceful.xmlnew file mode 100644index 00000000000..34ec9316445--- /dev/null+++ b/solr/server/etc/jetty-graceful.xml@@ -0,0 +1,22 @@+<?xml version="1.0"?>+<!DOCTYPE Configure PUBLIC "-//Jetty//Configure//EN" "https://www.eclipse.org/jetty/configure_10_0.dtd">++<!-- =============================================================== -->+<!-- Configure Jetty to finish existing requests on STOP. -->+<!-- -->+<!-- This is inspired by the "official" Jetty 12 graceful module, -->+<!-- adapted to use StatisticsHandler for Jetty 10 and 11. -->+<!-- -->+<!-- IF THIS IS MODIFIED, JettySolrRunner MUST BE MODIFIED! -->+<!-- =============================================================== -->++<Configure id="Server" class="org.eclipse.jetty.server.Server">+ <Call name="insertHandler">+ <Arg>+ <New id="GracefulHandler" class="org.eclipse.jetty.server.handler.StatisticsHandler">+ <Set name="gracefulShutdownWaitsForRequests">true</Set>+ </New>+ </Arg>+ </Call>+ <Set name="stopTimeout"><Property name="solr.jetty.stop.timeout" default="15000"/></Set>+</Configure>diff --git a/solr/server/modules/graceful.mod b/solr/server/modules/graceful.modnew file mode 100644index 00000000000..581ebca898f--- /dev/null+++ b/solr/server/modules/graceful.mod@@ -0,0 +1,8 @@+[description]+Enables Graceful processing of requests on STOP. Inspired by graceful.mod for Jetty-12, this works with Jetty 10 and 11.++[depend]+server++[xml]+etc/jetty-graceful.xmldiff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex 7f9a05229e8..8049fc59d70 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -775,9 +775,11 @@ public class MiniSolrCloudCluster {try {future.get();} catch (ExecutionException e) {+ log.error(message, e);parsed.addSuppressed(e.getCause());ok = false;} catch (InterruptedException e) {+ log.error(message, e);Thread.interrupted();throw e;}diff --git a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.javaindex 12baf8a0e37..93bf50e7471 100644--- a/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java+++ b/solr/test-framework/src/java/org/apache/solr/embedded/JettySolrRunner.java@@ -83,6 +83,7 @@ import org.eclipse.jetty.server.Server;import org.eclipse.jetty.server.ServerConnector;import org.eclipse.jetty.server.SslConnectionFactory;import org.eclipse.jetty.server.handler.HandlerWrapper;+import org.eclipse.jetty.server.handler.StatisticsHandler;import org.eclipse.jetty.server.handler.gzip.GzipHandler;import org.eclipse.jetty.server.session.DefaultSessionIdManager;import org.eclipse.jetty.servlet.FilterHolder;@@ -431,6 +432,12 @@ public class JettySolrRunner {gzipHandler.setIncludedMethods("GET");server.setHandler(gzipHandler);++ // Mimic "graceful.mod"+ final StatisticsHandler graceful = new StatisticsHandler();+ graceful.setGracefulShutdownWaitsForRequests(true);+ server.insertHandler(graceful);+ server.setStopTimeout(15 * 1000);}/**