Apache Solr open-source search software
SOLR-17277: Add 'warnOnly' mode to CircuitBreakers (#2476)
Jason Gerlowskie923ca510331
diff --git a/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.javaindex bef772989f4..b4df16a6fe7 100644--- a/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java+++ b/solr/core/src/java/org/apache/solr/handler/ContentStreamHandlerBase.java@@ -16,9 +16,6 @@*/package org.apache.solr.handler;-import static org.apache.solr.common.params.CommonParams.FAILURE;-import static org.apache.solr.common.params.CommonParams.STATUS;-import java.lang.invoke.MethodHandles;import java.util.List;import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;@@ -34,6 +31,7 @@ import org.apache.solr.update.processor.UpdateRequestProcessor;import org.apache.solr.update.processor.UpdateRequestProcessorChain;import org.apache.solr.util.circuitbreaker.CircuitBreaker;import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;+import org.apache.solr.util.circuitbreaker.CircuitBreakerUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -134,15 +132,7 @@ public abstract class ContentStreamHandlerBase extends RequestHandlerBase {if (circuitBreakerRegistry.isEnabled(SolrRequestType.UPDATE)) {List<CircuitBreaker> trippedCircuitBreakers =circuitBreakerRegistry.checkTripped(SolrRequestType.UPDATE);- if (trippedCircuitBreakers != null) {- String errorMessage = CircuitBreakerRegistry.toErrorMessage(trippedCircuitBreakers);- rsp.add(STATUS, FAILURE);- rsp.setException(- new SolrException(- CircuitBreaker.getExceptionErrorCode(),- "Circuit Breakers tripped " + errorMessage));- return true;- }+ return CircuitBreakerUtils.reportErrorIfBreakersTripped(rsp, trippedCircuitBreakers);}return false;}diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.javaindex 8d513ec1b19..d930967e2b3 100644--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java@@ -17,9 +17,7 @@package org.apache.solr.handler.component;import static org.apache.solr.common.params.CommonParams.DISTRIB;-import static org.apache.solr.common.params.CommonParams.FAILURE;import static org.apache.solr.common.params.CommonParams.PATH;-import static org.apache.solr.common.params.CommonParams.STATUS;import com.codahale.metrics.Counter;import java.io.PrintWriter;@@ -70,6 +68,7 @@ import org.apache.solr.util.SolrPluginUtils;import org.apache.solr.util.ThreadCpuTimer;import org.apache.solr.util.circuitbreaker.CircuitBreaker;import org.apache.solr.util.circuitbreaker.CircuitBreakerRegistry;+import org.apache.solr.util.circuitbreaker.CircuitBreakerUtils;import org.apache.solr.util.plugin.PluginInfoInitialized;import org.apache.solr.util.plugin.SolrCoreAware;import org.slf4j.Logger;@@ -380,15 +379,7 @@ public class SearchHandler extends RequestHandlerBasetrippedCircuitBreakers = circuitBreakerRegistry.checkTripped(SolrRequestType.QUERY);}- if (trippedCircuitBreakers != null) {- String errorMessage = CircuitBreakerRegistry.toErrorMessage(trippedCircuitBreakers);- rsp.add(STATUS, FAILURE);- rsp.setException(- new SolrException(- CircuitBreaker.getExceptionErrorCode(),- "Circuit Breakers tripped " + errorMessage));- return true;- }+ return CircuitBreakerUtils.reportErrorIfBreakersTripped(rsp, trippedCircuitBreakers);}return false;}diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.javaindex 72b91f8722c..435ca03ff86 100644--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CPUCircuitBreaker.java@@ -20,6 +20,7 @@ package org.apache.solr.util.circuitbreaker;import com.codahale.metrics.Gauge;import com.codahale.metrics.Metric;import java.lang.invoke.MethodHandles;+import java.util.Locale;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.SolrCore;import org.apache.solr.metrics.SolrMetricManager;@@ -136,6 +137,16 @@ public class CPUCircuitBreaker extends CircuitBreaker implements SolrCoreAware {enableIfSupported();}+ @Override+ public String toString() {+ return String.format(+ Locale.ROOT,+ "%s(threshold=%f, warnOnly=%b)",+ getClass().getSimpleName(),+ cpuUsageThreshold,+ isWarnOnly());+ }+private void enableIfSupported() {if (calculateLiveCPUUsage() < 0) {String msg =diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.javaindex 1bba4de7c6a..9b441e29e61 100644--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreaker.java@@ -49,12 +49,16 @@ public abstract class CircuitBreaker implements NamedListInitializedPlugin, Closprivate static SolrException.ErrorCode errorCode = resolveExceptionErrorCode();// Only query requests are checked by defaultprivate Set<SolrRequestType> requestTypes = Set.of(SolrRequestType.QUERY);+ private boolean warnOnly;private final List<SolrRequestType> SUPPORTED_TYPES =List.of(SolrRequestType.QUERY, SolrRequestType.UPDATE);@Overridepublic void init(NamedList<?> args) {SolrPluginUtils.invokeSetters(this, args);+ if (args.getBooleanArg("warnOnly") != null) {+ setWarnOnly(args.getBooleanArg("warnOnly"));+ }}public CircuitBreaker() {@@ -129,6 +133,14 @@ public abstract class CircuitBreaker implements NamedListInitializedPlugin, Clos.collect(Collectors.toSet());}+ public void setWarnOnly(boolean warnOnly) {+ this.warnOnly = warnOnly;+ }++ public boolean isWarnOnly() {+ return warnOnly;+ }+public Set<SolrRequestType> getRequestTypes() {return requestTypes;}diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.javaindex 1b7b2def483..d602706a2cd 100644--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerRegistry.java@@ -61,21 +61,28 @@ public class CircuitBreakerRegistry implements Closeable {public static final String SYSPROP_QUERY_CPU = SYSPROP_PREFIX + "query.cpu";public static final String SYSPROP_QUERY_MEM = SYSPROP_PREFIX + "query.mem";public static final String SYSPROP_QUERY_LOADAVG = SYSPROP_PREFIX + "query.loadavg";+ public static final String SYSPROP_WARN_ONLY_SUFFIX = ".warnonly";public CircuitBreakerRegistry(CoreContainer coreContainer) {initGlobal(coreContainer);}- private static void initGlobal(CoreContainer coreContainer) {+ public static List<CircuitBreaker> parseCircuitBreakersFromProperties(+ CoreContainer coreContainer) {// Read system properties to register global circuit breakers for update and query:// Example: solr.circuitbreaker.update.cpu = 50+ final var parsedBreakers = new ArrayList<CircuitBreaker>();EnvUtils.getProperties().keySet().stream().map(SYSPROP_REGEX::matcher)- .filter(Matcher::matches)- .collect(Collectors.groupingBy(m -> m.group(2) + ":" + System.getProperty(m.group(0))))+ .filter(Matcher::matches) // 0=solr.circuitbreaker.(update|query).(cpu|mem|loadavg),+ // 1=update|query, 2=cpu|mem|loadavg+ .collect(Collectors.groupingBy(m -> buildCircuitBreakerKey(m.group(2), m.group(0)))).forEach((breakerAndValue, breakers) -> {CircuitBreaker breaker;+ // See 'buildCircuitBreakerKey' for format+ // 0=metric name (e.g. loadavg), 1=threshold value (e.g. 70), 2=warnOnly flag (e.g.+ // "false")String[] breakerAndValueArr = breakerAndValue.split(":");switch (breakerAndValueArr[0]) {case "cpu":@@ -97,16 +104,30 @@ public class CircuitBreakerRegistry implements Closeable {throw new IllegalArgumentException("Unknown circuit breaker type: " + breakerAndValueArr[0]);}+ breaker.setWarnOnly(Boolean.parseBoolean(breakerAndValueArr[2]));breaker.setRequestTypes(breakers.stream().map(m -> m.group(1)).collect(Collectors.toList()));- registerGlobal(breaker);- if (log.isInfoEnabled()) {- log.info(- "Registered global circuit breaker {} for request type(s) {}",- breakerAndValue,- breaker.getRequestTypes());- }+ parsedBreakers.add(breaker);});+ return parsedBreakers;+ }++ private static String buildCircuitBreakerKey(String metricType, String thresholdProp) {+ final var warnOnly = EnvUtils.getProperty(thresholdProp + SYSPROP_WARN_ONLY_SUFFIX, "false");+ return metricType + ":" + System.getProperty(thresholdProp) + ":" + warnOnly;+ }++ private static void initGlobal(CoreContainer coreContainer) {+ final var parsedBreakers = parseCircuitBreakersFromProperties(coreContainer);+ for (CircuitBreaker breaker : parsedBreakers) {+ registerGlobal(breaker);+ if (log.isInfoEnabled()) {+ log.info(+ "Registered global circuit breaker {} for request type(s) {}",+ breaker,+ breaker.getRequestTypes());+ }+ }}/** List all registered circuit breakers for global context */diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerUtils.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerUtils.javanew file mode 100644index 00000000000..857230eba82--- /dev/null+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/CircuitBreakerUtils.java@@ -0,0 +1,73 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util.circuitbreaker;++import static org.apache.solr.common.params.CommonParams.FAILURE;+import static org.apache.solr.common.params.CommonParams.STATUS;++import java.lang.invoke.MethodHandles;+import java.util.List;+import java.util.function.Predicate;+import java.util.stream.Collectors;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.CollectionUtil;+import org.apache.solr.response.SolrQueryResponse;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class CircuitBreakerUtils {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ /**+ * @param trippedBreakers a (potentially null or empty) list of tripped breakers that may+ * short-circuit this request+ * @return true if any enabled "hard" (i.e. non-warnOnly) breakers were tripped; false otherwise+ */+ public static boolean reportErrorIfBreakersTripped(+ SolrQueryResponse rsp, List<CircuitBreaker> trippedBreakers) {+ if (CollectionUtil.isEmpty(trippedBreakers)) {+ return false;+ }++ final var warnOnlyBreakers =+ trippedBreakers.stream().filter(CircuitBreaker::isWarnOnly).collect(Collectors.toList());+ if (CollectionUtil.isNotEmpty(warnOnlyBreakers)) {+ if (log.isWarnEnabled()) {+ log.warn(+ "'warnOnly' circuit-breakers tripped for request: {}",+ CircuitBreakerRegistry.toErrorMessage(warnOnlyBreakers));+ }+ }++ final var hardBreakers =+ trippedBreakers.stream()+ .filter(Predicate.not(CircuitBreaker::isWarnOnly))+ .collect(Collectors.toList());+ if (CollectionUtil.isEmpty(hardBreakers)) {+ return false;+ }++ // Build the error message and add it to the response.+ String errorMessage = CircuitBreakerRegistry.toErrorMessage(hardBreakers);+ rsp.add(STATUS, FAILURE);+ rsp.setException(+ new SolrException(+ CircuitBreaker.getExceptionErrorCode(), "Circuit Breakers tripped " + errorMessage));+ return true;+ }+}diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.javaindex 0beb80f688f..64b786aa533 100644--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/LoadAverageCircuitBreaker.java@@ -20,6 +20,7 @@ package org.apache.solr.util.circuitbreaker;import java.lang.invoke.MethodHandles;import java.lang.management.ManagementFactory;import java.lang.management.OperatingSystemMXBean;+import java.util.Locale;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -96,4 +97,14 @@ public class LoadAverageCircuitBreaker extends CircuitBreaker {protected double calculateLiveLoadAverage() {return operatingSystemMXBean.getSystemLoadAverage();}++ @Override+ public String toString() {+ return String.format(+ Locale.ROOT,+ "%s(threshold=%f, warnOnly=%b)",+ getClass().getSimpleName(),+ loadAverageThreshold,+ isWarnOnly());+ }}diff --git a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.javaindex 65e352e5d72..3888283589e 100644--- a/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java+++ b/solr/core/src/java/org/apache/solr/util/circuitbreaker/MemoryCircuitBreaker.java@@ -21,6 +21,7 @@ import java.io.IOException;import java.lang.invoke.MethodHandles;import java.lang.management.ManagementFactory;import java.lang.management.MemoryMXBean;+import java.util.Locale;import org.apache.solr.util.RefCounted;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -131,4 +132,14 @@ public class MemoryCircuitBreaker extends CircuitBreaker {}}}++ @Override+ public String toString() {+ return String.format(+ Locale.ROOT,+ "%s(threshold=%d, warnOnly=%b)",+ getClass().getSimpleName(),+ heapMemoryThreshold,+ isWarnOnly());+ }}diff --git a/solr/core/src/test/org/apache/solr/util/TestCircuitBreakers.java b/solr/core/src/test/org/apache/solr/util/TestCircuitBreakers.javaindex d8cb3c9d17e..0f03a3dcb90 100644--- a/solr/core/src/test/org/apache/solr/util/TestCircuitBreakers.java+++ b/solr/core/src/test/org/apache/solr/util/TestCircuitBreakers.java@@ -22,11 +22,16 @@ import static org.hamcrest.CoreMatchers.containsString;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.ArrayList;+import java.util.Comparator;import java.util.List;+import java.util.Properties;+import java.util.Set;import java.util.concurrent.ExecutorService;import java.util.concurrent.Future;import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.Collectors;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.util.ExecutorUtil;@@ -128,6 +133,73 @@ public class TestCircuitBreakers extends SolrTestCaseJ4 {});}+ public void testCBsCanBeMarkedAsWarnOnly() throws Exception {+ removeAllExistingCircuitBreakers();++ final var warnCircuitBreaker = new MockCircuitBreaker(true);+ CircuitBreakerRegistry.registerGlobal(warnCircuitBreaker);++ // CB is warnOnly=false by default, so error is thrown.+ expectThrows(+ SolrException.class,+ () -> {+ h.query(req("name:\"john smith\""));+ });++ // When warnOnly=true is set, CB will trip but requests will not fail+ warnCircuitBreaker.setWarnOnly(true);+ final var tripList =+ h.getCore().getCircuitBreakerRegistry().checkTripped(SolrRequest.SolrRequestType.QUERY);+ assertEquals(1, tripList.size());+ assertEquals(warnCircuitBreaker, tripList.get(0));+ h.query(req("name:\"john smith\""));+ }++ public void testGlobalCBsCanBeParsedFromSystemProperties() {+ final var props = new Properties();+ props.setProperty("solr.circuitbreaker.query.mem", "12");+ props.setProperty("solr.circuitbreaker.update.loadavg", "3.4");+ props.setProperty("solr.circuitbreaker.update.cpu", "56");+ // Ensure 'warnOnly' property is picked up.+ props.setProperty("solr.circuitbreaker.update.cpu.warnonly", "true");+ System.setProperties(props);++ try {+ final var parsedBreakers =+ CircuitBreakerRegistry.parseCircuitBreakersFromProperties(h.getCoreContainer()).stream()+ .sorted(Comparator.comparing(breaker -> breaker.toString()))+ .collect(Collectors.toList());++ assertEquals(3, parsedBreakers.size());++ assertTrue(+ "Expected CPUCircuitBreaker, but got " + parsedBreakers.get(0).getClass().getName(),+ parsedBreakers.get(0) instanceof CPUCircuitBreaker);+ final var cpuBreaker = (CPUCircuitBreaker) parsedBreakers.get(0);+ assertEquals(56.0, cpuBreaker.getCpuUsageThreshold(), 0.1);+ assertEquals(true, cpuBreaker.isWarnOnly());+ assertEquals(Set.of(SolrRequest.SolrRequestType.UPDATE), cpuBreaker.getRequestTypes());++ assertTrue(+ "Expected LoadAverageCircuitBreaker, but got "+ + parsedBreakers.get(1).getClass().getName(),+ parsedBreakers.get(1) instanceof LoadAverageCircuitBreaker);+ final var loadAvgBreaker = (LoadAverageCircuitBreaker) parsedBreakers.get(1);+ assertEquals(3.4, loadAvgBreaker.getLoadAverageThreshold(), 0.1);+ assertEquals(false, loadAvgBreaker.isWarnOnly());+ assertEquals(Set.of(SolrRequest.SolrRequestType.UPDATE), loadAvgBreaker.getRequestTypes());++ assertTrue(+ "Expected MemoryCircuitBreaker, but got " + parsedBreakers.get(2).getClass().getName(),+ parsedBreakers.get(2) instanceof MemoryCircuitBreaker);+ final var memBreaker = (MemoryCircuitBreaker) parsedBreakers.get(2);+ assertEquals(false, memBreaker.isWarnOnly());+ assertEquals(Set.of(SolrRequest.SolrRequestType.QUERY), memBreaker.getRequestTypes());+ } finally {+ props.keySet().stream().forEach(k -> System.clearProperty((String) k));+ }+ }+@SuppressWarnings("resource")public void testCBAlwaysTripsInvalidErrorCodeSysProp() {synchronized (this) {diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/circuit-breakers.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/circuit-breakers.adocindex ac38f0f62b4..e8c4b5b115c 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/circuit-breakers.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/circuit-breakers.adoc@@ -23,9 +23,12 @@ resource configuration.== When To Use Circuit BreakersCircuit breakers should be used when the user wishes to trade request throughput for a higher Solr stability.If circuit breakers are enabled, requests may be rejected under the condition of high node duress with HTTP error code 429 'Too Many Requests'.-It is up to the client to handle this error and potentially build retry logic as this should be a transient situation.+Individual circuit breakers may also be enabled in a "warn only" mode.+"Warn only" breakers whose threshold has been exceeded are logged, but are not used to block or short-circuit requests.+This may be used as a way to tune circuit breaker thresholds without impacting traffic.+In a request to a sharded collection, the circuit breaker is only checked on the node handling the initial request, not for inter-node requests. It is therefore recommended to load balance client requests across Solr nodes to avoid hotspots.== Circuit Breaker Configurations@@ -45,10 +48,16 @@ Circuit breakers can be configured globally using environment variables, e.g. in|System Load Average |`SOLR_CIRCUITBREAKER_QUERY_LOADAVG`, `SOLR_CIRCUITBREAKER_UPDATE_LOADAVG` |`solr.circuitbreaker.query.loadavg`, `solr.circuitbreaker.update.loadavg`|===+Circuit breakers can be configured in "warn only" mode by adding a "warnonly"-suffixed environment variable or system property with a boolean value.+For example, you can enable a global CPU circuit breaker that rejects update requests when above 95% CPU load, by setting the following environment variable: `SOLR_CIRCUITBREAKER_UPDATE_CPU=95`.+If "warn only" mode is desired for this circuit breaker, the `SOLR_CIRCUITBREAKER_UPDATE_CPU_WARNONLY=true` environment variable or `solr.circuitbreaker.update.cpu.warnonly=true` system property could be set.=== Per Collection Circuit Breakers-Circuit breakers are configured as independent `<circuitBreaker>` entries in `solrconfig.xml` as shown in the below examples. By default, only search requests are affected.+Circuit breakers are configured as independent `<circuitBreaker>` entries in `solrconfig.xml` as shown in the below examples.+By default, only search requests are affected.+The syntax and semantics of available configuration options differs slightly based on the type of circuit breaker being configured.+However all circuit breakers support a boolean "warnOnly" setting" that can be used to set the circuit breaker into "warn only" mode (e.g. `<bool name="warnOnly">true</bool>`)[TIP]====