Apache Solr open-source search software
SOLR-16879: throttle BACKUPCORE, RESTORECORE, SPLIT (#1864)
Pierre Salagnac585ba9ba5f9a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex ce74fc3b1cb..e7e31b686d5 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -107,6 +107,10 @@ Improvements* SOLR-16927: Allow SolrClientCache clients to use Jetty HTTP2 clients (Alex Deparvu, David Smiley)+* SOLR-16879: Limit the number of concurrent expensive core admin operations by running them in a+ dedicated thread pool. Backup, Restore and Split are expensive operations.+ (Pierre Salagnac, David Smiley)+Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.javaindex bfd939c43b4..788fb494757 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java@@ -28,6 +28,11 @@ import org.apache.solr.handler.api.V2ApiUtils;class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {+ @Override+ public boolean isExpensive() {+ return true;+ }+@Overridepublic void execute(CoreAdminHandler.CallInfo it) throws Exception {final SolrParams params = it.req.getParams();diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.javaindex 94212c883cf..f9819dcaf80 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java@@ -97,6 +97,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNaprotected final CoreAdminAsyncTracker coreAdminAsyncTracker;public static String RESPONSE_STATUS = "STATUS";+public static String RESPONSE_MESSAGE = "msg";public static String OPERATION_RESPONSE = "response";@@ -129,13 +130,21 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa@Overridepublic void initializeMetrics(SolrMetricsContext parentContext, String scope) {super.initializeMetrics(parentContext, scope);- coreAdminAsyncTracker.parallelExecutor =+ coreAdminAsyncTracker.standardExecutor =MetricUtils.instrumentedExecutorService(- coreAdminAsyncTracker.parallelExecutor,+ coreAdminAsyncTracker.standardExecutor,this,solrMetricsContext.getMetricRegistry(),SolrMetricManager.mkName("parallelCoreAdminExecutor", getCategory().name(), scope, "threadPool"));++ coreAdminAsyncTracker.expensiveExecutor =+ MetricUtils.instrumentedExecutorService(+ coreAdminAsyncTracker.expensiveExecutor,+ this,+ solrMetricsContext.getMetricRegistry(),+ SolrMetricManager.mkName(+ "parallelCoreExpensiveAdminExecutor", getCategory().name(), scope, "threadPool"));}@Override@@ -196,8 +205,6 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa}// boolean doPersist = false;final String taskId = req.getParams().get(CommonAdminParams.ASYNC);- final CoreAdminAsyncTracker.TaskObject taskObject =- new CoreAdminAsyncTracker.TaskObject(taskId);// Pick the actionfinal String action = req.getParams().get(ACTION, STATUS.toString()).toLowerCase(Locale.ROOT);@@ -220,13 +227,16 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNaif (taskId == null) {callInfo.call();} else {- coreAdminAsyncTracker.submitAsyncTask(- taskObject,- action,+ Callable<SolrQueryResponse> task =() -> {callInfo.call();return callInfo.rsp;- });+ };++ var taskObject =+ new CoreAdminAsyncTracker.TaskObject(taskId, action, op.isExpensive(), task);++ coreAdminAsyncTracker.submitAsyncTask(taskObject);}} finally {rsp.setHttpCaching(false);@@ -332,7 +342,7 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa/** Method to ensure shutting down of the ThreadPool Executor. */public void shutdown() {- if (coreAdminAsyncTracker.parallelExecutor != null) coreAdminAsyncTracker.shutdown();+ coreAdminAsyncTracker.shutdown();}private static final Map<String, CoreAdminOp> opMap = new HashMap<>();@@ -393,6 +403,11 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa}public interface CoreAdminOp {++ default boolean isExpensive() {+ return false;+ }+/*** @param it request/response object* <p>If the request is invalid throw a SolrException with@@ -411,10 +426,18 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNapublic static final String FAILED = "failed";public final Map<String, Map<String, TaskObject>> requestStatusMap;- private ExecutorService parallelExecutor =+ // Executor for all standard tasks (the ones that are not flagged as expensive)+ // We always keep 50 live threads+ private ExecutorService standardExecutor =ExecutorUtil.newMDCAwareFixedThreadPool(50, new SolrNamedThreadFactory("parallelCoreAdminAPIBaseExecutor"));+ // Executor for expensive tasks+ // We keep the number number of max threads very low to have throttling for expensive tasks+ private ExecutorService expensiveExecutor =+ ExecutorUtil.newMDCAwareCachedThreadPool(+ 5, new SolrNamedThreadFactory("parallelCoreAdminAPIExpensiveExecutor"));+public CoreAdminAsyncTracker() {HashMap<String, Map<String, TaskObject>> map = new HashMap<>(3, 1.0f);map.put(RUNNING, Collections.synchronizedMap(new LinkedHashMap<>()));@@ -424,36 +447,41 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa}public void shutdown() {- ExecutorUtil.shutdownAndAwaitTermination(parallelExecutor);+ ExecutorUtil.shutdownAndAwaitTermination(standardExecutor);+ ExecutorUtil.shutdownAndAwaitTermination(expensiveExecutor);}public Map<String, TaskObject> getRequestStatusMap(String key) {return requestStatusMap.get(key);}- public void submitAsyncTask(- TaskObject taskObject, String action, Callable<SolrQueryResponse> task)- throws SolrException {+ public void submitAsyncTask(TaskObject taskObject) throws SolrException {ensureTaskIdNotInUse(taskObject.taskId);addTask(RUNNING, taskObject);+ Runnable command =+ () -> {+ boolean exceptionCaught = false;+ try {+ final SolrQueryResponse response = taskObject.task.call();+ taskObject.setRspObject(response);+ taskObject.setOperationRspObject(response);+ } catch (Exception e) {+ exceptionCaught = true;+ taskObject.setRspObjectFromException(e);+ } finally {+ finishTask(taskObject, !exceptionCaught);+ }+ };+try {MDC.put("CoreAdminHandler.asyncId", taskObject.taskId);- MDC.put("CoreAdminHandler.action", action);- parallelExecutor.execute(- () -> {- boolean exceptionCaught = false;- try {- final SolrQueryResponse response = task.call();- taskObject.setRspObject(response);- taskObject.setOperationRspObject(response);- } catch (Exception e) {- exceptionCaught = true;- taskObject.setRspObjectFromException(e);- } finally {- finishTask(taskObject, !exceptionCaught);- }- });+ MDC.put("CoreAdminHandler.action", taskObject.action);+ if (taskObject.expensive) {+ expensiveExecutor.execute(command);+ } else {+ standardExecutor.execute(command);+ }} finally {MDC.remove("CoreAdminHandler.asyncId");MDC.remove("CoreAdminHandler.action");@@ -503,12 +531,19 @@ public class CoreAdminHandler extends RequestHandlerBase implements PermissionNa* response (if available).*/public static class TaskObject {- public String taskId;+ final String taskId;+ final String action;+ final boolean expensive;+ final Callable<SolrQueryResponse> task;public String rspInfo;public Object operationRspInfo;- public TaskObject(String taskId) {+ public TaskObject(+ String taskId, String action, boolean expensive, Callable<SolrQueryResponse> task) {this.taskId = taskId;+ this.action = action;+ this.expensive = expensive;+ this.task = task;}public String getRspObject() {diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.javaindex bf82ae38c4e..31251759c76 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java@@ -309,6 +309,12 @@ public enum CoreAdminOperation implements CoreAdminOp {return log;}+ @Override+ public boolean isExpensive() {+ // delegates this to the actual implementation+ return fun.isExpensive();+ }+/*** Returns the core status for a particular core.*diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.javaindex b7f97469b49..c95af84d3d7 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/RestoreCoreOp.java@@ -24,6 +24,12 @@ import org.apache.solr.handler.admin.api.RestoreCoreAPI;import org.apache.solr.handler.api.V2ApiUtils;class RestoreCoreOp implements CoreAdminHandler.CoreAdminOp {++ @Override+ public boolean isExpensive() {+ return true;+ }+@Overridepublic void execute(CoreAdminHandler.CallInfo it) throws Exception {final SolrParams params = it.req.getParams();diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.javaindex 3b66794c5d5..fd29d31fcd0 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/SplitOp.java@@ -70,6 +70,11 @@ class SplitOp implements CoreAdminHandler.CoreAdminOp {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ @Override+ public boolean isExpensive() {+ return true;+ }+@Overridepublic void execute(CoreAdminHandler.CallInfo it) throws Exception {SolrParams params = it.req.getParams();diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/BackupCoreAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/BackupCoreAPI.javaindex 8f57c58f2d9..580f2849b2f 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/api/BackupCoreAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/BackupCoreAPI.java@@ -56,6 +56,11 @@ public class BackupCoreAPI extends CoreAdminAPIBase {super(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse);}+ @Override+ boolean isExpensive() {+ return true;+ }+@POST@Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2})@PermissionName(COLL_EDIT_PERM)diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreAdminAPIBase.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreAdminAPIBase.javaindex 6a4130e9132..69fe4c993ce 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/api/CoreAdminAPIBase.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CoreAdminAPIBase.java@@ -16,6 +16,7 @@*/package org.apache.solr.handler.admin.api;+import java.util.concurrent.Callable;import java.util.function.Supplier;import org.apache.solr.api.JerseyResource;import org.apache.solr.client.api.model.SolrJerseyResponse;@@ -52,6 +53,14 @@ public abstract class CoreAdminAPIBase extends JerseyResource {this.rsp = rsp;}+ /**+ * Can be overridden by operations that are expensive, so we don't execute too many of them+ * concurrently.+ */+ boolean isExpensive() {+ return false;+ }+/*** Wraps the subclasses logic with extra bookkeeping logic.*@@ -79,22 +88,23 @@ public abstract class CoreAdminAPIBase extends JerseyResource {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Core container instance missing");}- final CoreAdminHandler.CoreAdminAsyncTracker.TaskObject taskObject =- new CoreAdminHandler.CoreAdminAsyncTracker.TaskObject(taskId);MDCLoggingContext.setCoreName(coreName);TraceUtils.setDbInstance(req, coreName);if (taskId == null) {return supplier.get();} else {- coreAdminAsyncTracker.submitAsyncTask(- taskObject,- actionName,+ Callable<SolrQueryResponse> task =() -> {T response = supplier.get();V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, response);return rsp;- });+ };++ final CoreAdminHandler.CoreAdminAsyncTracker.TaskObject taskObject =+ new CoreAdminHandler.CoreAdminAsyncTracker.TaskObject(+ taskId, actionName, isExpensive(), task);+ coreAdminAsyncTracker.submitAsyncTask(taskObject);}} catch (CoreAdminAPIBaseException e) {throw e.trueException;diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCoreAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCoreAPI.javaindex 03358880450..7e8ccf29862 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCoreAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCoreAPI.java@@ -61,6 +61,11 @@ public class RestoreCoreAPI extends CoreAdminAPIBase {super(coreContainer, coreAdminAsyncTracker, solrQueryRequest, solrQueryResponse);}+ @Override+ boolean isExpensive() {+ return true;+ }+@POST@Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2})@PermissionName(CORE_EDIT_PERM)