Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15164: Implement Task Management Interface (#76)

Atri Sharmaaa4c4d3b9cba
diff --git a/solr/core/src/java/org/apache/solr/core/CancellableQueryTracker.java b/solr/core/src/java/org/apache/solr/core/CancellableQueryTracker.java
new file mode 100644
index 00000000000..8e0d2c5dd49
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/CancellableQueryTracker.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import org.apache.solr.client.solrj.util.Cancellable;
+import org.apache.solr.search.CancellableCollector;
+import org.apache.solr.request.SolrQueryRequest;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.apache.solr.common.params.CommonParams.QUERY_UUID;
+
+/**
+ * Tracks metadata for active queries and provides methods for access
+ */
+public class CancellableQueryTracker {
+ //TODO: This needs to become a time aware storage model
+ private final Map<String, Cancellable> activeCancellableQueries = new ConcurrentHashMap<>();
+ private final Map<String, String> activeQueriesGenerated = new ConcurrentHashMap<>();
+
+ /** Generates a UUID for the given query or if the user provided a UUID
+ * for this query, uses that.
+ */
+ public String generateQueryID(SolrQueryRequest req) {
+ String queryID;
+ String customQueryUUID = req.getParams().get(QUERY_UUID, null);
+
+ if (customQueryUUID != null) {
+ queryID = customQueryUUID;
+ } else {
+ //TODO: Use a different generator
+ queryID = UUID.randomUUID().toString();
+ }
+
+ if (activeQueriesGenerated.containsKey(queryID)) {
+ if (customQueryUUID != null) {
+ throw new IllegalArgumentException("Duplicate query UUID given");
+ } else {
+ while (activeQueriesGenerated.get(queryID) != null) {
+ queryID = UUID.randomUUID().toString();
+ }
+ }
+ }
+
+ activeQueriesGenerated.put(queryID, req.getHttpSolrCall().getReq().getQueryString());
+
+ return queryID;
+ }
+
+ public void releaseQueryID(String inputQueryID) {
+ if (inputQueryID == null) {
+ return;
+ }
+
+ activeQueriesGenerated.remove(inputQueryID);
+ }
+
+ public boolean isQueryIdActive(String queryID) {
+ return activeQueriesGenerated.containsKey(queryID);
+ }
+
+ public void addShardLevelActiveQuery(String queryID, CancellableCollector collector) {
+ if (queryID == null) {
+ return;
+ }
+
+ activeCancellableQueries.put(queryID, collector);
+ }
+
+ public Cancellable getCancellableTask(String queryID) {
+ if (queryID == null) {
+ throw new IllegalArgumentException("Input queryID is null");
+ }
+
+ return activeCancellableQueries.get(queryID);
+ }
+
+ public void removeCancellableQuery(String queryID) {
+ if (queryID == null) {
+ // Some components, such as CaffeineCache, use the searcher to fire internal queries which are not tracked
+ return;
+ }
+
+ activeCancellableQueries.remove(queryID);
+ }
+
+ public Iterator<Map.Entry<String, String>> getActiveQueriesGenerated() {
+ return activeQueriesGenerated.entrySet().iterator();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index be89bf64ac8..5857295d94f 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -199,6 +199,8 @@ public final class SolrCore implements SolrInfoBean, Closeable {
*/
public final UUID uniqueId = UUID.randomUUID();
+ private final CancellableQueryTracker cancellableQueryTracker = new CancellableQueryTracker();
+
private boolean isReloaded = false;
private final CoreDescriptor coreDescriptor;
@@ -3244,6 +3246,10 @@ public final class SolrCore implements SolrInfoBean, Closeable {
return blobRef;
}
+ public CancellableQueryTracker getCancellableQueryTracker() {
+ return cancellableQueryTracker;
+ }
+
/**
* Run an arbitrary task in it's own thread. This is an expert option and is
* a method you should use with great care. It would be bad to run something that never stopped
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListComponent.java
new file mode 100644
index 00000000000..9856e99d146
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListComponent.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/** List the active tasks that can be cancelled */
+public class ActiveTasksListComponent extends SearchComponent {
+ public static final String COMPONENT_NAME = "activetaskslist";
+
+ private boolean shouldProcess;
+
+ @Override
+ public void prepare(ResponseBuilder rb) throws IOException {
+ if (rb.isTaskListRequest()) {
+ shouldProcess = true;
+ }
+ }
+
+ @Override
+ public void process(ResponseBuilder rb) {
+ if (!shouldProcess) {
+ return;
+ }
+
+ if (rb.getTaskStatusCheckUUID() != null) {
+ boolean isActiveOnThisShard = rb.req.getCore().getCancellableQueryTracker().isQueryIdActive(rb.getTaskStatusCheckUUID());
+
+ rb.rsp.add("taskStatus", isActiveOnThisShard);
+ return;
+ }
+
+ rb.rsp.add("taskList", (MapWriter) ew -> {
+ Iterator<Map.Entry<String, String>> iterator = rb.req.getCore().getCancellableQueryTracker().getActiveQueriesGenerated();
+
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+ ew.put(entry.getKey(), entry.getValue());
+ }
+ });
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ if (!shouldProcess) {
+ return;
+ }
+
+ NamedList<String> resultList = new NamedList<>();
+
+ for (ShardResponse r : sreq.responses) {
+
+ if (rb.getTaskStatusCheckUUID() != null) {
+ boolean isTaskActiveOnShard = r.getSolrResponse().getResponse().getBooleanArg("taskStatus");
+
+ if (isTaskActiveOnShard) {
+ rb.rsp.getValues().add("taskStatus", "id:" + rb.getTaskStatusCheckUUID() + ", status: active");
+ return;
+ } else {
+ continue;
+ }
+ }
+
+ LinkedHashMap<String, String> result = (LinkedHashMap<String, String>) r.getSolrResponse()
+ .getResponse().get("taskList");
+
+ Iterator<Map.Entry<String, String>> iterator = result.entrySet().iterator();
+
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+
+ resultList.add(entry.getKey(), entry.getValue());
+ }
+ }
+
+ if (rb.getTaskStatusCheckUUID() != null) {
+ // We got here with the specific taskID check being specified -- this means that the taskID was not
+ // found in active tasks on any shard
+ rb.rsp.getValues().add("taskStatus", "id:" + rb.getTaskStatusCheckUUID() + ", status: inactive");
+ return;
+ }
+
+ rb.rsp.getValues().add("taskList", resultList);
+ }
+
+ @Override
+ public String getDescription() {
+ return "Responsible for listing all active cancellable tasks and also supports checking the status of " +
+ "a particular task";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.OTHER;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListHandler.java b/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListHandler.java
new file mode 100644
index 00000000000..8df15a5eaa3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/ActiveTasksListHandler.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.security.PermissionNameProvider;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.solr.common.params.CommonParams.TASK_CHECK_UUID;
+
+/**
+ * Handles request for listing all active cancellable tasks
+ */
+public class ActiveTasksListHandler extends TaskManagementHandler {
+ // This can be a parent level member but we keep it here to allow future handlers to have
+ // a custom list of components
+ private List<SearchComponent> components;
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ Map<String, String> extraParams = null;
+ ResponseBuilder rb = buildResponseBuilder(req, rsp, getComponentsList());
+
+ rb.setIsTaskListRequest(true);
+
+ String taskStatusCheckUUID = req.getParams().get(TASK_CHECK_UUID, null);
+
+ if (taskStatusCheckUUID != null) {
+ if (rb.isDistrib) {
+ extraParams = new HashMap<>();
+
+ extraParams.put(TASK_CHECK_UUID, taskStatusCheckUUID);
+ }
+
+ rb.setTaskStatusCheckUUID(taskStatusCheckUUID);
+ }
+
+ processRequest(req, rb, extraParams);
+ }
+
+ @Override
+ public String getDescription() {
+ return "activetaskslist";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.ADMIN;
+ }
+
+ @Override
+ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) {
+ return PermissionNameProvider.Name.READ_PERM;
+ }
+
+ @Override
+ public SolrRequestHandler getSubHandler(String path) {
+ if (path.startsWith("/tasks/list")) {
+ return this;
+ }
+
+ return null;
+ }
+
+ @Override
+ public Boolean registerV2() {
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Collection<Api> getApis() {
+ return ApiBag.wrapRequestHandlers(this, "core.tasks.list");
+ }
+
+ private List<SearchComponent> getComponentsList() {
+ if (components == null) {
+ components = buildComponentsList();
+ }
+
+ return components;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationComponent.java
new file mode 100644
index 00000000000..98e0355157e
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationComponent.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.client.solrj.util.Cancellable;
+
+import java.io.IOException;
+
+/** Responsible for handling query cancellation requests */
+public class QueryCancellationComponent extends SearchComponent {
+ public static final String COMPONENT_NAME = "querycancellation";
+
+ private boolean shouldProcess;
+
+ @Override
+ public void prepare(ResponseBuilder rb) throws IOException
+ {
+ if (rb.isCancellation()) {
+ shouldProcess = true;
+ }
+ }
+
+ @Override
+ public void process(ResponseBuilder rb) {
+ if (!shouldProcess) {
+ return;
+ }
+
+ String cancellationUUID = rb.getCancellationUUID();
+
+ if (cancellationUUID == null) {
+ throw new RuntimeException("Null query UUID seen");
+ }
+
+ Cancellable cancellableTask = rb.req.getCore().getCancellableQueryTracker().getCancellableTask(cancellationUUID);
+
+ if (cancellableTask != null) {
+ cancellableTask.cancel();
+ rb.rsp.add("cancellationResult", "success");
+ } else {
+ rb.rsp.add("cancellationResult", "not found");
+ }
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
+ if (!shouldProcess) {
+ return;
+ }
+
+ boolean queryFound = false;
+
+ for (ShardResponse r : sreq.responses) {
+
+ String cancellationResult = (String) r.getSolrResponse()
+ .getResponse().get("cancellationResult");
+
+ if (cancellationResult.equalsIgnoreCase("success")) {
+ queryFound = true;
+
+ break;
+ }
+ }
+
+ // If any shard sees the query as present, then we mark the query as successfully cancelled. If no shard found
+ // the query, then that can denote that the query was not found. This is important since the query cancellation
+ // request is broadcast to all shards, and the query might have completed on some shards but not on others
+
+ if(queryFound) {
+ rb.rsp.getValues().add("status", "Query with queryID " + rb.getCancellationUUID() +
+ " cancelled successfully");
+ rb.rsp.getValues().add("responseCode", 200 /* HTTP OK */);
+ } else {
+ rb.rsp.getValues().add("status", "Query with queryID " + rb.getCancellationUUID() +
+ " not found");
+ rb.rsp.getValues().add("responseCode", 404 /* HTTP NOT FOUND */);
+ }
+ }
+
+ @Override
+ public String getDescription() {
+ return "Supports cancellation of queries which are cancellable";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.OTHER;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationHandler.java b/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationHandler.java
new file mode 100644
index 00000000000..277bad51959
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryCancellationHandler.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.api.Api;
+import org.apache.solr.api.ApiBag;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.AuthorizationContext;
+import org.apache.solr.security.PermissionNameProvider;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.solr.common.params.CommonParams.QUERY_UUID;
+
+/**
+ * Handles requests for query cancellation for cancellable queries
+ */
+public class QueryCancellationHandler extends TaskManagementHandler {
+ // This can be a parent level member but we keep it here to allow future handlers to have
+ // a custom list of components
+ private List<SearchComponent> components;
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ ResponseBuilder rb = buildResponseBuilder(req, rsp, getComponentsList());
+ Map<String, String> extraParams = null;
+
+ rb.setCancellation(true);
+
+ String cancellationUUID = req.getParams().get(QUERY_UUID, null);
+
+ if (cancellationUUID == null) {
+ throw new IllegalArgumentException("Query cancellation was requested but no query UUID for cancellation was given");
+ }
+
+ if (rb.isDistrib) {
+ extraParams = new HashMap<>();
+
+ extraParams.put(QUERY_UUID, cancellationUUID);
+ }
+
+ // Let this be visible to handleResponses in the handling component
+ rb.setCancellationUUID(cancellationUUID);
+
+ processRequest(req, rb, extraParams);
+ }
+
+ @Override
+ public String getDescription() {
+ return "Cancel queries";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.ADMIN;
+ }
+
+ @Override
+ public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) {
+ return PermissionNameProvider.Name.READ_PERM;
+ }
+
+ @Override
+ public SolrRequestHandler getSubHandler(String path) {
+ if (path.startsWith("/tasks/cancel")) {
+ return this;
+ }
+
+ return null;
+ }
+
+ @Override
+ public Boolean registerV2() {
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Collection<Api> getApis() {
+ return ApiBag.wrapRequestHandlers(this, "core.tasks.cancel");
+ }
+
+ private List<SearchComponent> getComponentsList() {
+ if (components == null) {
+ components = buildComponentsList();
+ }
+
+ return components;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 6622c30ad4a..2f5f8cf6791 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -52,6 +52,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.InPlaceMergeSorter;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -114,6 +115,8 @@ import org.apache.solr.util.SolrPluginUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.params.CommonParams.QUERY_UUID;
+
/**
* TODO!
@@ -129,7 +132,6 @@ public class QueryComponent extends SearchComponent
@Override
public void prepare(ResponseBuilder rb) throws IOException
{
-
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
@@ -137,6 +139,15 @@ public class QueryComponent extends SearchComponent
}
SolrQueryResponse rsp = rb.rsp;
+ if (rb.isDistrib) {
+ boolean isCancellableQuery = params.getBool(CommonParams.IS_QUERY_CANCELLABLE, false);
+
+ if (isCancellableQuery) {
+ // Generate Query ID
+ rb.queryID = generateQueryID(req);
+ }
+ }
+
// Set field flags
ReturnFields returnFields = new SolrReturnFields( req );
rsp.setReturnFields( returnFields );
@@ -355,6 +366,25 @@ public class QueryComponent extends SearchComponent
cmd.setTimeAllowed(timeAllowed);
cmd.setMinExactCount(getMinExactCount(params));
+ boolean isCancellableQuery = params.getBool(CommonParams.IS_QUERY_CANCELLABLE, false);
+
+ if (isCancellableQuery) {
+ // Set the queryID for the searcher to consume
+ String queryID = params.get(ShardParams.QUERY_ID);
+
+ cmd.setQueryCancellable(true);
+
+ if (queryID == null) {
+ if (rb.isDistrib) {
+ throw new IllegalStateException("QueryID is null for distributed query");
+ }
+
+ queryID = rb.queryID;
+ }
+
+ cmd.setQueryID(queryID);
+ }
+
req.getContext().put(SolrIndexSearcher.STATS_SOURCE, statsCache.get(req));
QueryResult result = new QueryResult();
@@ -627,6 +657,12 @@ public class QueryComponent extends SearchComponent
if (rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
return;
}
+
+ if (rb.queryID != null) {
+ // Remove the current queryID from the active list
+ rb.req.getCore().getCancellableQueryTracker().releaseQueryID(rb.queryID);
+ }
+
if (rb.grouping()) {
groupedFinishStage(rb);
} else {
@@ -1523,6 +1559,25 @@ public class QueryComponent extends SearchComponent
doPrefetch(rb);
}
+ private static String generateQueryID(SolrQueryRequest req) {
+ ZkController zkController = req.getCore().getCoreContainer().getZkController();
+ String nodeName = req.getCore().getCoreContainer().getHostName();
+
+ if (zkController != null) {
+ nodeName = zkController.getNodeName();
+ }
+
+ final String localQueryID = req.getCore().getCancellableQueryTracker().generateQueryID(req);
+
+ boolean isCustom = req.getParams().get(QUERY_UUID, null) != null;
+
+ if (!isCustom && nodeName != null) {
+ return nodeName.concat(localQueryID);
+ }
+
+ return localQueryID;
+ }
+
/**
* Fake scorer for a single document
*
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
index cf4719817f9..ca74b840e99 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
@@ -65,12 +65,20 @@ public class ResponseBuilder
public boolean doAnalytics;
public MergeStrategy mergeFieldHandler;
+ public String queryID;
+
private boolean needDocList = false;
private boolean needDocSet = false;
private int fieldFlags = 0;
//private boolean debug = false;
private boolean debugTimings, debugQuery, debugResults, debugTrack;
+ private boolean isCancellation;
+ private String cancellationUUID;
+
+ private String taskStatusCheckUUID;
+ private boolean isTaskListRequest;
+
private QParser qparser = null;
private String queryString = null;
private Query query = null;
@@ -509,4 +517,36 @@ public class ResponseBuilder
public boolean isOlapAnalytics() {
return this._isOlapAnalytics;
}
+
+ public void setCancellation(boolean isCancellation) {
+ this.isCancellation = isCancellation;
+ }
+
+ public boolean isCancellation() {
+ return isCancellation;
+ }
+
+ public void setIsTaskListRequest(boolean isTaskListRequest) {
+ this.isTaskListRequest = isTaskListRequest;
+ }
+
+ public boolean isTaskListRequest() {
+ return isTaskListRequest;
+ }
+
+ public void setCancellationUUID(String queryID) {
+ this.cancellationUUID = queryID;
+ }
+
+ public String getCancellationUUID() {
+ return cancellationUUID;
+ }
+
+ public void setTaskStatusCheckUUID(String taskUUID) {
+ this.taskStatusCheckUUID = taskUUID;
+ }
+
+ public String getTaskStatusCheckUUID() {
+ return taskStatusCheckUUID;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index ea4f6f879fd..7d1f5600245 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -236,7 +236,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,
return result;
}
- private ShardHandler getAndPrepShardHandler(SolrQueryRequest req, ResponseBuilder rb) {
+ public ShardHandler getAndPrepShardHandler(SolrQueryRequest req, ResponseBuilder rb) {
ShardHandler shardHandler = null;
CoreContainer cc = req.getCore().getCoreContainer();
@@ -449,6 +449,9 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware,
params.set(ShardParams.SHARDS_PURPOSE, sreq.purpose);
params.set(ShardParams.SHARD_URL, shard); // so the shard knows what was asked
params.set(CommonParams.OMIT_HEADER, false);
+
+ // Distributed request -- need to send queryID as a part of the distributed request
+ params.setNonNull(ShardParams.QUERY_ID, rb.queryID);
if (rb.requestInfo != null) {
// we could try and detect when this is needed, but it could be tricky
params.set("NOW", Long.toString(rb.requestInfo.getNOW().getTime()));
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java b/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java
new file mode 100644
index 00000000000..8d935731946
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/TaskManagementHandler.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.security.PermissionNameProvider;
+import org.apache.solr.util.plugin.SolrCoreAware;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+import static org.apache.solr.common.params.CommonParams.PATH;
+
+/**
+ * Abstract class which serves as the root of all task managing handlers
+ */
+public abstract class TaskManagementHandler extends RequestHandlerBase implements SolrCoreAware, PermissionNameProvider {
+ private ShardHandlerFactory shardHandlerFactory;
+
+ @Override
+ public void inform(SolrCore core) {
+ this.shardHandlerFactory = core.getCoreContainer().getShardHandlerFactory();
+ }
+
+ /**
+ * Process the actual request.
+ * extraParams is required for allowing sub handlers to pass in custom parameters to be put in the
+ * outgoing shard request
+ */
+ protected void processRequest(SolrQueryRequest req, ResponseBuilder rb, Map<String, String> extraParams) throws IOException {
+ ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
+ List<SearchComponent> components = rb.components;
+
+ shardHandler.prepDistributed(rb);
+
+ for(SearchComponent c : components) {
+ c.prepare(rb);
+ }
+
+ if (!rb.isDistrib) {
+ for (SearchComponent component : components) {
+ component.process(rb);
+ }
+ } else {
+ ShardRequest sreq = new ShardRequest();
+
+ // Distribute to all shards
+ sreq.shards = rb.shards;
+ sreq.actualShards = sreq.shards;
+
+ sreq.responses = new ArrayList<>(sreq.actualShards.length);
+ rb.finished = new ArrayList<>();
+
+ for (String shard : sreq.actualShards) {
+ ModifiableSolrParams params = new ModifiableSolrParams(sreq.params);
+ String reqPath = (String) req.getContext().get(PATH);
+
+ params.set(CommonParams.QT, reqPath);
+ params.remove(ShardParams.SHARDS); // not a top-level request
+ params.set(DISTRIB, "false"); // not a top-level request
+ params.remove("indent");
+ params.remove(CommonParams.HEADER_ECHO_PARAMS);
+ params.set(ShardParams.IS_SHARD, true); // a sub (shard) request
+ params.set(ShardParams.SHARDS_PURPOSE, sreq.purpose);
+ params.set(ShardParams.SHARD_URL, shard); // so the shard knows what was asked
+ params.set(CommonParams.OMIT_HEADER, false);
+
+ if (extraParams != null) {
+ for(Map.Entry<String, String> entry : extraParams.entrySet()) {
+ params.set(entry.getKey(), entry.getValue());
+ }
+ }
+
+ shardHandler.submit(sreq, shard, params);
+ }
+
+ ShardResponse srsp = shardHandler.takeCompletedOrError();
+
+ if (srsp.getException() != null) {
+ shardHandler.cancelAll();
+ if (srsp.getException() instanceof SolrException) {
+ throw (SolrException) srsp.getException();
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException());
+ }
+ }
+
+ rb.finished.add(srsp.getShardRequest());
+
+ for (SearchComponent c : components) {
+ c.handleResponses(rb, srsp.getShardRequest());
+ }
+ }
+ }
+
+ public static List<SearchComponent> buildComponentsList() {
+ List<SearchComponent> components = new ArrayList<>(2);
+
+ QueryCancellationComponent component = new QueryCancellationComponent();
+ components.add(component);
+
+ ActiveTasksListComponent activeTasksListComponent = new ActiveTasksListComponent();
+ components.add(activeTasksListComponent);
+
+ return components;
+ }
+
+ public static ResponseBuilder buildResponseBuilder(SolrQueryRequest req, SolrQueryResponse rsp,
+ List<SearchComponent> components) {
+ CoreContainer cc = req.getCore().getCoreContainer();
+ boolean isZkAware = cc.isZooKeeperAware();
+
+ ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
+
+ rb.isDistrib = req.getParams().getBool(DISTRIB, isZkAware);
+
+ return rb;
+ }
+}
+
diff --git a/solr/core/src/java/org/apache/solr/search/CancellableCollector.java b/solr/core/src/java/org/apache/solr/search/CancellableCollector.java
new file mode 100644
index 00000000000..b08e5b07014
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/search/CancellableCollector.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.FilterLeafCollector;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.solr.client.solrj.util.Cancellable;
+
+/** Allows a query to be cancelled */
+public class CancellableCollector implements Collector, Cancellable {
+
+ /** Thrown when a query gets cancelled */
+ public static class QueryCancelledException extends RuntimeException {}
+
+ private final Collector collector;
+ private final AtomicBoolean isQueryCancelled;
+
+ public CancellableCollector(Collector collector) {
+ Objects.requireNonNull(collector, "Internal collector not provided but wrapper collector accessed");
+
+ this.collector = collector;
+ this.isQueryCancelled = new AtomicBoolean();
+ }
+
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+
+ if (isQueryCancelled.get()) {
+ throw new QueryCancelledException();
+ }
+
+ return new FilterLeafCollector(collector.getLeafCollector(context)) {
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (isQueryCancelled.get()) {
+ throw new QueryCancelledException();
+ }
+ in.collect(doc);
+ }
+ };
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return collector.scoreMode();
+ }
+
+ @Override
+ public void cancel() {
+ isQueryCancelled.compareAndSet(false, true);
+ }
+
+ public Collector getInternalCollector() {
+ return collector;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/search/QueryCommand.java b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
index e0b42569df0..6c77ef4d2d8 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryCommand.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
@@ -29,6 +29,8 @@ import org.apache.lucene.search.Sort;
public class QueryCommand {
private Query query;
+ private String queryID;
+ private boolean isQueryCancellable;
private List<Query> filterList;
private DocSet filter;
private Sort sort;
@@ -229,4 +231,16 @@ public class QueryCommand {
}
}
+ public void setQueryID(String queryID) {
+ this.queryID = queryID;
+ }
+
+ public String getQueryID() {
+ return queryID;
+ }
+
+ public void setQueryCancellable(boolean isQueryCancellable) { this.isQueryCancellable = isQueryCancellable; }
+
+ public boolean isQueryCancellable() { return isQueryCancellable; }
+
}
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index ae3ba08e8bb..e1c9c294a1e 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -206,9 +206,17 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
collector = postFilter;
}
+ if (cmd.isQueryCancellable()) {
+ collector = new CancellableCollector(collector);
+
+ // Add this to the local active queries map
+ core.getCancellableQueryTracker().addShardLevelActiveQuery(cmd.getQueryID(), (CancellableCollector) collector);
+ }
+
try {
super.search(query, collector);
- } catch (TimeLimitingCollector.TimeExceededException | ExitableDirectoryReader.ExitingReaderException x) {
+ } catch (TimeLimitingCollector.TimeExceededException | ExitableDirectoryReader.ExitingReaderException |
+ CancellableCollector.QueryCancelledException x) {
log.warn("Query: [{}]; ", query, x);
qr.setPartialResults(true);
} catch (EarlyTerminatingCollectorException etce) {
@@ -220,10 +228,15 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
if (earlyTerminatingSortingCollector != null) {
qr.setSegmentTerminatedEarly(earlyTerminatingSortingCollector.terminatedEarly());
}
+
+ if (cmd.isQueryCancellable()) {
+ core.getCancellableQueryTracker().removeCancellableQuery(cmd.getQueryID());
+ }
}
if (collector instanceof DelegatingCollector) {
((DelegatingCollector) collector).finish();
}
+
return collector;
}
diff --git a/solr/core/src/resources/ImplicitPlugins.json b/solr/core/src/resources/ImplicitPlugins.json
index bb17c3b07d1..f9f9be57dba 100644
--- a/solr/core/src/resources/ImplicitPlugins.json
+++ b/solr/core/src/resources/ImplicitPlugins.json
@@ -153,6 +153,20 @@
"echoParams": "explicit",
"echoHandler": true
}
+ },
+ "/tasks/cancel": {
+ "class": "solr.QueryCancellationHandler",
+ "useParams":"_TASK_CANCELLATION",
+ "components": [
+ "querycancellation"
+ ]
+ },
+ "/tasks/list": {
+ "class": "solr.ActiveTasksListHandler",
+ "useParams":"_LIST_TASKS",
+ "components": [
+ "activetaskslist"
+ ]
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java b/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java
index a5fc5457f80..8e0f8db6c12 100644
--- a/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java
+++ b/solr/core/src/test/org/apache/solr/MinimalSchemaTest.java
@@ -117,7 +117,8 @@ public class MinimalSchemaTest extends SolrTestCaseJ4 {
handler.startsWith("/terms") ||
handler.startsWith("/analysis/")||
handler.startsWith("/debug/") ||
- handler.startsWith("/replication")
+ handler.startsWith("/replication") ||
+ handler.startsWith("/tasks")
) {
continue;
}
diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
index b42a4aa8e13..6c70bd36cc3 100644
--- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
+++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java
@@ -114,6 +114,8 @@ public class SolrCoreTest extends SolrTestCaseJ4 {
++ihCount; assertEquals(pathToClassMap.get("/analysis/field"), "solr.FieldAnalysisRequestHandler");
++ihCount; assertEquals(pathToClassMap.get("/debug/dump"), "solr.DumpRequestHandler");
++ihCount; assertEquals(pathToClassMap.get("update"), "solr.UpdateRequestHandlerApi");
+ ++ihCount; assertEquals(pathToClassMap.get("/tasks/cancel"), "solr.QueryCancellationHandler");
+ ++ihCount; assertEquals(pathToClassMap.get("/tasks/list"), "solr.ActiveTasksListHandler");
}
assertEquals("wrong number of implicit handlers", ihCount, implicitHandlers.size());
}
diff --git a/solr/core/src/test/org/apache/solr/search/TestCancellableCollector.java b/solr/core/src/test/org/apache/solr/search/TestCancellableCollector.java
new file mode 100644
index 00000000000..a860d43d615
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/TestCancellableCollector.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.FilterLeafCollector;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NamedThreadFactory;
+import org.apache.solr.SolrTestCase;
+import org.apache.solr.common.util.ExecutorUtil;
+
+public class TestCancellableCollector extends SolrTestCase {
+ Directory dir;
+ IndexReader reader;
+ IndexSearcher searcher;
+ ExecutorService executor = null;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ dir = newDirectory();
+
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
+ Random random = random();
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(newStringField("field", Integer.toString(i), Field.Store.NO));
+ doc.add(newStringField("field2", Boolean.toString(i % 2 == 0), Field.Store.NO));
+ doc.add(new SortedDocValuesField("field2", new BytesRef(Boolean.toString(i % 2 == 0))));
+ iw.addDocument(doc);
+
+ if (random.nextBoolean()) {
+ iw.commit();
+ }
+ }
+ reader = iw.getReader();
+ iw.close();
+
+ searcher = new IndexSearcher(reader);
+
+ executor =
+ new ExecutorUtil.MDCAwareThreadPoolExecutor(
+ 4,
+ 4,
+ 0L,
+ TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ new NamedThreadFactory("TestIndexSearcher"));
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ reader.close();
+ dir.close();
+
+ if (executor != null) {
+ executor.shutdown();
+ }
+
+ executor = null;
+ }
+
+ private CancellableCollector buildCancellableCollector(
+ final int numHits, boolean delayStart, boolean delayCollection) {
+ TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(numHits, null, 1);
+ CancellableCollector collector = new CancellableCollector(topScoreDocCollector);
+
+ return new DummyCancellableCollector(collector, delayStart, delayCollection);
+ }
+
+ private void executeSearchTest(
+ IndexSearcher searcher, Query query, CancellableCollector cancellableCollector, int numHits)
+ throws Exception {
+ TopDocs topDocs = searcher.search(query, numHits);
+
+ searcher.search(query, cancellableCollector);
+
+ CancellableCollector internalCancellableCollector =
+ (CancellableCollector) cancellableCollector.getInternalCollector();
+ TopScoreDocCollector topScoreDocCollector =
+ (TopScoreDocCollector) internalCancellableCollector.getInternalCollector();
+
+ assertEquals(topDocs.totalHits.value, topScoreDocCollector.getTotalHits());
+ }
+
+ private void cancelQuery(CancellableCollector cancellableCollector, final int sleepTime) {
+ executor.submit(
+ () -> {
+
+ // Wait for some time to let the query start
+ try {
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+
+ cancellableCollector.cancel();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ });
+ }
+
+ public void testSearchWithoutCancellation() throws Exception {
+ CancellableCollector cancellableCollector = buildCancellableCollector(50, false, false);
+
+ Query query = new TermQuery(new Term("field", "1"));
+
+ executeSearchTest(searcher, query, cancellableCollector, 50);
+
+ query = new MatchAllDocsQuery();
+
+ cancellableCollector = buildCancellableCollector(100, false, false);
+
+ executeSearchTest(searcher, query, cancellableCollector, 50);
+ }
+
+ public void testSearchWithCancellationBeforeActualDocumentCollection() {
+ Query query = new MatchAllDocsQuery();
+
+ CancellableCollector cancellableCollector = buildCancellableCollector(5000, true, false);
+
+ expectThrows(
+ CancellableCollector.QueryCancelledException.class,
+ () -> {
+ // Cancel the query before the document collection starts
+ cancelQuery(cancellableCollector, 0);
+
+ executeSearchTest(searcher, query, cancellableCollector, 5000);
+ });
+ }
+
+ public void testSearchWithCancellationBetweenActualDocumentCollection() {
+ Query query = new MatchAllDocsQuery();
+
+ CancellableCollector cancellableCollector = buildCancellableCollector(5000, false, true);
+
+ expectThrows(
+ CancellableCollector.QueryCancelledException.class,
+ () -> {
+ // Cancel the query before the document collection starts
+ cancelQuery(cancellableCollector, 0);
+
+ executeSearchTest(searcher, query, cancellableCollector, 5000);
+ });
+ }
+
+ public class DummyCancellableCollector extends CancellableCollector {
+ private final CancellableCollector collector;
+ private final boolean delayStart;
+ private final boolean delayCollection;
+
+ public DummyCancellableCollector(
+ CancellableCollector cancellableCollector, boolean delayStart, boolean delayCollection) {
+ super(cancellableCollector);
+
+ this.collector = cancellableCollector;
+ this.delayStart = delayStart;
+ this.delayCollection = delayCollection;
+ }
+
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+
+ if (delayStart) {
+ try {
+ Thread.sleep(50);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+
+ return new FilterLeafCollector(collector.getLeafCollector(context)) {
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (delayCollection) {
+ try {
+ Thread.sleep(30);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e.getMessage());
+ }
+ }
+
+ in.collect(doc);
+ }
+ };
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return collector.scoreMode();
+ }
+
+ @Override
+ public void cancel() {
+ collector.cancel();
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/search/TestTaskManagement.java b/solr/core/src/test/org/apache/solr/search/TestTaskManagement.java
new file mode 100644
index 00000000000..0666a5be9a5
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/TestTaskManagement.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+
+public class TestTaskManagement extends SolrCloudTestCase {
+ private static final String COLLECTION_NAME = "collection1";
+
+ private ExecutorService executorService;
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ initCore("solrconfig.xml", "schema11.xml");
+
+ configureCluster(4)
+ .addConfig("conf", configset("sql"))
+ .configure();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ shutdownCluster();
+ }
+
+ @Before
+ public void setup() throws Exception {
+ super.setUp();
+
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf", 2, 1)
+ .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 2, 2);
+ cluster.getSolrClient().setDefaultCollection(COLLECTION_NAME);
+
+ executorService = ExecutorUtil.newMDCAwareCachedThreadPool("TestTaskManagement");
+
+ List<SolrInputDocument> docs = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", i);
+ doc.addField("foo1_s", Integer.toString(i));
+ doc.addField("foo2_s", Boolean.toString(i % 2 == 0));
+ doc.addField("foo4_s", new BytesRef(Boolean.toString(i % 2 == 0)));
+
+ docs.add(doc);
+ }
+
+ cluster.getSolrClient().add(docs);
+ cluster.getSolrClient().commit();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ CollectionAdminRequest.deleteCollection(COLLECTION_NAME).process(cluster.getSolrClient());
+ executorService.shutdown();
+ super.tearDown();
+ }
+
+ @Test
+ public void testNonExistentQuery() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.set("queryUUID", "foobar");
+ @SuppressWarnings({"rawtypes"})
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/tasks/cancel");
+
+ NamedList<Object> queryResponse;
+
+ queryResponse = cluster.getSolrClient().request(request);
+
+ assertEquals("Query with queryID foobar not found", queryResponse.get("status"));
+ assertEquals(404, queryResponse.get("responseCode"));
+ }
+
+ @Test
+ public void testCancellationQuery() {
+ Set<Integer> queryIdsSet = ConcurrentHashMap.newKeySet();
+ Set<Integer> notFoundIdsSet = ConcurrentHashMap.newKeySet();
+
+ List<CompletableFuture<Void>> queryFutures = new ArrayList<>();
+
+ for (int i = 0; i < 100; i++) {
+ CompletableFuture<Void> future = executeQueryAsync(Integer.toString(i));
+
+ queryFutures.add(future);
+ }
+
+ List<CompletableFuture<Void>> futures = new ArrayList<>();
+
+ for (int i = 0; i < 100; i++) {
+ CompletableFuture<Void> future = cancelQuery(Integer.toString(i), 4000, queryIdsSet, notFoundIdsSet);
+
+ futures.add(future);
+ }
+
+ futures.forEach(CompletableFuture::join);
+
+ queryFutures.forEach(CompletableFuture::join);
+
+ assertEquals("Total query count did not match the expected value",
+ queryIdsSet.size() + notFoundIdsSet.size(), 100);
+ }
+
+ @Test
+ public void testListCancellableQueries() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ @SuppressWarnings({"rawtypes"})
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/tasks/list");
+
+ for (int i = 0; i < 50; i++) {
+ executeQueryAsync(Integer.toString(i));
+ }
+
+ NamedList<Object> queryResponse;
+
+ queryResponse = cluster.getSolrClient().request(request);
+
+ @SuppressWarnings({"unchecked"})
+ NamedList<String> result = (NamedList<String>) queryResponse.get("taskList");
+
+ Iterator<Map.Entry<String, String>> iterator = result.iterator();
+
+ Set<Integer> presentQueryIDs = new HashSet<>();
+
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+
+ presentQueryIDs.add(Integer.parseInt(entry.getKey()));
+ }
+
+ assertTrue(presentQueryIDs.size() > 0 && presentQueryIDs.size() <= 50);
+
+ for (int value : presentQueryIDs) {
+ assertTrue(value >= 0 && value < 50);
+ }
+ }
+
+ @Test
+ public void testCheckSpecificQueryStatus() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.set("taskUUID", "25");
+
+ @SuppressWarnings({"rawtypes"})
+ SolrRequest request = new QueryRequest(params);
+
+ request.setPath("/tasks/list");
+
+ NamedList<Object> queryResponse = cluster.getSolrClient().request(request);
+
+ @SuppressWarnings({"unchecked"})
+ String result = (String) queryResponse.get("taskStatus");
+
+ assertTrue(result.contains("inactive"));
+ }
+
+ private CompletableFuture<Void> cancelQuery(final String queryID, final int sleepTime, Set<Integer> cancelledQueryIdsSet,
+ Set<Integer> notFoundQueryIdSet) {
+ return CompletableFuture.runAsync(() -> {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.set("queryUUID", queryID);
+ @SuppressWarnings({"rawtypes"})
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/tasks/cancel");
+
+ // Wait for some time to let the query start
+ try {
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+
+ try {
+ NamedList<Object> queryResponse;
+
+ queryResponse = cluster.getSolrClient().request(request);
+
+ int responseCode = (int) queryResponse.get("responseCode");
+
+ if (responseCode == 200 /* HTTP OK */) {
+ cancelledQueryIdsSet.add(Integer.parseInt(queryID));
+ } else if (responseCode == 404 /* HTTP NOT FOUND */) {
+ notFoundQueryIdSet.add(Integer.parseInt(queryID));
+ }
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new CompletionException(e);
+ }
+ }, executorService);
+ }
+
+ public void executeQuery(String queryId) throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.set("q", "*:*");
+ params.set("canCancel", "true");
+
+ if (queryId != null) {
+ params.set("queryUUID", queryId);
+ }
+
+ @SuppressWarnings({"rawtypes"})
+ SolrRequest request = new QueryRequest(params);
+
+ cluster.getSolrClient().request(request);
+ }
+
+ public CompletableFuture<Void> executeQueryAsync(String queryId) {
+ return CompletableFuture.runAsync(() -> {
+ try {
+ executeQuery(queryId);
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ });
+ }
+}
diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adoc
index a5be6f000cb..d14b8993fdb 100644
--- a/solr/solr-ref-guide/src/common-query-parameters.adoc
+++ b/solr/solr-ref-guide/src/common-query-parameters.adoc
@@ -84,6 +84,20 @@ You can use the `rows` parameter to paginate results from a query. The parameter
The default value is `10`. That is, by default, Solr returns 10 documents at a time in response to a query.
+== canCancel Parameter
+
+This parameter defines if this query is cancellable i.e. can be cancelled during execution using the
+task management interface.
+
+== queryUUID Parameter
+
+For cancellable queries, this allows specifying a custom UUID to identify the query with. If `canCancel` is specified and `queryUUID` is not set, an auto generated UUID will be assigned to the query.
+
+If `queryUUID` is specified, this UUID will be used for identifying the query. Note that if using `queryUUID`, the responsibility of ensuring uniqueness of the UUID lies with the caller. If a query UUID
+is reused while the original query UUID is still active, it will cause an exception to be throws for the second query.
+
+It is recommended that the user either uses all custom UUIDs or depends completely on the system to generate UUID. Mixing the two can lead to conflict of UUIDs.
+
== fq (Filter Query) Parameter
The `fq` parameter defines a query that can be used to restrict the superset of documents that can be returned, without influencing score. It can be very useful for speeding up complex queries, since the queries specified with `fq` are cached independently of the main query. When a later query uses the same filter, there's a cache hit, and filter results are returned quickly from the cache.
diff --git a/solr/solr-ref-guide/src/index.adoc b/solr/solr-ref-guide/src/index.adoc
index d1e8642cfe5..4c1d1c98850 100644
--- a/solr/solr-ref-guide/src/index.adoc
+++ b/solr/solr-ref-guide/src/index.adoc
@@ -12,6 +12,7 @@
legacy-scaling-and-distribution, \
circuit-breakers, \
rate-limiters, \
+ task-management, \
solr-plugins, \
the-well-configured-solr-instance, \
monitoring-solr, \
@@ -127,6 +128,8 @@ The *<<getting-started.adoc#,Getting Started>>* section guides you through the i
*<<circuit-breakers.adoc#,Circuit Breakers>>*: This section talks about circuit breakers, a way of allowing a higher stability of Solr nodes and increased service level guarantees of requests that are accepted by Solr.
*<<rate-limiters.adoc#,Request Rate Limiters>>*: This section talks about request rate limiters, a way of guaranteeing throughput per request type and dedicating resource quotas by resource type. Rate limiter configurations are per instance/JVM and applied to the entire JVM, not at a core/collection level.
+
+*<<task-management.adoc#,Task Management>>*: This section talks about task management interface, which can be used to control cancellable tasks.
****
.Advanced Configuration
diff --git a/solr/solr-ref-guide/src/task-management.adoc b/solr/solr-ref-guide/src/task-management.adoc
new file mode 100644
index 00000000000..9ec5b7e8613
--- /dev/null
+++ b/solr/solr-ref-guide/src/task-management.adoc
@@ -0,0 +1,66 @@
+= Task Management
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+Solr allows users to control their running tasks by monitoring them, specifying tasks as cancellation enabled and allowing
+cancellation of the same.
+
+This is achieved using the task management interface. Currently, this is supported for queries.
+
+== Types of Operations
+Task management interface supports the following types of operations:
+
+1. List all currently running cancellable tasks.
+2. Cancel a specific task.
+3. Query the status of a specific task.
+
+== Listing All Active Cancellable Tasks
+To list all the active cancellable tasks currently running, please use the following syntax:
+
+`\http://localhost:8983/solr/tasks/list`
+
+==== Sample Response
+
+`{responseHeader={status=0, QTime=11370}, taskList={0=q=*%3A*&canCancel=true&queryUUID=0&_stateVer_=collection1%3A4&wt=javabin&version=2, 5=q=*%3A*&canCancel=true&queryUUID=5&_stateVer_=collection1%3A4&wt=javabin&version=2, 7=q=*%3A*&canCancel=true&queryUUID=7&_stateVer_=collection1%3A4&wt=javabin&version=2}`
+
+== Cancelling An Active Cancellable Task
+To cancel an active task, please use the following syntax:
+
+`\http://localhost:8983/solr/tasks/cancel?queryUUID=foobar`
+
+==== Sample Response
+===== If the task UUID was found and successfully cancelled:
+
+`{responseHeader={status=0, QTime=39}, status=Query with queryID 85 cancelled successfully}`
+
+===== If the task UUID was not found
+
+`{responseHeader={status=0, QTime=39}, status=Query with queryID 85 not found}`
+
+== Check Status of a Specific Task
+To check the status of a specific task, please use the following syntax:
+
+`\http://localhost:8983/solr/tasks/list?taskUUID=foobar`
+
+==== taskUUID Parameter
+`taskUUID` parameter can be used to specify a task UUID whose status can be checked.
+
+==== Sample Response
+`{responseHeader={status=0, QTime=6128}, taskStatus=foobar:true}`
+
+
+
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
index eddd4d02d78..5bb5486c487 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/QueryResponse.java
@@ -55,6 +55,9 @@ public class QueryResponse extends SolrResponseBase
private NamedList<Object> _statsInfo = null;
private NamedList<NamedList<Object>> _termsInfo = null;
private NamedList<SolrDocumentList> _moreLikeThisInfo = null;
+ private NamedList<String> _tasksInfo = null;
+ private String _cancellationInfo = null;
+ private String _taskStatusCheckInfo = null;
private String _cursorMarkNext = null;
// Grouping response
@@ -184,6 +187,15 @@ public class QueryResponse extends SolrResponseBase
else if ( "moreLikeThis".equals( n ) ) {
_moreLikeThisInfo = (NamedList<SolrDocumentList>) res.getVal( i );
}
+ else if ("taskList".equals( n )) {
+ _tasksInfo = (NamedList<String>) res.getVal( i );
+ }
+ else if ("cancellationResult".equals (n)) {
+ _cancellationInfo = (String) res.getVal( i );
+ }
+ else if ("taskResult".equals (n)) {
+ _taskStatusCheckInfo = (String) res.getVal( i );
+ }
else if ( CursorMarkParams.CURSOR_MARK_NEXT.equals( n ) ) {
_cursorMarkNext = (String) res.getVal( i );
}
@@ -591,6 +603,18 @@ public class QueryResponse extends SolrResponseBase
public NamedList<SolrDocumentList> getMoreLikeThis() {
return _moreLikeThisInfo;
}
+
+ public NamedList<String> getTasksInfo() {
+ return _tasksInfo;
+ }
+
+ public String getCancellationInfo() {
+ return _cancellationInfo;
+ }
+
+ public String getTaskStatusCheckInfo() {
+ return _taskStatusCheckInfo;
+ }
/**
* See also: {@link #getLimitingFacets()}
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index ece61e7da1f..73316c53b88 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -160,6 +160,21 @@ public interface CommonParams {
*/
String TIME_ALLOWED = "timeAllowed";
+ /**
+ * Is the query cancellable?
+ */
+ String IS_QUERY_CANCELLABLE = "canCancel";
+
+ /**
+ * Custom query UUID if provided.
+ */
+ String QUERY_UUID = "queryUUID";
+
+ /**
+ * UUID of the task whose status is to be checked
+ */
+ String TASK_CHECK_UUID = "taskUUID";
+
/**
* The number of hits that need to be counted accurately. If more than {@link #MIN_EXACT_COUNT} documents
* match a query, then the value in "numFound" may be an estimate to speedup search.
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
index 088882a3f8a..05cdef380ef 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java
@@ -28,6 +28,9 @@ import org.apache.solr.common.util.StrUtils;
public interface ShardParams {
/** the shards to use (distributed configuration) */
String SHARDS = "shards";
+
+ /** UUID of the query */
+ String QUERY_ID = "queryID";
/** per-shard start and rows */
String SHARDS_ROWS = "shards.rows";
diff --git a/solr/solrj/src/resources/apispec/core.tasks.cancel.json b/solr/solrj/src/resources/apispec/core.tasks.cancel.json
new file mode 100644
index 00000000000..ffe9fc0d51e
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/core.tasks.cancel.json
@@ -0,0 +1,18 @@
+{
+ "documentation": "https://lucene.apache.org/solr/guide/task-management.html",
+ "description": "Cancel a currently running task",
+ "methods": [
+ "GET"
+ ],
+ "url": {
+ "paths": [
+ "/tasks/cancel"
+ ],
+ "params": {
+ "cancelUUID": {
+ "type":"string",
+ "description": "Specify the UUID for the query to cancel"
+ }
+ }
+ }
+}
diff --git a/solr/solrj/src/resources/apispec/core.tasks.list.json b/solr/solrj/src/resources/apispec/core.tasks.list.json
new file mode 100644
index 00000000000..a898f9fab88
--- /dev/null
+++ b/solr/solrj/src/resources/apispec/core.tasks.list.json
@@ -0,0 +1,12 @@
+{
+ "documentation": "https://lucene.apache.org/solr/guide/task-management.html",
+ "description": "List currently running tasks",
+ "methods": [
+ "GET"
+ ],
+ "url": {
+ "paths": [
+ "/tasks/list"
+ ]
+ }
+}