Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15146: Collection and Config Set API can run distributed on all nodes (#70)

Ilan Ginzburg4a78b458047c
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e56a5c48165..afe4751addd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -260,6 +260,9 @@ Other Changes
* SOLR-15341: Lucene has removed CodecReader#ramBytesUsed in LUCENE-9387, so ramBytesUsed will no longer be reported
in SegmentsInfo handler (janhoy)
+* SOLR-15146: Allow Collection API and Config Set API to be done in a distributed fashion without going through Overseer (Ilan Ginzburg)
+
+
Bug Fixes
---------------------
* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
diff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java
new file mode 100644
index 00000000000..7200969cbc8
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This class implements a higher level locking abstraction for the Config Set API using lower level read and write locks.
+ */
+public class ConfigSetApiLockFactory {
+
+ private final DistributedConfigSetLockFactory lockFactory;
+
+ public ConfigSetApiLockFactory(DistributedConfigSetLockFactory lockFactory) {
+ this.lockFactory = lockFactory;
+ }
+
+ /**
+ * For the {@link org.apache.solr.common.params.CollectionParams.LockLevel} of the passed {@code action}, obtains the
+ * required locks (if any) and returns.<p>
+ *
+ * This method obtains a write lock on {@code configSetName} as well as (when not {@code null}), a read lock on {@code baseConfigSetName}.
+ *
+ * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding Config Set API command
+ * can execute safely.
+ * The returned lock <b>MUST</b> be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would
+ * prevent other threads from locking.
+ */
+ public DistributedMultiLock createConfigSetApiLock(String configSetName, String baseConfigSetName) {
+ List<DistributedLock> locks = new ArrayList<>(2);
+
+ locks.add(lockFactory.createLock(true, configSetName));
+ if (baseConfigSetName != null) {
+ locks.add(lockFactory.createLock(false, baseConfigSetName));
+ }
+
+ return new DistributedMultiLock(locks);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java
new file mode 100644
index 00000000000..5d5983a5f0c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.jayway.jsonpath.internal.Utils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ConfigSetParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.ConfigSetProperties;
+import org.apache.solr.core.ConfigSetService;
+import org.apache.solr.core.CoreContainer;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
+import static org.apache.solr.common.util.Utils.toJSONString;
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+
+/**
+ * This class contains methods dealing with Config Sets and called for Config Set API execution, called
+ * from the {@link OverseerConfigSetMessageHandler} or from
+ * {@link org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner#runConfigSetCommand} depending
+ * on whether Collection and Config Set APIs are Overseer based or distributed.
+ */
+public class ConfigSetCmds {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Name of the ConfigSet to copy from for CREATE
+ */
+ public static final String BASE_CONFIGSET = "baseConfigSet";
+
+ /**
+ * Prefix for properties that should be applied to the ConfigSet for CREATE
+ */
+ public static final String CONFIG_SET_PROPERTY_PREFIX = "configSetProp.";
+
+ public static String getBaseConfigSetName(ConfigSetParams.ConfigSetAction action, String baseConfigSetName) {
+ if (action == CREATE) {
+ return Utils.isEmpty(baseConfigSetName) ? DEFAULT_CONFIGSET_NAME : baseConfigSetName;
+ }
+ return null;
+ }
+
+
+ @SuppressWarnings({"rawtypes"})
+ private static NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {
+ byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);
+ if (oldPropsData != null) {
+ InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);
+ try {
+ return ConfigSetProperties.readFromInputStream(reader);
+ } finally {
+ reader.close();
+ }
+ }
+ return null;
+ }
+
+ private static Map<String, Object> getNewProperties(ZkNodeProps message) {
+ Map<String, Object> properties = null;
+ for (Map.Entry<String, Object> entry : message.getProperties().entrySet()) {
+ if (entry.getKey().startsWith(CONFIG_SET_PROPERTY_PREFIX)) {
+ if (properties == null) {
+ properties = new HashMap<String, Object>();
+ }
+ properties.put(entry.getKey().substring((CONFIG_SET_PROPERTY_PREFIX).length()),
+ entry.getValue());
+ }
+ }
+ return properties;
+ }
+
+ private static void mergeOldProperties(Map<String, Object> newProps, @SuppressWarnings({"rawtypes"})NamedList oldProps) {
+ @SuppressWarnings({"unchecked"})
+ Iterator<Map.Entry<String, Object>> it = oldProps.iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, Object> oldEntry = it.next();
+ if (!newProps.containsKey(oldEntry.getKey())) {
+ newProps.put(oldEntry.getKey(), oldEntry.getValue());
+ }
+ }
+ }
+
+ private static byte[] getPropertyData(Map<String, Object> newProps) {
+ if (newProps != null) {
+ String propertyDataStr = toJSONString(newProps);
+ if (propertyDataStr == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid property specification");
+ }
+ return propertyDataStr.getBytes(StandardCharsets.UTF_8);
+ }
+ return null;
+ }
+
+ public static void createConfigSet(ZkNodeProps message, CoreContainer coreContainer) throws IOException {
+ String configSetName = message.getStr(NAME);
+ if (configSetName == null || configSetName.length() == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
+ }
+
+ String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
+
+ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);
+ }
+
+ // is there a base config that already exists
+ if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Base ConfigSet does not exist: " + baseConfigSetName);
+ }
+
+ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
+ Map<String, Object> props = getNewProperties(message);
+ if (props != null) {
+ // read the old config properties and do a merge, if necessary
+ @SuppressWarnings({"rawtypes"})
+ NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);
+ if (oldProps != null) {
+ mergeOldProperties(props, oldProps);
+ }
+ }
+ byte[] propertyData = getPropertyData(props);
+
+ try {
+ coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);
+ if (propertyData != null) {
+ coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);
+ }
+ } catch (Exception e) {
+ // copying the config dir or writing the properties file may have failed.
+ // we should delete the ConfigSet because it may be invalid,
+ // assuming we actually wrote something. E.g. could be
+ // the entire baseConfig set with the old properties, including immutable,
+ // that would make it impossible for the user to delete.
+ try {
+ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
+ deleteConfigSet(configSetName, true, coreContainer);
+ }
+ } catch (IOException ioe) {
+ log.error("Error while trying to delete partially created ConfigSet", ioe);
+ }
+ throw e;
+ }
+ }
+
+ public static void deleteConfigSet(ZkNodeProps message, CoreContainer coreContainer) throws IOException {
+ String configSetName = message.getStr(NAME);
+ if (configSetName == null || configSetName.length() == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
+ }
+
+ deleteConfigSet(configSetName, false, coreContainer);
+ }
+
+ private static void deleteConfigSet(String configSetName, boolean force, CoreContainer coreContainer) throws IOException {
+ if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);
+ }
+
+ ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();
+
+ for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {
+ String configName = null;
+ try {
+ configName = zkStateReader.readConfigName(entry.getKey());
+ } catch (KeeperException ex) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");
+ }
+ if (configSetName.equals(configName))
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");
+ }
+
+ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
+ @SuppressWarnings({"rawtypes"})
+ NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);
+ if (properties != null) {
+ Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);
+ boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
+ if (!force && isImmutableConfigSet) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);
+ }
+ }
+ coreContainer.getConfigSetService().deleteConfig(configSetName);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java
new file mode 100644
index 00000000000..cc007fd18de
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
+/**
+ * Class holding the implementation required for tracking asynchronous Collection API (or other) tasks when the Collection
+ * API is distributed.<p>
+ *
+ * This replaces the features provided by the distributed maps on ZK paths /overseer/collection-map-completed,
+ * /overseer/collection-map-failure and /overseer/async_ids when the Collection API commands are handled by the Overseer.<p>
+ *
+ * It works by using two Zookeeper directories, one for persistent nodes for each new async id and one for ephemeral nodes
+ * for each async id currently being processed (in flight).<br>
+ * A persistent async node has either no data, or has a serialized OverseerSolrResponse as content.
+ * An ephemeral async node has two possible states (content): 'S' or 'R'.<p>
+ *
+ * The actual state of an async task is built from a combination of the two nodes:
+ * <pre>
+ * +===================+=========================================+=================================================+====================+
+ * | | persistent=success OverseerSolrResponse | persistent=null or failed OverseerSolrResponse | No persistent node |
+ * +===================+=========================================+=================================================+====================+
+ * | ephemeral="S" | Task completed successfully | Task submitted | Unknown task |
+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------+
+ * | ephemeral="R" | Task completed successfully | Task running | Unknown task |
+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------+
+ * | No ephemeral node | Task completed successfully | Task failed (see response or null=node failure) | Unknown task |
+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------+
+ * </pre>
+ */
+public class DistributedApiAsyncTracker {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Corresponds to Overseer.NUM_RESPONSES_TO_STORE. The size of the persistent store of async ID's put in Zookeeper.
+ * This is the max total tracked async request ID's over all nodes running in the distributed Collection API.
+ */
+ public static int MAX_TRACKED_ASYNC_TASKS = 10000;
+
+ private static final String ZK_ASYNC_PERSISTENT = "/persistent";
+ private static final String ZK_ASYNC_INFLIGHT = "/inflight";
+
+ private final String persistentIdsPath;
+ private final String inFlightIdsPath;
+
+ /**
+ * Persistent storage in Zookeeper under path {@link #ZK_ASYNC_PERSISTENT} of all currently known (in flight, completed
+ * with success or error) async request id's.
+ */
+ private final SizeLimitedDistributedMap trackedAsyncTasks;
+
+ private final InFlightJobs inFlightAsyncTasks;
+
+ public DistributedApiAsyncTracker(SolrZkClient zkClient, String rootPath) {
+ this(zkClient, rootPath, MAX_TRACKED_ASYNC_TASKS);
+ }
+
+ @VisibleForTesting
+ DistributedApiAsyncTracker(SolrZkClient zkClient, String rootPath, int maxTrackedTasks) {
+ persistentIdsPath = rootPath + ZK_ASYNC_PERSISTENT;
+ inFlightIdsPath = rootPath + ZK_ASYNC_INFLIGHT;
+
+ trackedAsyncTasks = new SizeLimitedDistributedMap(zkClient, persistentIdsPath, maxTrackedTasks, null);
+ inFlightAsyncTasks = new InFlightJobs(zkClient, inFlightIdsPath);
+ }
+
+ /**
+ * After a successful call to this method, caller MUST eventually call {@link #setTaskCompleted} or {@link #cancelAsyncId}
+ * otherwise the task will forever be considered as in progress.
+ * @param asyncId if {@code null} this method will do nothing.
+ * @return {@code true} if the asyncId was not already in use (or is {@code null}) and {@code false} if it is already
+ * in use and can't be allocated again.
+ */
+ public boolean createNewAsyncJobTracker(String asyncId) {
+ if (asyncId == null) {
+ return true;
+ }
+ try {
+ // First create the persistent node, with no content. If that fails, it means the asyncId has been previously used
+ // and not yet cleared...
+ if (!trackedAsyncTasks.putIfAbsent(asyncId, null)) {
+ return false;
+ }
+
+ // ...then create the transient node. If the corresponding ephemeral node already exists, it means the persistent node
+ // was removed (maybe trackedAsyncTasks grew too large? It has a max size then evicts). We cannot then track the new
+ // provided asyncId, and have simply "revived" its persistent node...
+ try {
+ inFlightAsyncTasks.createNewInFlightTask(asyncId);
+ return true;
+ } catch (KeeperException.NodeExistsException nee) {
+ log.warn("Async id {} was not found in trackedAsyncTasks but was still present in inFlightAsyncTasks", asyncId);
+ return false;
+ }
+ } catch (KeeperException ke) {
+ throw new SolrException(SERVER_ERROR, "Error creating new async job tracking " + asyncId, ke);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SERVER_ERROR, "Interrupted creating new async job tracking " + asyncId, ie);
+ }
+ }
+
+ /**
+ * Initially an async task is submitted. Just before it actually starts execution it is set to running.
+ */
+ public void setTaskRunning(String asyncId) {
+ if (asyncId == null) {
+ return;
+ }
+ try {
+ inFlightAsyncTasks.setTaskRunning(asyncId);
+ } catch (KeeperException ke) {
+ throw new SolrException(SERVER_ERROR, "Error setting async task as running " + asyncId, ke);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SERVER_ERROR, "Interrupted setting async task as running " + asyncId, ie);
+ }
+ }
+
+ /**
+ * Mark the completion (success or error) of an async task. The success or error is judged by the contents
+ * of the {@link OverseerSolrResponse}.
+ */
+ public void setTaskCompleted(String asyncId, OverseerSolrResponse solrResponse) {
+ if (asyncId == null) {
+ return;
+ }
+ // First update the persistent node with the execution result, only then remove the transient node
+ // (otherwise a status check might report the task in error)
+ try {
+ try {
+ trackedAsyncTasks.put(asyncId, OverseerSolrResponseSerializer.serialize(solrResponse));
+ } finally {
+ inFlightAsyncTasks.deleteInFlightTask(asyncId);
+ }
+ } catch (KeeperException ke) {
+ throw new SolrException(SERVER_ERROR, "Error setting async task as completed " + asyncId, ke);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SERVER_ERROR, "Interrupted setting async task as completed " + asyncId, ie);
+ }
+ }
+
+ /**
+ * Cancels the tracking of an asyncId, if the corresponding command could not be executed.
+ */
+ public void cancelAsyncId(String asyncId) {
+ if (asyncId == null) {
+ return;
+ }
+ try {
+ try {
+ trackedAsyncTasks.remove(asyncId);
+ } finally {
+ inFlightAsyncTasks.deleteInFlightTask(asyncId);
+ }
+ } catch (KeeperException ke) {
+ throw new SolrException(SERVER_ERROR, "Error canceling async task " + asyncId, ke);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SERVER_ERROR, "Interrupted canceling async task " + asyncId, ie);
+ }
+ }
+
+ /**
+ * This method implements the logic described in the class Javadoc table ({@link DistributedApiAsyncTracker}), using the
+ * two sets of tracking info to build the actual state of an async task.<p>
+ *
+ * Returns the status of an async task, and when relevant the corresponding response from the command execution. The
+ * returned {@link OverseerSolrResponse} will not be {@code null} when the returned
+ * {@link RequestStatusState} is {@link RequestStatusState#COMPLETED} or {@link RequestStatusState#FAILED} (and will be
+ * {@code null} in all other cases).
+ */
+ @SuppressWarnings("unchecked")
+ public Pair<RequestStatusState, OverseerSolrResponse> getAsyncTaskRequestStatus(String asyncId) throws Exception {
+ if (asyncId == null || !trackedAsyncTasks.contains(asyncId)) {
+ // This return addresses the whole "No persistent node" column from the table
+ return new Pair<>(RequestStatusState.NOT_FOUND, null);
+ }
+
+ byte[] data = trackedAsyncTasks.get(asyncId);
+ OverseerSolrResponse response = data != null ? OverseerSolrResponseSerializer.deserialize(data) : null;
+
+ if (response != null && response.getResponse().get("failure") == null && response.getResponse().get("exception") == null) {
+ // This return addresses the whole "persistent=success OverseerSolrResponse" column from the table
+ return new Pair<>(RequestStatusState.COMPLETED, response);
+ }
+
+ // Now dealing with the middle column "persistent=null or failed OverseerSolrResponse"
+ InFlightJobs.State ephemeralState = inFlightAsyncTasks.getInFlightState(asyncId);
+ if (ephemeralState == InFlightJobs.State.SUBMITTED) {
+ return new Pair<>(RequestStatusState.SUBMITTED, null);
+ } else if (ephemeralState == InFlightJobs.State.RUNNING) {
+ return new Pair<>(RequestStatusState.RUNNING, null);
+ }
+
+ // The task has failed, but there are two options: if response is null, it has failed because the node on which it was
+ // running has crashed. If it is not null, it has failed because the execution has failed. Because caller expects a non
+ // null response in any case, let's make up one if needed...
+ if (response == null) {
+ // Node crash has removed the ephemeral node, but the command did not complete execution (or didn't even start it, who
+ // knows). We have a failure to report though so let's create a reasonable return response.
+ @SuppressWarnings({"rawtypes"})
+ NamedList results = new NamedList();
+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
+ nl.add("msg", "Operation (asyncId: " + asyncId + ") failed due to server restart. Please resubmit.");
+ nl.add("rspCode", SERVER_ERROR.code);
+ results.add("exception", nl);
+ response = new OverseerSolrResponse(results);
+ }
+
+ return new Pair<>(RequestStatusState.FAILED, response);
+ }
+
+
+ /**
+ * Deletes a single async tracking ID if the corresponding job has completed or failed.
+ * @return {@code true} if the {@code asyncId} was found to be of a completed or failed job and was successfully removed,
+ * {@code false} if the id was not found or was found for a submitted or running job (these are not removed).
+ */
+ public boolean deleteSingleAsyncId(String asyncId) throws Exception {
+ return inFlightAsyncTasks.getInFlightState(asyncId) == InFlightJobs.State.NOT_FOUND && trackedAsyncTasks.remove(asyncId);
+ }
+
+ /**
+ * Deletes all async id's for completed or failed async jobs. Does not touch id's for submitted or running jobs.
+ */
+ public void deleteAllAsyncIds() throws Exception {
+ Collection<String> allTracked = trackedAsyncTasks.keys();
+
+ for (String asyncId : allTracked) {
+ deleteSingleAsyncId(asyncId);
+ }
+ }
+
+ /**
+ * Manages the ephemeral nodes for tasks currently being processed (running or waiting for a lock) by a node.
+ */
+ private static class InFlightJobs {
+ enum State {
+ SUBMITTED("S"),
+ RUNNING("R"),
+ NOT_FOUND(null);
+
+ private final String shorthand;
+
+ State(String shorthand) {
+ this.shorthand = shorthand;
+ }
+ }
+
+ private final SolrZkClient zkClient;
+ private final String rootNodePath;
+
+ InFlightJobs(SolrZkClient zkClient, String rootNodePath) {
+ this.zkClient = zkClient;
+ this.rootNodePath = rootNodePath;
+
+ try {
+ if (!zkClient.exists(rootNodePath, true)) {
+ zkClient.makePath(rootNodePath, new byte[0], CreateMode.PERSISTENT, true);
+ }
+ } catch (KeeperException.NodeExistsException nee) {
+ // Some other thread (on this or another JVM) beat us to create the node, that's ok, the node exists.
+ } catch (KeeperException ke) {
+ throw new SolrException(SERVER_ERROR, "Error creating root node " + rootNodePath, ke);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SERVER_ERROR, "Interrupted creating root node " + rootNodePath, ie);
+ }
+ }
+
+ void createNewInFlightTask(String asyncId) throws KeeperException, InterruptedException {
+ zkClient.create(getPath(asyncId), State.SUBMITTED.shorthand.getBytes(StandardCharsets.UTF_8), CreateMode.EPHEMERAL, true);
+ }
+
+ void setTaskRunning(String asyncId) throws KeeperException, InterruptedException {
+ zkClient.setData(getPath(asyncId), State.RUNNING.shorthand.getBytes(StandardCharsets.UTF_8), true);
+ }
+
+ void deleteInFlightTask(String asyncId) throws KeeperException, InterruptedException {
+ zkClient.delete(getPath(asyncId), -1, true);
+ }
+
+ State getInFlightState(String asyncId) throws KeeperException, InterruptedException {
+ if (!zkClient.exists(getPath(asyncId), true)) {
+ return State.NOT_FOUND;
+ }
+
+ final byte[] bytes;
+ try {
+ bytes = zkClient.getData(getPath(asyncId), null, null, true);
+ } catch (KeeperException.NoNodeException nne) {
+ // Unlikely race, but not impossible...
+ if (log.isInfoEnabled()) {
+ log.info("AsyncId ephemeral node " + getPath(asyncId) + " vanished from underneath us. Funny."); // nowarn
+ }
+ return State.NOT_FOUND;
+ }
+
+ if (bytes == null) {
+ // This is not expected. The ephemeral nodes are always created with content.
+ log.error("AsyncId ephemeral node " + getPath(asyncId) + " has null content. This is unexpected (bug)."); // nowarn
+ return State.NOT_FOUND;
+ }
+
+ String content = new String(bytes, StandardCharsets.UTF_8);
+ if (State.RUNNING.shorthand.equals(content)) {
+ return State.RUNNING;
+ } else if (State.SUBMITTED.shorthand.equals(content)) {
+ return State.SUBMITTED;
+ } else {
+ log.error("AsyncId ephemeral node " + getPath(asyncId) + " has unexpected content \"" + content + "\". This is unexpected (bug)."); // nowarn
+ return State.NOT_FOUND;
+ }
+ }
+
+ private String getPath(String asyncId) {
+ return rootNodePath + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + asyncId;
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java
new file mode 100644
index 00000000000..d87dafc956a
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import org.apache.solr.cloud.api.collections.CollectionApiLockFactory;
+import org.apache.solr.common.params.CollectionParams;
+
+public interface DistributedCollectionLockFactory {
+ /**
+ * Create a new lock of the specified type (read or write) entering the "competition" for actually getting the lock at
+ * the given level for the given path i.e. a lock at {@code collName} or a lock at {@code collName/shardId} or a lock
+ * at {@code collName/shardId/replicaName}, depending on the passed {@code level}.<p>
+ *
+ * The paths are used to define which locks compete with each other (locks of equal paths compete).<p>
+ *
+ * Upon return from this call, the lock <b>has not been acquired</b> but the it had entered the lock acquiring "competition",
+ * and the caller can decide to wait until the lock is granted by calling {@link DistributedLock#waitUntilAcquired()}.<br>
+ * Separating the lock creation from lock acquisition allows a more deterministic release of the locks when/if they can't be
+ * acquired.<p>
+ *
+ * Locks at different paths are independent of each other, multiple {@link DistributedLock} are therefore requested for
+ * a single operation and are packaged together and returned as an {@link DistributedMultiLock},
+ * see {@link CollectionApiLockFactory#createCollectionApiLock}.
+ * @param isWriteLock {@code true} if requesting a write lock, {@code false} for a read lock.
+ * @param level The requested locking level. Can be one of:
+ * <ul><li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}</li>
+ * <li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD}</li>
+ * <li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#REPLICA}</li></ul>
+ * @param collName the collection name, can never be {@code null} as is needed for all locks.
+ * @param shardId is ignored and can be {@code null} if {@code level} is {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}
+ * @param replicaName is ignored and can be {@code null} if {@code level} is {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}
+ * or {@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD}
+ * @return a lock instance that must be {@link DistributedLock#release()}'ed in a {@code finally},
+ * regardless of the lock having been acquired or not.
+ */
+ DistributedLock createLock(boolean isWriteLock, CollectionParams.LockLevel level, String collName, String shardId,
+ String replicaName);
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java
new file mode 100644
index 00000000000..4f2752740b6
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+public interface DistributedConfigSetLockFactory {
+ /**
+ * Create a new lock of the specified type (read or write) entering the "competition" for actually getting the lock for
+ * the {@code configSetName}<p>
+ *
+ * Upon return from this call, the lock <b>has not been acquired</b> but the it had entered the lock acquiring "competition",
+ * and the caller can decide to wait until the lock is granted by calling {@link DistributedLock#waitUntilAcquired()}.<br>
+ * Separating the lock creation from lock acquisition allows a more deterministic release of the locks when/if they can't be
+ * acquired.<p>
+ *
+ * Locks at different paths are independent of each other, multiple {@link DistributedLock} are therefore requested for
+ * a single operation and are packaged together and returned as an {@link DistributedMultiLock},
+ * see {@link org.apache.solr.cloud.api.collections.CollectionApiLockFactory#createCollectionApiLock} or
+ * {@link org.apache.solr.cloud.ConfigSetApiLockFactory#createConfigSetApiLock}.
+ *
+ * @param isWriteLock {@code true} if requesting a write lock, {@code false} for a read lock.
+ * @param configSetName the config set name, can never be {@code null}.
+ * @return a lock instance that must be {@link DistributedLock#release()}'ed in a {@code finally},
+ * regardless of the lock having been acquired or not.
+ */
+ DistributedLock createLock(boolean isWriteLock, String configSetName);
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java
new file mode 100644
index 00000000000..de618e7618d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+/**
+ * A lock that can be used across cluster nodes.
+ */
+public interface DistributedLock {
+ void waitUntilAcquired();
+ void release();
+ boolean isAcquired();
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java
new file mode 100644
index 00000000000..d0fe596e5f5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A lock as acquired for running a single API command (Collection or Config Set or anything else in the future).
+ * Internally it is composed of multiple {@link DistributedLock}'s.
+ */
+public class DistributedMultiLock {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final List<DistributedLock> locks;
+ private volatile boolean isReleased = false;
+
+ public DistributedMultiLock(List<DistributedLock> locks) {
+ this.locks = locks;
+ }
+
+ public void waitUntilAcquired() {
+ if (isReleased) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Released lock can't be waited upon");
+ }
+
+ for (DistributedLock lock : locks) {
+ log.debug("DistributedMultiLock.waitUntilAcquired. About to wait on lock {}", lock);
+ lock.waitUntilAcquired();
+ log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock);
+ }
+ }
+
+ public void release() {
+ isReleased = true;
+ for (DistributedLock lock : locks) {
+ lock.release();
+ }
+ }
+
+ public boolean isAcquired() {
+ if (isReleased) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Released lock can't be tested");
+ }
+ for (DistributedLock lock : locks) {
+ if (!lock.isAcquired()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @VisibleForTesting
+ public int getCountInternalLocks() {
+ return locks.size();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
index 3a6434e03d7..2596b296ed5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
@@ -16,36 +16,22 @@
*/
package org.apache.solr.cloud;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
import java.util.Set;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.ConfigSetProperties;
-import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
-import static org.apache.solr.common.util.Utils.toJSONString;
-import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
/**
* A {@link OverseerMessageHandler} that handles ConfigSets API related
@@ -58,16 +44,6 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
*/
public static final String CONFIGSETS_ACTION_PREFIX = "configsets:";
- /**
- * Name of the ConfigSet to copy from for CREATE
- */
- public static final String BASE_CONFIGSET = "baseConfigSet";
-
- /**
- * Prefix for properties that should be applied to the ConfigSet for CREATE
- */
- public static final String PROPERTY_PREFIX = "configSetProp";
-
private ZkStateReader zkStateReader;
private CoreContainer coreContainer;
@@ -112,10 +88,10 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
}
switch (action) {
case CREATE:
- createConfigSet(message);
+ ConfigSetCmds.createConfigSet(message, coreContainer);
break;
case DELETE:
- deleteConfigSet(message);
+ ConfigSetCmds.deleteConfigSet(message, coreContainer);
break;
default:
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
@@ -213,160 +189,12 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
private String getBaseConfigSetIfCreate(ZkNodeProps message) {
- String operation = message.getStr(Overseer.QUEUE_OPERATION);
- if (operation != null) {
- operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());
- ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
- if (action == CREATE) {
- String baseConfigSetName = message.getStr(BASE_CONFIGSET);
- if (baseConfigSetName == null || baseConfigSetName.length() == 0) {
- baseConfigSetName = DEFAULT_CONFIGSET_NAME;
- }
- return baseConfigSetName;
- }
- }
- return null;
- }
-
- @SuppressWarnings({"rawtypes"})
- private NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {
- byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);
- if (oldPropsData != null) {
- InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);
- try {
- return ConfigSetProperties.readFromInputStream(reader);
- } finally {
- reader.close();
- }
- }
- return null;
- }
-
- private Map<String, Object> getNewProperties(ZkNodeProps message) {
- Map<String, Object> properties = null;
- for (Map.Entry<String, Object> entry : message.getProperties().entrySet()) {
- if (entry.getKey().startsWith(PROPERTY_PREFIX + ".")) {
- if (properties == null) {
- properties = new HashMap<String, Object>();
- }
- properties.put(entry.getKey().substring((PROPERTY_PREFIX + ".").length()),
- entry.getValue());
- }
- }
- return properties;
- }
-
- private void mergeOldProperties(Map<String, Object> newProps, @SuppressWarnings({"rawtypes"})NamedList oldProps) {
- @SuppressWarnings({"unchecked"})
- Iterator<Map.Entry<String, Object>> it = oldProps.iterator();
- while (it.hasNext()) {
- Map.Entry<String, Object> oldEntry = it.next();
- if (!newProps.containsKey(oldEntry.getKey())) {
- newProps.put(oldEntry.getKey(), oldEntry.getValue());
- }
- }
- }
-
- private byte[] getPropertyData(Map<String, Object> newProps) {
- if (newProps != null) {
- String propertyDataStr = toJSONString(newProps);
- if (propertyDataStr == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid property specification");
- }
- return propertyDataStr.getBytes(StandardCharsets.UTF_8);
- }
- return null;
- }
-
- private void createConfigSet(ZkNodeProps message) throws IOException {
- String configSetName = getTaskKey(message);
- if (configSetName == null || configSetName.length() == 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
- }
-
- String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
-
- if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);
- }
-
- // is there a base config that already exists
- if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {
- throw new SolrException(ErrorCode.BAD_REQUEST,
- "Base ConfigSet does not exist: " + baseConfigSetName);
- }
-
- String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
- Map<String, Object> props = getNewProperties(message);
- if (props != null) {
- // read the old config properties and do a merge, if necessary
- @SuppressWarnings({"rawtypes"})
- NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);
- if (oldProps != null) {
- mergeOldProperties(props, oldProps);
- }
- }
- byte[] propertyData = getPropertyData(props);
-
- try {
- coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);
- if (propertyData != null) {
- coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);
- }
- } catch (Exception e) {
- // copying the config dir or writing the properties file may have failed.
- // we should delete the ConfigSet because it may be invalid,
- // assuming we actually wrote something. E.g. could be
- // the entire baseConfig set with the old properties, including immutable,
- // that would make it impossible for the user to delete.
- try {
- if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
- deleteConfigSet(configSetName, true);
- }
- } catch (IOException ioe) {
- log.error("Error while trying to delete partially created ConfigSet", ioe);
- }
- throw e;
- }
+ String operation = message.getStr(Overseer.QUEUE_OPERATION).substring(CONFIGSETS_ACTION_PREFIX.length());
+ ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
+ return ConfigSetCmds.getBaseConfigSetName(action, message.getStr(ConfigSetCmds.BASE_CONFIGSET));
}
- private void deleteConfigSet(ZkNodeProps message) throws IOException {
- String configSetName = getTaskKey(message);
- if (configSetName == null || configSetName.length() == 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
- }
- deleteConfigSet(configSetName, false);
- }
- private void deleteConfigSet(String configSetName, boolean force) throws IOException {
- if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);
- }
- for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {
- String configName = null;
- try {
- configName = zkStateReader.readConfigName(entry.getKey());
- } catch (KeeperException ex) {
- throw new SolrException(ErrorCode.BAD_REQUEST,
- "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");
- }
- if (configSetName.equals(configName))
- throw new SolrException(ErrorCode.BAD_REQUEST,
- "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");
- }
-
- String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
- @SuppressWarnings({"rawtypes"})
- NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);
- if (properties != null) {
- Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);
- boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
- if (!force && isImmutableConfigSet) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);
- }
- }
- coreContainer.getConfigSetService().deleteConfig(configSetName);
- }
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java
index 0cb6cbe8e87..853db094f26 100644
--- a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java
+++ b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java
@@ -17,6 +17,9 @@
package org.apache.solr.cloud;
import java.util.List;
+import java.util.Map;
+
+import com.google.common.collect.Maps;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.KeeperException;
@@ -50,6 +53,17 @@ public class SizeLimitedDistributedMap extends DistributedMap {
@Override
public void put(String trackingId, byte[] data) throws KeeperException, InterruptedException {
+ shrinkIfNeeded();
+ super.put(trackingId, data);
+ }
+
+ @Override
+ public boolean putIfAbsent(String trackingId, byte[] data) throws KeeperException, InterruptedException {
+ shrinkIfNeeded();
+ return super.putIfAbsent(trackingId, data);
+ }
+
+ private void shrinkIfNeeded() throws KeeperException, InterruptedException {
if (this.size() >= maxSize) {
// Bring down the size
List<String> children = zookeeper.getChildren(dir, null, true);
@@ -63,23 +77,25 @@ public class SizeLimitedDistributedMap extends DistributedMap {
}
};
+ Map<String, Long> childToModificationZxid = Maps.newHashMapWithExpectedSize(children.size());
for (String child : children) {
Stat stat = zookeeper.exists(dir + "/" + child, null, true);
- priorityQueue.insertWithOverflow(stat.getMzxid());
+ if (stat != null) {
+ priorityQueue.insertWithOverflow(stat.getMzxid());
+ childToModificationZxid.put(child, stat.getMzxid());
+ }
}
- long topElementMzxId = priorityQueue.top();
+ long topElementMzxId = priorityQueue.top(); // can be null, but highly unlikely
for (String child : children) {
- Stat stat = zookeeper.exists(dir + "/" + child, null, true);
- if (stat.getMzxid() <= topElementMzxId) {
+ Long id = childToModificationZxid.get(child);
+ if (id != null && id <= topElementMzxId) {
zookeeper.delete(dir + "/" + child, -1, true);
if (onOverflowObserver != null) onOverflowObserver.onChildDelete(child.substring(PREFIX.length()));
}
}
}
-
- super.put(trackingId, data);
}
interface OnOverflowObserver {
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java
new file mode 100644
index 00000000000..d3d00cb5436
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import com.google.common.base.Preconditions;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.CollectionParams;
+
+/**
+ * A distributed lock implementation using Zookeeper "directory" nodes created within a collection znode hierarchy, for
+ * use with the distributed Collection API implementation.
+ * The locks are implemented using ephemeral nodes placed below the "directory" nodes.
+ *
+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>
+ */
+public class ZkDistributedCollectionLockFactory extends ZkDistributedLockFactory implements DistributedCollectionLockFactory {
+
+ public ZkDistributedCollectionLockFactory(SolrZkClient zkClient, String rootPath) {
+ super(zkClient, rootPath);
+ }
+
+ public DistributedLock createLock(boolean isWriteLock, CollectionParams.LockLevel level, String collName, String shardId,
+ String replicaName) {
+ Preconditions.checkArgument(collName != null, "collName can't be null");
+ Preconditions.checkArgument(level == CollectionParams.LockLevel.COLLECTION || shardId != null,
+ "shardId can't be null when getting lock for shard or replica");
+ Preconditions.checkArgument(level != CollectionParams.LockLevel.REPLICA || replicaName != null,
+ "replicaName can't be null when getting lock for replica");
+
+ String lockPath = getLockPath(level, collName, shardId, replicaName);
+ return doCreateLock(isWriteLock, lockPath);
+ }
+
+ /**
+ * Returns the Zookeeper path to the lock, creating missing nodes if needed.
+ * Note that the complete lock hierarchy (/locks and below) can be deleted if SolrCloud is stopped.
+ *
+ * The tree of lock directories for a given collection {@code collName} is as follows:
+ * <pre>
+ * rootPath/
+ * collName/
+ * Locks <-- EPHEMERAL collection level locks go here
+ * _ShardName1/
+ * Locks <-- EPHEMERAL shard level locks go here
+ * _replicaNameS1R1 <-- EPHEMERAL replica level locks go here
+ * _replicaNameS1R2 <-- EPHEMERAL replica level locks go here
+ * _ShardName2/
+ * Locks <-- EPHEMERAL shard level locks go here
+ * _replicaNameS2R1 <-- EPHEMERAL replica level locks go here
+ * _replicaNameS2R2 <-- EPHEMERAL replica level locks go here
+ * </pre>
+ * This method will create the path where the {@code EPHEMERAL} lock nodes should go. That path is:
+ * <ul>
+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION} -
+ * {@code /locks/collections/collName/Locks}</li>
+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD} -
+ * {@code /locks/collections/collName/_shardName/Locks}</li>
+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#REPLICA} -
+ * {@code /locks/collections/collName/_shardName/_replicaName}. There is no {@code Locks} subnode here because replicas do
+ * not have children so no need to separate {@code EPHEMERAL} lock nodes from children nodes as is the case for shards
+ * and collections</li>
+ * </ul>
+ * Note the {@code _} prefixing shards and replica names is to support shards or replicas called "{@code Locks}".
+ * Also note the returned path does not contain the separator ({@code "/"}) at the end.
+ */
+ private String getLockPath(CollectionParams.LockLevel level, String collName, String shardId, String replicaName) {
+ StringBuilder path = getPathPrefix();
+ path.append(collName).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR);
+
+ final String LOCK_NODENAME = "Locks"; // Should not start with SUBNODE_PREFIX :)
+ final String SUBNODE_PREFIX = "_";
+
+ if (level == CollectionParams.LockLevel.COLLECTION) {
+ return path.append(LOCK_NODENAME).toString();
+ } else if (level == CollectionParams.LockLevel.SHARD) {
+ return path.append(SUBNODE_PREFIX).append(shardId).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR).append(LOCK_NODENAME).toString();
+ } else if (level == CollectionParams.LockLevel.REPLICA) {
+ return path.append(SUBNODE_PREFIX).append(shardId).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR).append(SUBNODE_PREFIX).append(replicaName).toString();
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported lock level " + level);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java
new file mode 100644
index 00000000000..b1386a7b88b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import com.google.common.base.Preconditions;
+import org.apache.solr.common.cloud.SolrZkClient;
+
+/**
+ * A distributed lock implementation using Zookeeper "directory" nodes created within a collection znode hierarchy, for
+ * use with the distributed Collection API implementation.
+ * The locks are implemented using ephemeral nodes placed below the "directory" nodes.
+ *
+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>
+ */
+public class ZkDistributedConfigSetLockFactory extends ZkDistributedLockFactory implements DistributedConfigSetLockFactory {
+
+ public ZkDistributedConfigSetLockFactory(SolrZkClient zkClient, String rootPath) {
+ super(zkClient, rootPath);
+ }
+
+ public DistributedLock createLock(boolean isWriteLock, String configSetName) {
+ Preconditions.checkArgument(configSetName != null, "configSetName can't be null");
+
+ String lockPath = getLockPath(configSetName);
+ return doCreateLock(isWriteLock, lockPath);
+ }
+
+ /**
+ * Returns the Zookeeper path to the lock, creating missing nodes if needed.
+ * Note that the complete lock hierarchy (/rootPath and below) can be deleted if SolrCloud is stopped.
+ *
+ * The tree of lock directories is very flat, given there's no real structure to what's being locked in a config set:
+ * <pre>
+ * rootPath/
+ * configSet1/ <-- EPHEMERAL config set locks go here
+ * configSet2/ <-- EPHEMERAL config set locks go here
+ * etc...
+ * </pre>
+ * This method will create the path where the {@code EPHEMERAL} lock nodes should go.<p>
+ * The returned path does not contain the separator ({@code "/"}) at the end.
+ */
+ private String getLockPath(String configSetName) {
+ return getPathPrefix().append(configSetName).toString();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java
new file mode 100644
index 00000000000..6094929fa49
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
+
+/**
+ * A lock to be used across cluster nodes based on Zookeeper. Used for the distributed Collection API and distributed
+ * Config Set API implementations.
+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>
+ */
+abstract class ZkDistributedLock implements DistributedLock {
+ /**
+ * End of the lock node name prefix before the sequential part
+ */
+ static final char LOCK_PREFIX_SUFFIX = '_';
+ /**
+ * Prefix of EPHEMERAL read lock node names
+ */
+ static final String READ_LOCK_PREFIX = "R" + LOCK_PREFIX_SUFFIX;
+ /**
+ * Prefix of EPHEMERAL write lock node names
+ */
+ static final String WRITE_LOCK_PREFIX = "W" + LOCK_PREFIX_SUFFIX;
+
+ /**
+ * Read lock.
+ */
+ static class Read extends ZkDistributedLock {
+ protected Read(SolrZkClient zkClient, String lockPath) throws KeeperException, InterruptedException {
+ super(zkClient, lockPath, READ_LOCK_PREFIX);
+ }
+
+ @Override
+ boolean isBlockedByNodeType(String otherLockName) {
+ // A read lock is only blocked by a lower numbered write lock
+ // Lower numbered read locks are ok, they can coexist.
+ return otherLockName.startsWith(WRITE_LOCK_PREFIX);
+ }
+ }
+
+ /**
+ * Write lock.
+ */
+ static class Write extends ZkDistributedLock {
+ protected Write(SolrZkClient zkClient, String lockPath) throws KeeperException, InterruptedException {
+ super(zkClient, lockPath, WRITE_LOCK_PREFIX);
+ }
+
+ @Override
+ boolean isBlockedByNodeType(String otherLockName) {
+ // A write lock is blocked by another read or write lock with a lower sequence number
+ return true;
+ }
+ }
+
+ private final SolrZkClient zkClient;
+ private final String lockDir;
+ private final String lockNode;
+ protected final long sequence;
+ protected volatile boolean released = false;
+
+ protected ZkDistributedLock(SolrZkClient zkClient, String lockDir, String lockNodePrefix) throws KeeperException, InterruptedException {
+ this.zkClient = zkClient;
+ this.lockDir = lockDir;
+
+ // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually call release() or we block others.
+ lockNode = zkClient.create(lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + lockNodePrefix, null,
+ CreateMode.EPHEMERAL_SEQUENTIAL, false);
+
+ sequence = getSequenceFromNodename(lockNode);
+ }
+
+ private static class DeletedNodeWatcher implements Watcher {
+ final CountDownLatch latch;
+ final String nodeBeingWatched;
+ String errorMessage = null; // non null means error
+
+ DeletedNodeWatcher(String nodeBeingWatched) {
+ this.latch = new CountDownLatch(1);
+ this.nodeBeingWatched = nodeBeingWatched;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType() == Event.EventType.None) {
+ return;
+ } else if (event.getType() != Event.EventType.NodeDeleted) {
+ synchronized (this) {
+ errorMessage = "Received unexpected watch event " + event.getType() + " on " + nodeBeingWatched;
+ }
+ }
+ latch.countDown();
+ }
+
+ void await() throws InterruptedException {
+ latch.await();
+ synchronized (this) {
+ if (errorMessage != null) {
+ throw new SolrException(SERVER_ERROR, errorMessage);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void waitUntilAcquired() {
+ try {
+ if (released) {
+ throw new IllegalStateException("Bug. waitUntilAcquired() should not be called after release(). " + lockNode);
+ }
+ String nodeToWatch = nodeToWatch();
+ while (nodeToWatch != null) {
+ final DeletedNodeWatcher watcher = new DeletedNodeWatcher(nodeToWatch);
+ if (zkClient.exists(nodeToWatch, watcher, true) != null) {
+ watcher.await();
+ }
+ nodeToWatch = nodeToWatch();
+ }
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ @Override
+ public void release() {
+ try {
+ zkClient.delete(lockNode, -1, true);
+ released = true;
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ @Override
+ public boolean isAcquired() {
+ try {
+ if (released) {
+ throw new IllegalStateException("Bug. isAcquired() should not be called after release(). " + lockNode);
+ }
+ return nodeToWatch() == null;
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ /**
+ * @return Another lock node (complete path) that must go away for us to acquire the lock, or {@code null} if the lock is ours.
+ */
+ String nodeToWatch() throws KeeperException, InterruptedException {
+ List<String> locks = zkClient.getChildren(lockDir, null, true);
+ boolean foundSelf = false; // For finding bugs or ZK bad behavior
+ // We deviate from the ZK recipe here: we do not sort the list of nodes, and we stop waiting on the first one we find
+ // that blocks us. This is done in O(n), whereas sorting is more expensive. And we iterate only once over the set of children.
+ // Might cause more wakeups than needed though (multiple locks watching the same one rather than one another) but lock
+ // contention expected low (how many concurrent and conflicting Collection API requests a reasonable user can issue?)
+ // and the code below is simpler. Changing to a sorted "optimal" approach implies only changes in this method.
+ for (String lock : locks) {
+ long seq = getSequenceFromNodename(lock);
+ if (seq == sequence) {
+ foundSelf = true;
+ } else if (seq < sequence && isBlockedByNodeType(lock)) {
+ // Return the full path to the node to watch
+ return lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + lock;
+ }
+ // seq is bigger than sequence, can't block us. If the iteration was sorted we could avoid iterating over those.
+ }
+ if (!foundSelf) {
+ // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only here.
+ throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode);
+ }
+
+ // Didn't return early on any other blocking lock, means we own it
+ return null;
+ }
+
+ /**
+ * @param otherLockName name of a competing lock node. Used to find out the type of the lock (read or write)
+ * @return {@code true} if a lock node of the given type with a lower sequence number blocks us from acquiring the lock.
+ */
+ abstract boolean isBlockedByNodeType(String otherLockName);
+
+ static long getSequenceFromNodename(String lockNode) {
+ // Javadoc of ZooKeeper.create() specifies "The sequence number is always fixed length of 10 digits, 0 padded"
+ // for sequential nodes
+ final int SEQUENCE_LENGTH = 10;
+ // Before the sequence we have our specific char
+ if (lockNode.charAt(lockNode.length() - SEQUENCE_LENGTH - 1) != LOCK_PREFIX_SUFFIX) {
+ throw new SolrException(SERVER_ERROR, "Unexpected lock node path " + lockNode);
+ }
+
+ return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH));
+ }
+
+ @Override
+ public String toString() {
+ return lockNode;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java
new file mode 100644
index 00000000000..4ddffc368f5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+
+abstract class ZkDistributedLockFactory {
+
+ private final SolrZkClient zkClient;
+ private final String rootPath;
+
+ ZkDistributedLockFactory(SolrZkClient zkClient, String rootPath) {
+ this.zkClient = zkClient;
+ this.rootPath = rootPath;
+ }
+
+ protected DistributedLock doCreateLock(boolean isWriteLock, String lockPath) {
+ try {
+ // TODO optimize by first attempting to create the ZkDistributedLock without calling makeLockPath() and only call it
+ // if the lock creation fails. This will be less costly on high contention (and slightly more on low contention)
+ makeLockPath(lockPath);
+
+ return isWriteLock ? new ZkDistributedLock.Write(zkClient, lockPath) : new ZkDistributedLock.Read(zkClient, lockPath);
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ protected StringBuilder getPathPrefix() {
+ StringBuilder path = new StringBuilder(100);
+ path.append(rootPath).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR);
+ return path;
+ }
+
+ private void makeLockPath(String lockNodePath) throws KeeperException, InterruptedException {
+ try {
+ if (!zkClient.exists(lockNodePath, true)) {
+ zkClient.makePath(lockNodePath, new byte[0], CreateMode.PERSISTENT, true);
+ }
+ } catch (KeeperException.NodeExistsException nee) {
+ // Some other thread (on this or another JVM) beat us to create the node, that's ok.
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
index 8acb7024c09..164ee63c4e6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java
@@ -24,9 +24,11 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.OverseerNodePrioritizer;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
@@ -35,6 +37,7 @@ import org.apache.solr.common.cloud.UrlScheme;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
@@ -55,16 +58,15 @@ import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
- * This class contains "smaller" Collection API commands implementation as well as the interface implemented by all commands.
- * Previously these implementations in {@link OverseerCollectionMessageHandler} were relying on methods implementing the
- * functional interface.
+ * This class contains "smaller" Collection API commands implementation, the interface implemented by all commands and the
+ * class mapping a collection action to the actual command.
+ * Previously the "smaller" command implementations in {@link OverseerCollectionMessageHandler} were relying on methods
+ * implementing the functional interface.
*/
public class CollApiCmds {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -76,6 +78,75 @@ public class CollApiCmds {
void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception;
}
+ /**
+ * Map {@link org.apache.solr.common.params.CollectionParams.CollectionAction} to instances of {@link CollApiCmds.CollectionApiCommand} and
+ * being usable by both {@link OverseerCollectionMessageHandler} and {@link DistributedCollectionConfigSetCommandRunner} so that
+ * the mappings do not have to be maintained in two places.
+ */
+ protected static class CommandMap {
+ final private Map<CollectionParams.CollectionAction, CollApiCmds.CollectionApiCommand> commandMap;
+
+ /**
+ * Constructor used when Collection API is run on the Overseer. Called by {@link OverseerCollectionMessageHandler}
+ */
+ CommandMap(CollectionCommandContext ccc, OverseerNodePrioritizer overseerPrioritizer) {
+ this(overseerPrioritizer, ccc);
+ assert !ccc.isDistributedCollectionAPI();
+ }
+
+ /**
+ * Constructor used when Collection API execution is distributed
+ */
+ CommandMap(CollectionCommandContext ccc) {
+ // Overseer prioritizer will not be used. Running in distributed Collection API mode
+ this(null, ccc);
+ assert ccc.isDistributedCollectionAPI();
+ }
+
+ private CommandMap(OverseerNodePrioritizer overseerPrioritizer, CollectionCommandContext ccc) {
+ commandMap = new ImmutableMap.Builder<CollectionParams.CollectionAction, CollApiCmds.CollectionApiCommand>()
+ .put(REPLACENODE, new ReplaceNodeCmd(ccc))
+ .put(DELETENODE, new DeleteNodeCmd(ccc))
+ .put(BACKUP, new BackupCmd(ccc))
+ .put(RESTORE, new RestoreCmd(ccc))
+ .put(DELETEBACKUP, new DeleteBackupCmd(ccc))
+ .put(CREATESNAPSHOT, new CreateSnapshotCmd(ccc))
+ .put(DELETESNAPSHOT, new DeleteSnapshotCmd(ccc))
+ .put(SPLITSHARD, new SplitShardCmd(ccc))
+ .put(ADDROLE, new OverseerRoleCmd(ccc, ADDROLE, overseerPrioritizer))
+ .put(REMOVEROLE, new OverseerRoleCmd(ccc, REMOVEROLE, overseerPrioritizer))
+ .put(MOCK_COLL_TASK, new CollApiCmds.MockOperationCmd())
+ .put(MOCK_SHARD_TASK, new CollApiCmds.MockOperationCmd())
+ .put(MOCK_REPLICA_TASK, new CollApiCmds.MockOperationCmd())
+ .put(CREATESHARD, new CreateShardCmd(ccc))
+ .put(MIGRATE, new MigrateCmd(ccc))
+ .put(CREATE, new CreateCollectionCmd(ccc))
+ .put(MODIFYCOLLECTION, new CollApiCmds.ModifyCollectionCmd(ccc))
+ .put(ADDREPLICAPROP, new CollApiCmds.AddReplicaPropCmd(ccc))
+ .put(DELETEREPLICAPROP, new CollApiCmds.DeleteReplicaPropCmd(ccc))
+ .put(BALANCESHARDUNIQUE, new CollApiCmds.BalanceShardsUniqueCmd(ccc))
+ .put(REBALANCELEADERS, new CollApiCmds.RebalanceLeadersCmd(ccc))
+ .put(RELOAD, new CollApiCmds.ReloadCollectionCmd(ccc))
+ .put(DELETE, new DeleteCollectionCmd(ccc))
+ .put(CREATEALIAS, new CreateAliasCmd(ccc))
+ .put(DELETEALIAS, new DeleteAliasCmd(ccc))
+ .put(ALIASPROP, new SetAliasPropCmd(ccc))
+ .put(MAINTAINROUTEDALIAS, new MaintainRoutedAliasCmd(ccc))
+ .put(OVERSEERSTATUS, new OverseerStatusCmd(ccc))
+ .put(DELETESHARD, new DeleteShardCmd(ccc))
+ .put(DELETEREPLICA, new DeleteReplicaCmd(ccc))
+ .put(ADDREPLICA, new AddReplicaCmd(ccc))
+ .put(MOVEREPLICA, new MoveReplicaCmd(ccc))
+ .put(REINDEXCOLLECTION, new ReindexCollectionCmd(ccc))
+ .put(RENAME, new RenameCmd(ccc))
+ .build();
+ }
+
+ CollApiCmds.CollectionApiCommand getActionCommand(CollectionParams.CollectionAction action) {
+ return commandMap.get(action);
+ }
+ }
+
static public class MockOperationCmd implements CollectionApiCommand {
@SuppressForbidden(reason = "Needs currentTimeMillis for mock requests")
@SuppressWarnings({"unchecked"})
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java
new file mode 100644
index 00000000000..70c7cf785bd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.cloud.DistributedCollectionLockFactory;
+import org.apache.solr.cloud.DistributedLock;
+import org.apache.solr.cloud.DistributedMultiLock;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CollectionParams;
+
+/**
+ * This class implements a higher level locking abstraction for the Collection API using lower level read and write locks.
+ */
+public class CollectionApiLockFactory {
+
+ private final DistributedCollectionLockFactory lockFactory;
+
+ CollectionApiLockFactory(DistributedCollectionLockFactory lockFactory) {
+ this.lockFactory = lockFactory;
+ }
+
+ /**
+ * For the {@link org.apache.solr.common.params.CollectionParams.LockLevel} of the passed {@code action}, obtains the
+ * required locks (if any) and returns.<p>
+ *
+ * This method obtains a write lock at the actual level and path of the action, and also obtains read locks on "lower"
+ * lock levels. For example for a lock at the shard level, a write lock will be requested at the corresponding shard path
+ * and a read lock on the corresponding collection path (in order to prevent an operation locking at the collection level
+ * from executing concurrently with an operation on one of the shards of the collection).
+ * See documentation linked to SOLR-14840 regarding Collection API locking.
+ *
+ * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding Collection
+ * API command can execute safely.
+ * The returned lock <b>MUST</b> be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would
+ * prevent other threads from locking.
+ */
+ DistributedMultiLock createCollectionApiLock(CollectionParams.LockLevel lockLevel, String collName, String shardId, String replicaName) {
+ if (lockLevel == CollectionParams.LockLevel.NONE) {
+ return new DistributedMultiLock(List.of());
+ }
+
+ if (lockLevel == CollectionParams.LockLevel.CLUSTER) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug. Not expecting locking at cluster level.");
+ }
+
+ if (collName == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug. collName can't be null");
+ }
+
+ // There is a bug in the way locks are managed in the Overseer, it sometimes locks at a given level (for example shard)
+ // without knowing on which shard to lock. This can happen for example for SPLITSHARD using a split.key (details below)
+ // or DELETEREPLICA when specifying a count of replicas to delete but no shard.
+ //
+ // When doing a splitshard using a split.key (and not specifying the actual shard name), when the OverseerCollectionMessageHandler
+ // grabs a lock for the Collection API it does not have the shard name and the lock is obtained for a null key.
+ // This means that this lock does not prevent another operation specifically targeting the shard from executing concurrently
+ // (can even be another split if it is specified with a shard name). The null shard lock does prevent another split.key split
+ // for another shard of the same collection at the same time (because trying to obtain another lock on the null key below the
+ // collection will fail).
+ // The splitshard request is built in CollectionAdminRequest.splitShard(). The case of concern is when SplitShard.setSplitKey()
+ // is then called (see test CollectionsAPISolrJTest.testSplitShard()). CollectionOperation.SPLITSHARD_OP in CollectionsHandler
+ // verifies that when the key is specified no shard name is specified.
+ // The locking happens from OverseerTaskProcessor.run() calling OverseerCollectionMessageHandler.lockTask().
+ // The ArrayList passed to LockTree.Session.lock() contains the collection name (index 0) but not shard name (value null
+ // at index 1) yet the action.lockLevel is SHARD. Note that there ends up being a single shard name associated with the
+ // split key. It is computed in SplitShardCmd.getParentSlice().
+ //
+ // We deal with the issue to trigger the same (wrong) behavior for compatibility of distributed Collection API vs Overseer
+ // based collection API. We make up a shard name that is highly unlikely to otherwise exist and lock it instead.
+ // Another option would be to bump up the locking level from SHARD to COLLECTION when no shard is specified (would be
+ // "more correct" from the perspective of preventing competing Collection API commands from running concurrently, but
+ // would restrict concurrency a bit more than existing solution, and if some installations depend on current behavior...).
+ if (lockLevel == CollectionParams.LockLevel.SHARD && shardId == null) {
+ shardId = "MadeUpShardNameWhenNoShardNameWasProvided";
+ // Bumping locking level to collection would be instead: lockLevel = CollectionParams.LockLevel.COLLECTION;
+ }
+
+ // The first requested lock is a write one (on the target object for the action, depending on lock level), then requesting
+ // read locks on "higher" levels (collection > shard > replica here for the level. Note LockLevel "height" is other way around).
+ boolean requestWriteLock = true;
+ final CollectionParams.LockLevel[] iterationOrder = { CollectionParams.LockLevel.REPLICA, CollectionParams.LockLevel.SHARD, CollectionParams.LockLevel.COLLECTION };
+ List<DistributedLock> locks = new ArrayList<>(iterationOrder.length);
+ for (CollectionParams.LockLevel level : iterationOrder) {
+ // This comparison is based on the LockLevel height value that classifies replica > shard > collection.
+ if (lockLevel.isHigherOrEqual(level)) {
+ locks.add(lockFactory.createLock(requestWriteLock, level, collName, shardId, replicaName));
+ requestWriteLock = false;
+ }
+ }
+
+ return new DistributedMultiLock(locks);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java
index b9fe9bd34d2..92a226b6448 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java
@@ -37,6 +37,11 @@ import java.util.concurrent.ExecutorService;
* distributed way, unrelated to and not depending upon Overseer abstractions (including overseer collection message handling).
*/
public interface CollectionCommandContext {
+ /**
+ * When this method returns {@code true}, Overseer specific actions do not make sense and commands should not be doing them.
+ */
+ boolean isDistributedCollectionAPI();
+
ShardHandler newShardHandler();
SolrCloudManager getSolrCloudManager();
@@ -61,11 +66,12 @@ public interface CollectionCommandContext {
ExecutorService getExecutorService();
/**
- * This method enables the commands to enqueue to the overseer cluster state update. This should only be used when the command
- * is running in the Overseer (and will throw an exception if called when Collection API is distributed)
+ * This method enables the commands to enqueue to the overseer cluster state update. This should only be used when the
+ * cluster state update is running in the Overseer (and will throw an exception if called when Collection API is distributed
+ * given that distributed Collection API implies distributed Cluster State Update)
*/
default void offerStateUpdate(byte[] data) throws KeeperException, InterruptedException {
- throw new IllegalStateException("Bug! offerStateUpdate() should not be called when distributed state updates are enabled");
+ throw new IllegalStateException("Bug! offerStateUpdate() should not be called when distributed cluster state updates are enabled");
}
default String getOverseerId() {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index f5e8c1e159b..4dab2e8295a 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -54,7 +54,6 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.CoreContainer;
@@ -83,13 +82,9 @@ import static org.apache.solr.handler.admin.ConfigSetsHandler.getSuffixedNameFor
public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final CollectionCommandContext ccc;
- private final TimeSource timeSource;
- private final DistribStateManager stateManager;
public CreateCollectionCmd(CollectionCommandContext ccc) {
this.ccc = ccc;
- this.stateManager = ccc.getSolrCloudManager().getDistribStateManager();
- this.timeSource = ccc.getSolrCloudManager().getTimeSource();
}
@Override
@@ -133,7 +128,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
ZkStateReader zkStateReader = ccc.getZkStateReader();
// this also creates the collection zk node as a side-effect
- CollectionHandlingUtils.createConfNode(stateManager, configName, collectionName);
+ CollectionHandlingUtils.createConfNode(ccc.getSolrCloudManager().getDistribStateManager(), configName, collectionName);
Map<String,String> collectionParams = new HashMap<>();
Map<String,Object> collectionProps = message.getProperties();
@@ -144,7 +139,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
}
}
- createCollectionZkNode(stateManager, collectionName, collectionParams, ccc.getCoreContainer().getConfigSetService());
+ createCollectionZkNode(ccc.getSolrCloudManager().getDistribStateManager(), collectionName, collectionParams, ccc.getCoreContainer().getConfigSetService());
// Note that in code below there are two main execution paths: Overseer based cluster state updates and distributed
// cluster state updates (look for isDistributedStateUpdate() conditions).
@@ -169,6 +164,8 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
// When cluster state updates are handled by Overseer, ask it to load that collection it doesn't know about.
// When cluster state updates are distributed, ZK is the source of truth for all nodes so no reload needed.
if (!ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {
+ // If cluster state update is not distributed and we execute here, the Collection API is not distributed either
+ // and this execution happens on the Overseer node, so direct memory access as done below is ok.
ccc.submitIntraProcessMessage(new RefreshCollectionMessage(collectionName));
}
} else {
@@ -182,7 +179,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
}
// wait for a while until we see the collection
- TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, timeSource);
+ TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());
boolean created = false;
while (!waitUntil.hasTimedOut()) {
waitUntil.sleep(100);
@@ -341,7 +338,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
@SuppressWarnings({"rawtypes"})
boolean failure = results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0;
if (isPRS) {
- TimeOut timeout = new TimeOut(Integer.getInteger("solr.waitToSeeReplicasInStateTimeoutSeconds", 120), TimeUnit.SECONDS, timeSource); // could be a big cluster
+ TimeOut timeout = new TimeOut(Integer.getInteger("solr.waitToSeeReplicasInStateTimeoutSeconds", 120), TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource()); // could be a big cluster
PerReplicaStates prs = PerReplicaStates.fetch(collectionPath, ccc.getZkStateReader().getZkClient(), null);
while (!timeout.hasTimedOut()) {
if(prs.allActive()) break;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index 6612deda1fb..df7895c06e7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -175,7 +175,7 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {
boolean configSetIsUsedByOtherCollection = false;
// make sure the configSet is not shared with other collections
- // Similar to what happens in: OverseerConfigSetMessageHandler::deleteConfigSet
+ // Similar to what happens in: ConfigSetCmds::deleteConfigSet
for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {
String otherConfigSetName = null;
try {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.java
new file mode 100644
index 00000000000..e63c5e067a2
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.util.concurrent.ExecutorService;
+
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.cloud.DistributedClusterStateUpdater;
+import org.apache.solr.common.SolrCloseable;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.component.ShardHandler;
+
+public class DistributedCollectionCommandContext implements CollectionCommandContext {
+ private final CoreContainer coreContainer;
+ private final DistributedClusterStateUpdater getDistributedClusterStateUpdater;
+ private final ExecutorService executorService;
+
+ private final SolrCloudManager solrCloudManager;
+ private final ZkStateReader zkStateReader;
+
+ public DistributedCollectionCommandContext(CoreContainer coreContainer, ExecutorService executorService) {
+ this.coreContainer = coreContainer;
+ this.getDistributedClusterStateUpdater = new DistributedClusterStateUpdater(coreContainer.getConfig().getCloudConfig().getDistributedClusterStateUpdates());;
+ this.executorService = executorService;
+
+ solrCloudManager = this.coreContainer.getZkController().getSolrCloudManager();
+ zkStateReader = this.coreContainer.getZkController().getZkStateReader();
+ }
+
+ @Override
+ public boolean isDistributedCollectionAPI() {
+ return true;
+ }
+
+ @Override
+ public ShardHandler newShardHandler() {
+ // This method builds a new shard handler! A given shard handler can't be reused because it can only serve a single thread.
+ return this.coreContainer.getShardHandlerFactory().getShardHandler();
+ }
+
+ @Override
+ public SolrCloudManager getSolrCloudManager() {
+ return solrCloudManager;
+ }
+
+ @Override
+ public CoreContainer getCoreContainer() {
+ return this.coreContainer;
+ }
+
+ @Override
+ public ZkStateReader getZkStateReader() {
+ return zkStateReader;
+ }
+
+ @Override
+ public DistributedClusterStateUpdater getDistributedClusterStateUpdater() {
+ return this.getDistributedClusterStateUpdater;
+ }
+
+ @Override
+ public SolrCloseable getCloseableToLatchOn() {
+ // Debatable: SolrCloudManager instance is tracked for closing to interrupt some command execution. Could very well monitor something else.
+ return getSolrCloudManager();
+ }
+
+ @Override
+ public ExecutorService getExecutorService() {
+ return this.executorService;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java
new file mode 100644
index 00000000000..57ad7635370
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.cloud.ConfigSetApiLockFactory;
+import org.apache.solr.cloud.ConfigSetCmds;
+import org.apache.solr.cloud.DistributedApiAsyncTracker;
+import org.apache.solr.cloud.DistributedMultiLock;
+import org.apache.solr.cloud.OverseerSolrResponse;
+import org.apache.solr.cloud.ZkDistributedCollectionLockFactory;
+import org.apache.solr.cloud.ZkDistributedConfigSetLockFactory;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.ConfigSetParams;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.admin.ConfigSetsHandler;
+import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonParams.NAME;
+
+/**
+ * Class for execution Collection API and Config Set API commands in a distributed way, without going through Overseer and
+ * {@link OverseerCollectionMessageHandler} or {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler}.<p>
+ *
+ * This class is only called when Collection and Config Set API calls are configured to be distributed, which implies
+ * cluster state updates are distributed as well.
+ */
+public class DistributedCollectionConfigSetCommandRunner {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static final String ZK_PATH_SEPARATOR = "/";
+
+ private static final String ZK_DISTRIBUTED_API_ROOT = "/distributedapi";
+
+ /**
+ * Zookeeper node below which the locking hierarchy is anchored
+ */
+ private static final String ZK_COLLECTION_LOCKS = ZK_DISTRIBUTED_API_ROOT + "/collectionlocks";
+ private static final String ZK_CONFIG_SET_LOCKS = ZK_DISTRIBUTED_API_ROOT + "/configsetlocks";
+
+ /**
+ * Zookeeper node below which the async id (requestId) tracking is done (for async requests)
+ */
+ private static final String ZK_ASYNC_ROOT = ZK_DISTRIBUTED_API_ROOT + "/async";
+
+ private final ExecutorService distributedCollectionApiExecutorService;
+ /**
+ * All Collection API commands are executed as if they are asynchronous to stick to the same behavior as the Overseer
+ * based Collection API execution. The difference between sync and async is twofold:
+ * 1. Non async execution does wait for a while to see if the command has completed and if so, returns success (and if
+ * not, returns failure)
+ * 2. There is no way to query the status of a non async command that is still running, or to kill it (short of shutting
+ * down the node on which it's running). This is not the best design, but at this stage that's what happens with
+ * Overseer based Collection API, so implementing the same.
+ * <p>
+ * The common aspects between sync and async is that actual command execution is given an infinite time to eventually
+ * complete (or fail). Again this is not great, but that's how it is for now. Likely some cleanup required later (once
+ * Overseer is actually remove? Like Solr 10 or 11?).
+ */
+ private final ExecutorService commandsExecutor;
+ private final CoreContainer coreContainer;
+ final private CollApiCmds.CommandMap commandMapper;
+ private final CollectionCommandContext ccc;
+ private final DistributedApiAsyncTracker asyncTaskTracker;
+
+ private volatile boolean shuttingDown = false;
+
+ public DistributedCollectionConfigSetCommandRunner(CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+
+ if (log.isInfoEnabled()) {
+ // Note is it hard to print a log when Collection API is handled by Overseer because Overseer is started regardless
+ // of how Collection API is handled, so it doesn't really know...
+ log.info("Creating DistributedCollectionConfigSetCommandRunner. Collection and ConfigSet APIs are running distributed (not Overseer based)");
+ }
+
+ // TODO we should look at how everything is getting closed when the node is shutdown.
+ // But it seems that CollectionsHandler (that creates instances of this class) is not really closed, so maybe it doesn't matter?
+ // With distributed Collection API execution, each node will have such an executor but given how thread pools work,
+ // threads will only be created if needed (including the corePoolSize threads).
+ distributedCollectionApiExecutorService = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 10, 0L, TimeUnit.MILLISECONDS,
+ new SynchronousQueue<>(),
+ new SolrNamedThreadFactory("DistributedCollectionApiExecutorService"));
+
+ commandsExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 20, 0L, TimeUnit.MILLISECONDS,
+ new SynchronousQueue<>(),
+ new SolrNamedThreadFactory("DistributedCollectionApiCommandExecutor"));
+
+ ccc = new DistributedCollectionCommandContext(this.coreContainer, this.distributedCollectionApiExecutorService);
+ commandMapper = new CollApiCmds.CommandMap(ccc);
+ asyncTaskTracker = new DistributedApiAsyncTracker(ccc.getZkStateReader().getZkClient(), ZK_ASYNC_ROOT);
+ }
+
+ /**
+ * See {@link DistributedApiAsyncTracker#getAsyncTaskRequestStatus(String)}
+ */
+ public Pair<RequestStatusState, OverseerSolrResponse> getAsyncTaskRequestStatus(String asyncId) throws Exception {
+ return asyncTaskTracker.getAsyncTaskRequestStatus(asyncId);
+ }
+
+ /**
+ * See {@link DistributedApiAsyncTracker#deleteSingleAsyncId(String)}
+ */
+ public boolean deleteSingleAsyncId(String asyncId) throws Exception {
+ return asyncTaskTracker.deleteSingleAsyncId(asyncId);
+ }
+
+ /**
+ * See {@link DistributedApiAsyncTracker#deleteAllAsyncIds()}
+ */
+ public void deleteAllAsyncIds() throws Exception {
+ asyncTaskTracker.deleteAllAsyncIds();
+ }
+
+
+ /**
+ * When {@link org.apache.solr.handler.admin.CollectionsHandler#invokeAction} does not enqueue to overseer queue and
+ * instead calls this method, this method is expected to do the equivalent of what Overseer does in
+ * {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler#processMessage}.
+ * <p>
+ * The steps leading to that call in the Overseer execution path are (and the equivalent is done here):
+ * <ul>
+ * <li>{@link org.apache.solr.cloud.OverseerTaskProcessor#run()} gets the message from the ZK queue, grabs the
+ * corresponding locks (write lock on the config set target of the API command and a read lock on the base config set
+ * if any - the case for config set creation) then executes the command using an executor service (it also checks the
+ * asyncId if any is specified but async calls are not supported for Config Set API calls).</li>
+ * <li>In {@link org.apache.solr.cloud.OverseerTaskProcessor}.{@code Runner.run()} (run on an executor thread) a call is made to
+ * {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler#processMessage} which does a few checks and calls the
+ * appropriate Config Set method.
+ * </ul>
+ */
+ public void runConfigSetCommand(SolrQueryResponse rsp, ConfigSetsHandler.ConfigSetOperation operation, Map<String, Object> result, long timeoutMs) throws Exception {
+ // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it through this method earlier).
+ // See stopAndWaitForPendingTasksToComplete() below
+ if (shuttingDown) {
+ throw new SolrException(SolrException.ErrorCode.CONFLICT, "Solr is shutting down, no more Config Set API tasks may be executed");
+ }
+
+ ConfigSetParams.ConfigSetAction action = operation.getAction();
+
+ // never null
+ String configSetName = (String) result.get(NAME);
+ // baseConfigSetName will be null if we're not creating a new config set
+ String baseConfigSetName = ConfigSetCmds.getBaseConfigSetName(action, (String) result.get(ConfigSetCmds.BASE_CONFIGSET));
+
+ if (log.isInfoEnabled()) {
+ log.info("Running Config Set API locally for " + action + " " + configSetName); // nowarn
+ }
+
+ ConfigSetCommandRunner commandRunner = new ConfigSetCommandRunner(new ZkNodeProps(result), action, configSetName, baseConfigSetName);
+ final Future<Void> taskFuture;
+ try {
+ taskFuture = commandsExecutor.submit(commandRunner);
+ } catch (RejectedExecutionException ree) {
+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree);
+ }
+
+ // Wait for a while... Just like Overseer based Config Set API (wait can timeout but actual command execution does not)
+ try {
+ taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException te) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " " + configSetName + " timed out after " + timeoutMs + "ms");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " " + configSetName + " interrupted", e);
+ }
+ }
+
+ /**
+ * When {@link org.apache.solr.handler.admin.CollectionsHandler#invokeAction} does not enqueue to overseer queue and
+ * instead calls this method, this method is expected to do the equivalent of what Overseer does in
+ * {@link org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler#processMessage}.
+ * <p>
+ * The steps leading to that call in the Overseer execution path are (and the equivalent is done here):
+ * <ul>
+ * <li>{@link org.apache.solr.cloud.OverseerTaskProcessor#run()} gets the message from the ZK queue, grabs the
+ * corresponding lock (Collection API calls do locking to prevent non compatible concurrent modifications of a collection),
+ * marks the async id of the task as running then executes the command using an executor service</li>
+ * <li>In {@link org.apache.solr.cloud.OverseerTaskProcessor}.{@code Runner.run()} (run on an executor thread) a call is made to
+ * {@link org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler#processMessage} which sets the logging
+ * context, calls {@link CollApiCmds.CollectionApiCommand#call}
+ * </ul>
+ */
+ public OverseerSolrResponse runCollectionCommand(ZkNodeProps message, CollectionParams.CollectionAction action, long timeoutMs) {
+ // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it through this method earlier).
+ // See stopAndWaitForPendingTasksToComplete() below
+ if (shuttingDown) {
+ throw new SolrException(SolrException.ErrorCode.CONFLICT, "Solr is shutting down, no more Collection API tasks may be executed");
+ }
+
+ final String asyncId = message.getStr(ASYNC);
+
+ if (log.isInfoEnabled()) {
+ log.info("Running Collection API locally for " + action.name() + " asyncId=" + asyncId); // nowarn
+ }
+
+ // Following the call below returning true, we must eventually cancel or complete the task. Happens either in the
+ // CollectionCommandRunner below or in the catch when the runner would not execute.
+ if (!asyncTaskTracker.createNewAsyncJobTracker(asyncId)) {
+ NamedList<String> resp = new NamedList<>();
+ resp.add("error", "Task with the same requestid already exists. (" + asyncId + ")");
+ resp.add(CoreAdminParams.REQUESTID, asyncId);
+ return new OverseerSolrResponse(resp);
+ }
+
+ CollectionCommandRunner commandRunner = new CollectionCommandRunner(message, action, asyncId);
+ final Future<OverseerSolrResponse> taskFuture;
+ try {
+ taskFuture = commandsExecutor.submit(commandRunner);
+ } catch (RejectedExecutionException ree) {
+ // The command will not run, need to cancel the async ID so it can be reused on a subsequent attempt by the client
+ asyncTaskTracker.cancelAsyncId(asyncId);
+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree);
+ }
+
+ if (asyncId == null) {
+ // Non async calls wait for a while in case the command completes. If they time out, there's no way to track the
+ // job progress (improvement suggestion: decorrelate having a task ID from the fact of waiting for the job to complete)
+ try {
+ return taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException te) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " timed out after " + timeoutMs + "ms");
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " interrupted", e);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " failed", e);
+ }
+ } else {
+ // Async calls do not wait for the command to finish but get instead back the async id (that they just sent...)
+ NamedList<String> resp = new NamedList<>();
+ resp.add(CoreAdminParams.REQUESTID, asyncId);
+ return new OverseerSolrResponse(resp);
+ }
+ }
+
+ /**
+ * Best effort wait for termination of all tasks for a (short) while, then return. This method is called when the JVM
+ * shuts down so tasks that did not complete are expected to be stopped mid processing.
+ */
+ public void stopAndWaitForPendingTasksToComplete() {
+ shuttingDown = true;
+
+ commandsExecutor.shutdown();
+ distributedCollectionApiExecutorService.shutdown();
+
+ // Duration we are willing to wait for threads to terminate, total.
+ final long TOTAL_WAIT_NS = 10L * 1000 * 1000 * 1000;
+
+ long start = System.nanoTime();
+ try {
+ commandsExecutor.awaitTermination(TOTAL_WAIT_NS, TimeUnit.NANOSECONDS);
+ long remaining = TOTAL_WAIT_NS - (System.nanoTime() - start);
+ if (remaining > 0L) {
+ distributedCollectionApiExecutorService.awaitTermination(remaining, TimeUnit.NANOSECONDS);
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Collection name can be found in either of two message parameters (why??). Return it from where it's defined.
+ * (see also parameter {@code collectionNameParamName} of {@link org.apache.solr.cloud.DistributedClusterStateUpdater.MutatingCommand#MutatingCommand(CollectionParams.CollectionAction, String)})
+ */
+ public static String getCollectionName(ZkNodeProps message) {
+ return message.containsKey(COLLECTION_PROP) ?
+ message.getStr(COLLECTION_PROP) : message.getStr(NAME);
+ }
+
+ /**
+ * All Collection API commands are executed in separate threads so that the command can run to completion even if the
+ * client request times out, or for async tasks, the client request returns and the command is executed (which really
+ * is very similar).
+ * This provides a behavior very similar to the one provided by Overseer based Collection API execution.
+ */
+ private class CollectionCommandRunner implements Callable<OverseerSolrResponse> {
+ private final ZkNodeProps message;
+ private final CollectionParams.CollectionAction action;
+ private final String asyncId;
+
+ private CollectionCommandRunner(ZkNodeProps message, CollectionParams.CollectionAction action, String asyncId) {
+ this.message = message;
+ this.action = action;
+ this.asyncId = asyncId;
+ }
+
+ /**
+ * This method does always free the Collection API lock it grabs and updates the tracking of the {@code asyncId}
+ * (initiated before this method got called) on all execution paths out of this method!
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public OverseerSolrResponse call() {
+ final String collName = getCollectionName(message);
+ final String shardId = message.getStr(SHARD_ID_PROP);
+ final String replicaName = message.getStr(REPLICA_PROP);
+
+ MDCLoggingContext.setCollection(collName);
+ MDCLoggingContext.setShard(shardId);
+ MDCLoggingContext.setReplica(replicaName);
+
+ @SuppressWarnings({"rawtypes"})
+ NamedList results = new NamedList();
+ try {
+ // Create API lock for executing the command. This call is non blocking (not blocked on waiting for a lock to be acquired anyway,
+ // might be blocked on access to ZK etc)
+ // We create a new CollectionApiLockFactory using a new ZkDistributedCollectionLockFactory because earlier (in the constructor of this class)
+ // the ZkStateReader was not yet available, due to how CoreContainer is built in part in the constructor and in part in its
+ // load() method. And this class is built from the CoreContainer constructor...
+ // The cost of these creations is low, and these classes do not hold state but only serve as an interface to Zookeeper.
+ // Note that after this call, we MUST execute the lock.release(); in the finally below
+ DistributedMultiLock lock = new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)).createCollectionApiLock(action.lockLevel, collName, shardId, replicaName);
+
+ try {
+ log.debug("CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}",
+ action, action.lockLevel, collName, shardId, replicaName);
+
+ // Block this thread until all required locks are acquired.
+ lock.waitUntilAcquired();
+
+ // Got the lock so moving from submitted to running if we run for an async task (if asyncId is null the asyncTaskTracker
+ // calls do nothing).
+ asyncTaskTracker.setTaskRunning(asyncId);
+
+ log.debug("DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", action, message);
+
+ CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action);
+ if (command != null) {
+ command.call(ccc.getSolrCloudManager().getClusterStateProvider().getClusterState(), message, results);
+ } else {
+ asyncTaskTracker.cancelAsyncId(asyncId);
+ // Seeing this is a bug, not bad user data
+ String message = "Bug: Unknown operation " + action;
+ log.error(message);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message);
+ }
+ } finally {
+ try {
+ // TODO If the Collection API command failed because the collection does not exist, we've just created some lock directory
+ // structure for a non existent collection... Maybe try to remove it here? No big deal for now as leftover nodes in the
+ // lock hierarchy do no harm, and there shouldn't be too many of those.
+ lock.release();
+ } catch (SolrException se) {
+ log.error("Error when releasing collection locks for operation " + action, se); // nowarn
+ }
+ }
+ } catch (Exception e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ // Output some error logs
+ if (collName == null) {
+ SolrException.log(log, "Operation " + action + " failed", e);
+ } else {
+ SolrException.log(log, "Collection " + collName + ", operation " + action + " failed", e);
+ }
+
+ results.add("Operation " + action + " caused exception:", e);
+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
+ nl.add("msg", e.getMessage());
+ nl.add("rspCode", e instanceof SolrException ? ((SolrException)e).code() : -1);
+ results.add("exception", nl);
+ }
+
+ OverseerSolrResponse res = new OverseerSolrResponse(results);
+ // Following call marks success or failure depending on the contents of res
+ asyncTaskTracker.setTaskCompleted(asyncId, res);
+ return res;
+ }
+ }
+
+ /**
+ * All Config Set API commands are executed in separate threads so that the command can run to completion even if the
+ * client request times out.<br>
+ * This provides a behavior very similar to the one provided by Overseer based Config Set API execution.<p>
+ *
+ * The logic of this class is similar to (and simpler than) the one of {@link CollectionCommandRunner}, more detailed
+ * comments there.
+ */
+ private class ConfigSetCommandRunner implements Callable<Void> {
+ private final ZkNodeProps message;
+ private final ConfigSetParams.ConfigSetAction action;
+ private final String configSetName;
+ private final String baseConfigSetName;
+
+ private ConfigSetCommandRunner(ZkNodeProps message, ConfigSetParams.ConfigSetAction action, String configSetName, String baseConfigSetName) {
+ this.message = message;
+ this.action = action;
+ this.configSetName = configSetName;
+ this.baseConfigSetName = baseConfigSetName;
+ }
+
+ /**
+ * This method does always free the Collection API lock it grabs.
+ */
+ @Override
+ public Void call() throws IOException {
+ // After this call, we MUST execute the lock.release(); in the finally below
+ DistributedMultiLock lock =
+ new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(ccc.getZkStateReader().getZkClient(), ZK_CONFIG_SET_LOCKS))
+ .createConfigSetApiLock(configSetName, baseConfigSetName);
+
+ try {
+ log.debug("ConfigSetCommandRunner about to acquire lock for action {} config set {} base config set {}",
+ action, configSetName, baseConfigSetName);
+
+ // Block this thread until all required locks are acquired.
+ lock.waitUntilAcquired();
+
+ log.debug("ConfigSetCommandRunner. Lock acquired. Calling: {}, {}", action, message);
+
+ switch (action) {
+ case CREATE:
+ ConfigSetCmds.createConfigSet(message, coreContainer);
+ break;
+ case DELETE:
+ ConfigSetCmds.deleteConfigSet(message, coreContainer);
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug! Unknown Config Set action: " + action);
+ }
+ } finally {
+ lock.release();
+ }
+
+ return null;
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
index 280eb407cca..0df599a6cf0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
@@ -58,12 +58,12 @@ public class MaintainRoutedAliasCmd extends AliasCmd {
*/
static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol)
throws Exception {
- final String operation = CollectionParams.CollectionAction.MAINTAINROUTEDALIAS.toLower();
+ final CollectionParams.CollectionAction maintainroutedalias = CollectionParams.CollectionAction.MAINTAINROUTEDALIAS;
Map<String, Object> msg = new HashMap<>();
- msg.put(Overseer.QUEUE_OPERATION, operation);
+ msg.put(Overseer.QUEUE_OPERATION, maintainroutedalias.toLower());
msg.put(CollectionParams.NAME, aliasName);
msg.put(MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol);
- final SolrResponse rsp = collHandler.sendToOCPQueue(new ZkNodeProps(msg));
+ final SolrResponse rsp = collHandler.submitCollectionApiCommand(new ZkNodeProps(msg), maintainroutedalias);
if (rsp.getException() != null) {
throw rsp.getException();
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java
index 582c4738c8a..4416f236309 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java
@@ -39,6 +39,11 @@ public class OcmhCollectionCommandContext implements CollectionCommandContext {
this.ocmh = ocmh;
}
+ @Override
+ public boolean isDistributedCollectionAPI() {
+ return false;
+ }
+
@Override
public ShardHandler newShardHandler() {
return ocmh.shardHandlerFactory.getShardHandler();
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index fa79e5a3829..44c8f3d6d44 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud.api.collections;
-import com.google.common.collect.ImmutableMap;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.cloud.LockTree;
import org.apache.solr.cloud.Overseer;
@@ -43,20 +42,19 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Arrays;
-import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonParams.NAME;
/**
* A {@link OverseerMessageHandler} that handles Collections API related overseer messages.<p>
*
- * A lot of the content that was in this class got moved to {@link CollectionHandlingUtils} and {@link CollApiCmds}.
+ * A lot of the content that was in this class got moved to {@link CollectionHandlingUtils} and {@link CollApiCmds}.<p>
+ * The equivalent of this class for distributed Collection API command execution is {@link DistributedCollectionConfigSetCommandRunner}.
*/
public class OverseerCollectionMessageHandler implements OverseerMessageHandler, SolrCloseable {
@@ -71,7 +69,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
String myId;
Stats stats;
TimeSource timeSource;
- private final CollectionCommandContext ccc;
// Set that tracks collections that are currently being processed by a running task.
// This is used for handling mutual exclusion of the tasks.
@@ -81,7 +78,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
new SynchronousQueue<>(),
new SolrNamedThreadFactory("OverseerCollectionMessageHandlerThreadFactory"));
- final private Map<CollectionAction, CollApiCmds.CollectionApiCommand> commandMap;
+ final private CollApiCmds.CommandMap commandMapper;
private volatile boolean isClosed;
@@ -100,44 +97,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
this.cloudManager = overseer.getSolrCloudManager();
this.timeSource = cloudManager.getTimeSource();
this.isClosed = false;
- ccc = new OcmhCollectionCommandContext(this);
- commandMap = new ImmutableMap.Builder<CollectionAction, CollApiCmds.CollectionApiCommand>()
- .put(REPLACENODE, new ReplaceNodeCmd(ccc))
- .put(DELETENODE, new DeleteNodeCmd(ccc))
- .put(BACKUP, new BackupCmd(ccc))
- .put(RESTORE, new RestoreCmd(ccc))
- .put(DELETEBACKUP, new DeleteBackupCmd(ccc))
- .put(CREATESNAPSHOT, new CreateSnapshotCmd(ccc))
- .put(DELETESNAPSHOT, new DeleteSnapshotCmd(ccc))
- .put(SPLITSHARD, new SplitShardCmd(ccc))
- .put(ADDROLE, new OverseerRoleCmd(ccc, ADDROLE, overseerPrioritizer))
- .put(REMOVEROLE, new OverseerRoleCmd(ccc, REMOVEROLE, overseerPrioritizer))
- .put(MOCK_COLL_TASK, new CollApiCmds.MockOperationCmd())
- .put(MOCK_SHARD_TASK, new CollApiCmds.MockOperationCmd())
- .put(MOCK_REPLICA_TASK, new CollApiCmds.MockOperationCmd())
- .put(CREATESHARD, new CreateShardCmd(ccc))
- .put(MIGRATE, new MigrateCmd(ccc))
- .put(CREATE, new CreateCollectionCmd(ccc))
- .put(MODIFYCOLLECTION, new CollApiCmds.ModifyCollectionCmd(ccc))
- .put(ADDREPLICAPROP, new CollApiCmds.AddReplicaPropCmd(ccc))
- .put(DELETEREPLICAPROP, new CollApiCmds.DeleteReplicaPropCmd(ccc))
- .put(BALANCESHARDUNIQUE, new CollApiCmds.BalanceShardsUniqueCmd(ccc))
- .put(REBALANCELEADERS, new CollApiCmds.RebalanceLeadersCmd(ccc))
- .put(RELOAD, new CollApiCmds.ReloadCollectionCmd(ccc))
- .put(DELETE, new DeleteCollectionCmd(ccc))
- .put(CREATEALIAS, new CreateAliasCmd(ccc))
- .put(DELETEALIAS, new DeleteAliasCmd(ccc))
- .put(ALIASPROP, new SetAliasPropCmd(ccc))
- .put(MAINTAINROUTEDALIAS, new MaintainRoutedAliasCmd(ccc))
- .put(OVERSEERSTATUS, new OverseerStatusCmd(ccc))
- .put(DELETESHARD, new DeleteShardCmd(ccc))
- .put(DELETEREPLICA, new DeleteReplicaCmd(ccc))
- .put(ADDREPLICA, new AddReplicaCmd(ccc))
- .put(MOVEREPLICA, new MoveReplicaCmd(ccc))
- .put(REINDEXCOLLECTION, new ReindexCollectionCmd(ccc))
- .put(RENAME, new RenameCmd(ccc))
- .build()
- ;
+ commandMapper = new CollApiCmds.CommandMap(new OcmhCollectionCommandContext(this), overseerPrioritizer);
}
@Override
@@ -152,7 +112,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
NamedList results = new NamedList();
try {
CollectionAction action = getCollectionAction(operation);
- CollApiCmds.CollectionApiCommand command = commandMap.get(action);
+ CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action);
if (command != null) {
command.call(cloudManager.getClusterStateProvider().getClusterState(), message, results);
} else {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
index d071e392390..c8bb3e45b0b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
@@ -56,6 +56,11 @@ public class OverseerRoleCmd implements CollApiCmds.CollectionApiCommand {
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
+ if (ccc.isDistributedCollectionAPI()) {
+ // No Overseer (not accessible from Collection API command execution in any case) so this command can't be run...
+ log.error("Cluster is running with distributed Collection API execution. Ignoring Collection API operation " + operation); // nowarn
+ return;
+ }
ZkStateReader zkStateReader = ccc.getZkStateReader();
SolrZkClient zkClient = zkStateReader.getZkClient();
Map roles = null;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java
index c7efe4b8ae3..6cc6b9bdfe5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java
@@ -41,6 +41,18 @@ import org.slf4j.LoggerFactory;
* <b>within the current Overseer node</b> (this is important because distributed operations occurring on other nodes
* are <b>not included</b> in these stats, for example distributed cluster state updates or Per Replica States updates).<p>
*
+ * More fundamentally, when the Collection API command execution is distributed, this specific command is <b>not</b> being
+ * run on the Overseer anyway (but then not much is running on the overseer as cluster state updates are distributed as well)
+ * so Overseer stats and status can't be returned and actually do not even make sense. Zookeeper based queue metrics do not
+ * make sense either because Zookeeper queues are then not used.<p>
+ *
+ * The {@link Stats} instance returned by {@link CollectionCommandContext#getOverseerStats()} when running in the Overseer
+ * is created in Overseer.start() and passed to the cluster state updater from where it is also propagated to the various
+ * Zookeeper queues to register various events. This class is the only place where it is used in the Collection API implementation,
+ * and only to return results.
+ *
+ * TODO: create a new command returning node specific Collection API/Config set API/cluster state updates stats such as success and failures?<p>
+ *
* The structure of the returned results is as follows:
* <ul>
* <li><b>{@code leader}:</b> {@code ID} of the current overseer leader node</li>
@@ -59,8 +71,8 @@ import org.slf4j.LoggerFactory;
* (not all of them!) and {@link org.apache.solr.cloud.overseer.OverseerAction} (the complete list: {@code create},
* {@code delete}, {@code createshard}, {@code deleteshard}, {@code addreplica}, {@code addreplicaprop}, {@code deletereplicaprop},
* {@code balanceshardunique}, {@code modifycollection}, {@code state}, {@code leader}, {@code deletecore}, {@code addroutingrule},
- * {@code removeroutingrule}, {@code updateshardstate}, {@code downnode} and {@code quit} with this like one unlikely
- * to be observed since the Overseer is existing right away)</li>
+ * {@code removeroutingrule}, {@code updateshardstate}, {@code downnode} and {@code quit} with this last one unlikely
+ * to be observed since the Overseer is exiting right away)</li>
* <li>{@code update_state} (when Overseer cluster state updater persists changes in Zookeeper)</li>
* </ul>
* For each key, the value is a map composed of:
@@ -137,6 +149,13 @@ public class OverseerStatusCmd implements CollApiCmds.CollectionApiCommand {
@Override
@SuppressWarnings("unchecked")
public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
+ // If Collection API execution is distributed, we're not running on the Overseer node so can't return any Overseer stats.
+ if (ccc.getCoreContainer().getDistributedCollectionCommandRunner().isPresent()) {
+ // TODO: introduce a per node status command allowing insight into how Cluster state updates, Collection API and
+ // config set API execution went on that node...
+ return;
+ }
+
ZkStateReader zkStateReader = ccc.getZkStateReader();
String leaderNode = OverseerTaskProcessor.getLeaderNode(zkStateReader.getZkClient());
results.add("leader", leaderNode);
diff --git a/solr/core/src/java/org/apache/solr/core/CloudConfig.java b/solr/core/src/java/org/apache/solr/core/CloudConfig.java
index 440f1288fe7..fdfa9a4bd87 100644
--- a/solr/core/src/java/org/apache/solr/core/CloudConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/CloudConfig.java
@@ -50,10 +50,12 @@ public class CloudConfig {
private final boolean useDistributedClusterStateUpdates;
+ private final boolean useDistributedCollectionConfigSetExecution;
+
CloudConfig(String zkHost, int zkClientTimeout, int hostPort, String hostName, String hostContext, boolean useGenericCoreNames,
int leaderVoteWait, int leaderConflictResolveWait, String zkCredentialsProviderClass, String zkACLProviderClass,
int createCollectionWaitTimeTillActive, boolean createCollectionCheckLeaderActive, String pkiHandlerPrivateKeyPath,
- String pkiHandlerPublicKeyPath, boolean useDistributedClusterStateUpdates) {
+ String pkiHandlerPublicKeyPath, boolean useDistributedClusterStateUpdates, boolean useDistributedCollectionConfigSetExecution) {
this.zkHost = zkHost;
this.zkClientTimeout = zkClientTimeout;
this.hostPort = hostPort;
@@ -69,6 +71,11 @@ public class CloudConfig {
this.pkiHandlerPrivateKeyPath = pkiHandlerPrivateKeyPath;
this.pkiHandlerPublicKeyPath = pkiHandlerPublicKeyPath;
this.useDistributedClusterStateUpdates = useDistributedClusterStateUpdates;
+ this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;
+
+ if (useDistributedCollectionConfigSetExecution && !useDistributedClusterStateUpdates) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'useDistributedCollectionConfigSetExecution' can't be true if useDistributedClusterStateUpdates is false");
+ }
if (this.hostPort == -1)
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'hostPort' must be configured to run SolrCloud");
@@ -136,6 +143,10 @@ public class CloudConfig {
return useDistributedClusterStateUpdates;
}
+ public boolean getDistributedCollectionConfigSetExecution() {
+ return useDistributedCollectionConfigSetExecution;
+ }
+
public static class CloudConfigBuilder {
private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 45000;
@@ -159,6 +170,7 @@ public class CloudConfig {
private String pkiHandlerPrivateKeyPath;
private String pkiHandlerPublicKeyPath;
private boolean useDistributedClusterStateUpdates = false;
+ private boolean useDistributedCollectionConfigSetExecution = false;
public CloudConfigBuilder(String hostName, int hostPort) {
this(hostName, hostPort, null);
@@ -230,10 +242,16 @@ public class CloudConfig {
return this;
}
+ public CloudConfigBuilder setUseDistributedCollectionConfigSetExecution(boolean useDistributedCollectionConfigSetExecution) {
+ this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;
+ return this;
+ }
+
public CloudConfig build() {
return new CloudConfig(zkHost, zkClientTimeout, hostPort, hostName, hostContext, useGenericCoreNames, leaderVoteWait,
leaderConflictResolveWait, zkCredentialsProviderClass, zkACLProviderClass, createCollectionWaitTimeTillActive,
- createCollectionCheckLeaderActive, pkiHandlerPrivateKeyPath, pkiHandlerPublicKeyPath, useDistributedClusterStateUpdates);
+ createCollectionCheckLeaderActive, pkiHandlerPrivateKeyPath, pkiHandlerPublicKeyPath,
+ useDistributedClusterStateUpdates, useDistributedCollectionConfigSetExecution);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 88c71f5d9db..21ffc7497b9 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -72,6 +72,7 @@ import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ClusterSingleton;
import org.apache.solr.cloud.OverseerTaskQueue;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
import org.apache.solr.cluster.events.ClusterEventProducer;
import org.apache.solr.cluster.events.impl.ClusterEventProducerFactory;
import org.apache.solr.cluster.placement.PlacementPluginConfig;
@@ -276,6 +277,12 @@ public class CoreContainer {
private ExecutorService coreContainerAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Container Async Task");
+ /**
+ * Non empty if the Collection API is executed in a distributed way and not on Overseer, once the CoreContainer has been initialized
+ * properly, i.e. method {@link #load()} called. Until then it is null, and it is not expected to be read.
+ */
+ private volatile Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionCommandRunner;
+
private enum CoreInitFailedAction {fromleader, none}
/**
@@ -353,9 +360,10 @@ public class CoreContainer {
}
public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrCoreLoad) {
+ this.cfg = requireNonNull(config);
this.loader = config.getSolrResourceLoader();
this.solrHome = config.getSolrHome();
- this.cfg = requireNonNull(config);
+
try {
containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(cfg.getCloudConfig()));
} catch (IOException | InvalidKeySpecException e) {
@@ -594,6 +602,7 @@ public class CoreContainer {
cfg = null;
containerProperties = null;
replayUpdatesExecutor = null;
+ distributedCollectionCommandRunner = Optional.empty();
}
public static CoreContainer createAndLoad(Path solrHome) {
@@ -759,6 +768,15 @@ public class CoreContainer {
coreConfigService = ConfigSetService.createConfigSetService(this);
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);
+
+ // CoreContainer is initialized enough at this stage so we can set distributedCollectionCommandRunner (the
+ // construction of DistributedCollectionConfigSetCommandRunner uses Zookeeper so can't be done from the CoreContainer constructor
+ // because there Zookeeper is not yet ready). Given this is used in the CollectionsHandler created next line, this is
+ // the latest point where distributedCollectionCommandRunner can be initialized without refactoring this method...
+ // TODO: manage to completely build CoreContainer in the constructor and not in the load() method... Requires some test refactoring.
+ this.distributedCollectionCommandRunner = isZooKeeperAware() && cfg.getCloudConfig().getDistributedCollectionConfigSetExecution() ?
+ Optional.of(new DistributedCollectionConfigSetCommandRunner(this)) : Optional.empty();
+
collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
final CollectionsAPI collectionsAPI = new CollectionsAPI(collectionsHandler);
containerHandlers.getApiBag().registerObject(collectionsAPI);
@@ -938,7 +956,9 @@ public class CoreContainer {
});
clusterSingletons.setReady();
- zkSys.getZkController().checkOverseerDesignate();
+ if (!distributedCollectionCommandRunner.isPresent()) {
+ zkSys.getZkController().checkOverseerDesignate();
+ }
}
// This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.
@@ -1038,8 +1058,14 @@ public class CoreContainer {
ZkController zkController = getZkController();
if (zkController != null) {
- OverseerTaskQueue overseerCollectionQueue = zkController.getOverseerCollectionQueue();
- overseerCollectionQueue.allowOverseerPendingTasksToComplete();
+ if (distributedCollectionCommandRunner.isPresent()) {
+ // Local (i.e. distributed) Collection API processing
+ distributedCollectionCommandRunner.get().stopAndWaitForPendingTasksToComplete();
+ } else {
+ // Overseer based processing
+ OverseerTaskQueue overseerCollectionQueue = zkController.getOverseerCollectionQueue();
+ overseerCollectionQueue.allowOverseerPendingTasksToComplete();
+ }
}
if (log.isInfoEnabled()) {
log.info("Shutting down CoreContainer instance={}", System.identityHashCode(this));
@@ -2217,6 +2243,10 @@ public class CoreContainer {
return placementPluginFactory;
}
+ public Optional<DistributedCollectionConfigSetCommandRunner> getDistributedCollectionCommandRunner() {
+ return this.distributedCollectionCommandRunner;
+ }
+
static {
ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());
}
diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
index 7c941eeee74..d40c18afaa8 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
@@ -491,6 +491,9 @@ public class SolrXmlConfig {
case "distributedClusterStateUpdates":
builder.setUseDistributedClusterStateUpdates(Boolean.parseBoolean(value));
break;
+ case "distributedCollectionConfigSetExecution":
+ builder.setUseDistributedCollectionConfigSetExecution(Boolean.parseBoolean(value));
+ break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown configuration parameter in <solrcloud> section of solr.xml: " + name);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
index e8ef968287e..09fb2a3ae21 100644
--- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
@@ -26,7 +26,7 @@ import org.apache.solr.api.PayloadObj;
import org.apache.solr.client.solrj.request.beans.ClusterPropPayload;
import org.apache.solr.client.solrj.request.beans.CreateConfigPayload;
import org.apache.solr.client.solrj.request.beans.RateLimiterPayload;
-import org.apache.solr.cloud.OverseerConfigSetMessageHandler;
+import org.apache.solr.cloud.ConfigSetCmds;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.ClusterProperties;
@@ -131,7 +131,7 @@ public class ClusterAPI {
Map<String, Object> mapVals = obj.get().toMap(new HashMap<>());
Map<String,Object> customProps = (Map<String, Object>) mapVals.remove("properties");
if(customProps!= null) {
- customProps.forEach((k, o) -> mapVals.put(OverseerConfigSetMessageHandler.PROPERTY_PREFIX+"."+ k, o));
+ customProps.forEach((k, o) -> mapVals.put(ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX + k, o));
}
mapVals.put("action", ConfigSetParams.ConfigSetAction.CREATE.toString());
configSetsHandler.handleRequestBody(wrapParams(obj.getRequest(), mapVals), obj.getResponse());
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 1d689d0105d..5c5499278ba 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -28,7 +28,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.client.solrj.util.SolrIdentifierValidator;
-import org.apache.solr.cloud.DistributedClusterStateUpdater;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.cloud.OverseerSolrResponseSerializer;
import org.apache.solr.cloud.OverseerTaskQueue;
@@ -36,6 +35,7 @@ import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkController.NotInClusterStateException;
import org.apache.solr.cloud.ZkShardTerms;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
import org.apache.solr.cloud.api.collections.ReindexCollectionCmd;
import org.apache.solr.cloud.api.collections.RoutedAlias;
import org.apache.solr.cloud.overseer.SliceMutator;
@@ -63,6 +63,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
@@ -99,6 +100,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -137,7 +139,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
protected final CoreContainer coreContainer;
private final CollectionHandlerApi v2Handler;
- private final DistributedClusterStateUpdater distributedClusterStateUpdater;
+ private final Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionConfigSetCommandRunner;
public CollectionsHandler() {
// Unlike most request handlers, CoreContainer initialization
@@ -154,17 +156,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
public CollectionsHandler(final CoreContainer coreContainer) {
this.coreContainer = coreContainer;
v2Handler = new CollectionHandlerApi(this);
- // Get the state change factory to know if need to enqueue to Overseer or process distributed.
- // Some SolrCloud tests do not need Zookeeper and end up with a null cloudConfig in NodeConfig (because
- // TestHarness.buildTestNodeConfig() uses the zkHost to decide it's SolrCloud).
- // These tests do not use Zookeeper and do not do state updates (see subclasses of TestBaseStatsCacheCloud).
- // Some non SolrCloud tests do not even pass a config at all, so let be cautious here (code is not pretty).
- // We do want to initialize here and not do it lazy to not deal with synchronization for actual prod code.
- if (coreContainer == null || coreContainer.getConfig() == null || coreContainer.getConfig().getCloudConfig() == null) {
- distributedClusterStateUpdater = null;
- } else {
- distributedClusterStateUpdater = new DistributedClusterStateUpdater(coreContainer.getConfig().getCloudConfig().getDistributedClusterStateUpdates());
- }
+ distributedCollectionConfigSetCommandRunner = coreContainer != null ? coreContainer.getDistributedCollectionCommandRunner() : Optional.empty();
}
@Override
@@ -179,7 +171,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
@Override
- final public void init(@SuppressWarnings({"rawtypes"})NamedList args) {
+ final public void init(@SuppressWarnings({"rawtypes"}) NamedList args) {
}
@@ -259,7 +251,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
props.put(QUEUE_OPERATION, operation.action.toLower());
ZkNodeProps zkProps = new ZkNodeProps(props);
- SolrResponse overseerResponse = sendToOCPQueue(zkProps, operation.timeOut);
+ final SolrResponse overseerResponse;
+
+ overseerResponse = submitCollectionApiCommand(zkProps, operation.action, operation.timeOut);
+
rsp.getValues().addAll(overseerResponse.getResponse());
Exception exp = overseerResponse.getException();
if (exp != null) {
@@ -279,70 +274,75 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
public static long DEFAULT_COLLECTION_OP_TIMEOUT = 180 * 1000;
- public SolrResponse sendToOCPQueue(ZkNodeProps m) throws KeeperException, InterruptedException {
- return sendToOCPQueue(m, DEFAULT_COLLECTION_OP_TIMEOUT);
+ public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action) throws KeeperException, InterruptedException {
+ return submitCollectionApiCommand(m, action, DEFAULT_COLLECTION_OP_TIMEOUT);
}
- public SolrResponse sendToOCPQueue(ZkNodeProps m, long timeout) throws KeeperException, InterruptedException {
- String operation = m.getStr(QUEUE_OPERATION);
- if (operation == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION);
- }
- if (m.get(ASYNC) != null) {
-
- String asyncId = m.getStr(ASYNC);
-
- if (asyncId.equals("-1")) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "requestid can not be -1. It is reserved for cleanup purposes.");
+ public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action, long timeout) throws KeeperException, InterruptedException {
+ // Collection API messages are either sent to Overseer and processed there, or processed locally.
+ // Distributing Collection API implies we're also distributing Cluster State Updates. Indeed collection creation
+ // with non distributed cluster state updates requires for "Per Replica States" that the Collection API be running
+ // on Overseer, which means that it is not possible to distributed Collection API while keeping cluster state updates
+ // on Overseer. See the call to CollectionCommandContext.submitIntraProcessMessage() in CreateCollectionCmd.call() which
+ // can only be done if the Collection API command runs on the same JVM as the Overseer based cluster state update...
+ // The configuration handling includes these checks to not allow distributing collection API without distributing
+ // cluster state updates (but the other way around is ok). See constructor of CloudConfig.
+ if (distributedCollectionConfigSetCommandRunner.isPresent()) {
+ return distributedCollectionConfigSetCommandRunner.get().runCollectionCommand(m, action, timeout);
+ } else { // Sending the Collection API message to Overseer via a Zookeeper queue
+ String operation = m.getStr(QUEUE_OPERATION);
+ if (operation == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION);
}
-
- NamedList<String> r = new NamedList<>();
-
-
- if (coreContainer.getZkController().claimAsyncId(asyncId)) {
- boolean success = false;
- try {
- coreContainer.getZkController().getOverseerCollectionQueue()
- .offer(Utils.toJSON(m));
- success = true;
- } finally {
- if (!success) {
- try {
- coreContainer.getZkController().clearAsyncId(asyncId);
- } catch (Exception e) {
- // let the original exception bubble up
- log.error("Unable to release async ID={}", asyncId, e);
- SolrZkClient.checkInterrupted(e);
+ if (m.get(ASYNC) != null) {
+ String asyncId = m.getStr(ASYNC);
+ NamedList<String> r = new NamedList<>();
+
+ if (coreContainer.getZkController().claimAsyncId(asyncId)) {
+ boolean success = false;
+ try {
+ coreContainer.getZkController().getOverseerCollectionQueue()
+ .offer(Utils.toJSON(m));
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ coreContainer.getZkController().clearAsyncId(asyncId);
+ } catch (Exception e) {
+ // let the original exception bubble up
+ log.error("Unable to release async ID={}", asyncId, e);
+ SolrZkClient.checkInterrupted(e);
+ }
}
}
+ } else {
+ r.add("error", "Task with the same requestid already exists. (" + asyncId + ")");
}
- } else {
- r.add("error", "Task with the same requestid already exists.");
- }
- r.add(CoreAdminParams.REQUESTID, (String) m.get(ASYNC));
+ r.add(CoreAdminParams.REQUESTID, (String) m.get(ASYNC));
- return new OverseerSolrResponse(r);
- }
+ return new OverseerSolrResponse(r);
+ }
- long time = System.nanoTime();
- QueueEvent event = coreContainer.getZkController()
- .getOverseerCollectionQueue()
- .offer(Utils.toJSON(m), timeout);
- if (event.getBytes() != null) {
- return OverseerSolrResponseSerializer.deserialize(event.getBytes());
- } else {
- if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
- throw new SolrException(ErrorCode.SERVER_ERROR, operation
- + " the collection time out:" + timeout / 1000 + "s");
- } else if (event.getWatchedEvent() != null) {
- throw new SolrException(ErrorCode.SERVER_ERROR, operation
- + " the collection error [Watcher fired on path: "
- + event.getWatchedEvent().getPath() + " state: "
- + event.getWatchedEvent().getState() + " type "
- + event.getWatchedEvent().getType() + "]");
+ long time = System.nanoTime();
+ QueueEvent event = coreContainer.getZkController()
+ .getOverseerCollectionQueue()
+ .offer(Utils.toJSON(m), timeout);
+ if (event.getBytes() != null) {
+ return OverseerSolrResponseSerializer.deserialize(event.getBytes());
} else {
- throw new SolrException(ErrorCode.SERVER_ERROR, operation
- + " the collection unknown case");
+ if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the collection time out:" + timeout / 1000 + "s");
+ } else if (event.getWatchedEvent() != null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the collection error [Watcher fired on path: "
+ + event.getWatchedEvent().getPath() + " state: "
+ + event.getWatchedEvent().getState() + " type "
+ + event.getWatchedEvent().getType() + "]");
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the collection unknown case");
+ }
}
}
}
@@ -813,29 +813,53 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
REQUESTSTATUS_OP(REQUESTSTATUS, (req, rsp, h) -> {
req.getParams().required().check(REQUESTID);
- final CoreContainer coreContainer1 = h.coreContainer;
+ final CoreContainer coreContainer = h.coreContainer;
final String requestId = req.getParams().get(REQUESTID);
- final ZkController zkController = coreContainer1.getZkController();
-
- final NamedList<Object> results = new NamedList<>();
- if (zkController.getOverseerCompletedMap().contains(requestId)) {
- final byte[] mapEntry = zkController.getOverseerCompletedMap().get(requestId);
- rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());
- addStatusToResponse(results, COMPLETED, "found [" + requestId + "] in completed tasks");
- } else if (zkController.getOverseerFailureMap().contains(requestId)) {
- final byte[] mapEntry = zkController.getOverseerFailureMap().get(requestId);
- rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());
- addStatusToResponse(results, FAILED, "found [" + requestId + "] in failed tasks");
- } else if (zkController.getOverseerRunningMap().contains(requestId)) {
- addStatusToResponse(results, RUNNING, "found [" + requestId + "] in running tasks");
- } else if (h.overseerCollectionQueueContains(requestId)) {
- addStatusToResponse(results, SUBMITTED, "found [" + requestId + "] in submitted tasks");
+ final ZkController zkController = coreContainer.getZkController();
+
+ final NamedList<Object> status = new NamedList<>();
+ if (coreContainer.getDistributedCollectionCommandRunner().isEmpty()) {
+ if (zkController.getOverseerCompletedMap().contains(requestId)) {
+ final byte[] mapEntry = zkController.getOverseerCompletedMap().get(requestId);
+ rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());
+ addStatusToResponse(status, COMPLETED, "found [" + requestId + "] in completed tasks");
+ } else if (zkController.getOverseerFailureMap().contains(requestId)) {
+ final byte[] mapEntry = zkController.getOverseerFailureMap().get(requestId);
+ rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());
+ addStatusToResponse(status, FAILED, "found [" + requestId + "] in failed tasks");
+ } else if (zkController.getOverseerRunningMap().contains(requestId)) {
+ addStatusToResponse(status, RUNNING, "found [" + requestId + "] in running tasks");
+ } else if (h.overseerCollectionQueueContains(requestId)) {
+ addStatusToResponse(status, SUBMITTED, "found [" + requestId + "] in submitted tasks");
+ } else {
+ addStatusToResponse(status, NOT_FOUND, "Did not find [" + requestId + "] in any tasks queue");
+ }
} else {
- addStatusToResponse(results, NOT_FOUND, "Did not find [" + requestId + "] in any tasks queue");
+ Pair<RequestStatusState, OverseerSolrResponse> sr =
+ coreContainer.getDistributedCollectionCommandRunner().get().getAsyncTaskRequestStatus(requestId);
+ final String message;
+ switch (sr.first()) {
+ case COMPLETED:
+ message = "found [" + requestId + "] in completed tasks";
+ rsp.getValues().addAll(sr.second().getResponse());
+ break;
+ case FAILED:
+ message = "found [" + requestId + "] in failed tasks";
+ rsp.getValues().addAll(sr.second().getResponse());
+ break;
+ case RUNNING:
+ message = "found [" + requestId + "] in running tasks";
+ break;
+ case SUBMITTED:
+ message = "found [" + requestId + "] in submitted tasks";
+ break;
+ default:
+ message = "Did not find [" + requestId + "] in any tasks queue";
+ }
+ addStatusToResponse(status, sr.first(), message);
}
- final SolrResponse response = new OverseerSolrResponse(results);
- rsp.getValues().addAll(response.getResponse());
+ rsp.getValues().addAll(status);
return null;
}),
DELETESTATUS_OP(DELETESTATUS, new CollectionOp() {
@@ -856,30 +880,42 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
"Both requestid and flush parameters can not be specified together.");
}
- if (flush) {
- Collection<String> completed = zkController.getOverseerCompletedMap().keys();
- Collection<String> failed = zkController.getOverseerFailureMap().keys();
- for (String asyncId : completed) {
- zkController.getOverseerCompletedMap().remove(asyncId);
- zkController.clearAsyncId(asyncId);
- }
- for (String asyncId : failed) {
- zkController.getOverseerFailureMap().remove(asyncId);
- zkController.clearAsyncId(asyncId);
+ if (coreContainer.getDistributedCollectionCommandRunner().isEmpty()) {
+ if (flush) {
+ Collection<String> completed = zkController.getOverseerCompletedMap().keys();
+ Collection<String> failed = zkController.getOverseerFailureMap().keys();
+ for (String asyncId : completed) {
+ zkController.getOverseerCompletedMap().remove(asyncId);
+ zkController.clearAsyncId(asyncId);
+ }
+ for (String asyncId : failed) {
+ zkController.getOverseerFailureMap().remove(asyncId);
+ zkController.clearAsyncId(asyncId);
+ }
+ rsp.getValues().add("status", "successfully cleared stored collection api responses");
+ } else {
+ // Request to cleanup
+ if (zkController.getOverseerCompletedMap().remove(requestId)) {
+ zkController.clearAsyncId(requestId);
+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");
+ } else if (zkController.getOverseerFailureMap().remove(requestId)) {
+ zkController.clearAsyncId(requestId);
+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");
+ } else {
+ rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");
+ // Don't call zkController.clearAsyncId for this, since it could be a running/pending task
+ }
}
- rsp.getValues().add("status", "successfully cleared stored collection api responses");
- return null;
} else {
- // Request to cleanup
- if (zkController.getOverseerCompletedMap().remove(requestId)) {
- zkController.clearAsyncId(requestId);
- rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");
- } else if (zkController.getOverseerFailureMap().remove(requestId)) {
- zkController.clearAsyncId(requestId);
- rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");
+ if (flush) {
+ coreContainer.getDistributedCollectionCommandRunner().get().deleteAllAsyncIds();
+ rsp.getValues().add("status", "successfully cleared stored collection api responses");
} else {
- rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");
- // Don't call zkController.clearAsyncId for this, since it could be a running/pending task
+ if (coreContainer.getDistributedCollectionCommandRunner().get().deleteSingleAsyncId(requestId)) {
+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");
+ } else {
+ rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");
+ }
}
}
return null;
@@ -906,7 +942,14 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX);
}),
OVERSEERSTATUS_OP(OVERSEERSTATUS, (req, rsp, h) -> new LinkedHashMap<>()),
-
+ @SuppressWarnings({"unchecked"})
+ DISTRIBUTEDAPIPROCESSING_OP(DISTRIBUTEDAPIPROCESSING, (req, rsp, h) -> {
+ NamedList<Object> results = new NamedList<>();
+ boolean isDistributedApi = h.coreContainer.getDistributedCollectionCommandRunner().isPresent();
+ results.add("isDistributedApi", isDistributedApi);
+ rsp.getValues().addAll(results);
+ return null;
+ }),
/**
* Handle list collection request.
* Do list collection request to zk host
@@ -1313,7 +1356,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
"shard",
FOLLOW_ALIASES);
}),
- DELETENODE_OP(DELETENODE, (req, rsp, h) -> copy(req.getParams().required(), null, "node"));
+ DELETENODE_OP(DELETENODE, (req, rsp, h) -> copy(req.getParams().required(), null, "node")),
+ MOCK_COLL_TASK_OP(MOCK_COLL_TASK, (req, rsp, h) -> {
+ Map<String, Object> map = copy(req.getParams().required(), null, COLLECTION_PROP);
+ return copy(req.getParams(), map, "sleep");});
/**
* Places all prefixed properties in the sink map (or a new map) using the prefix as the key and a map of
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
index 39608a0d13c..75b040e519d 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
@@ -31,9 +32,11 @@ import java.util.zip.ZipInputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.cloud.ConfigSetCmds;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.cloud.OverseerSolrResponseSerializer;
import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -57,9 +60,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
-import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.BASE_CONFIGSET;
import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.CONFIGSETS_ACTION_PREFIX;
-import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.PROPERTY_PREFIX;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE;
@@ -75,7 +76,8 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
final public static String AUTOCREATED_CONFIGSET_SUFFIX = ".AUTOCREATED";
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected final CoreContainer coreContainer;
- public static long DEFAULT_ZK_TIMEOUT = 300 * 1000;
+ private final Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionConfigSetCommandRunner;
+ public static long CONFIG_SET_TIMEOUT = 300 * 1000;
/**
* Overloaded ctor to inject CoreContainer into the handler.
*
@@ -83,6 +85,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
*/
public ConfigSetsHandler(final CoreContainer coreContainer) {
this.coreContainer = coreContainer;
+ distributedCollectionConfigSetCommandRunner = coreContainer != null ? coreContainer.getDistributedCollectionCommandRunner() : Optional.empty();
}
public static String getSuffixedNameForAutoGeneratedConfigSet(String configName) {
@@ -136,18 +139,22 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
log.info("Invoked ConfigSet Action :{} with params {} ", action.toLower(), req.getParamString());
}
Map<String, Object> result = operation.call(req, rsp, this);
- sendToZk(rsp, operation, result);
+ if (result != null) {
+ if (distributedCollectionConfigSetCommandRunner.isPresent()) {
+ distributedCollectionConfigSetCommandRunner.get().runConfigSetCommand(rsp, operation, result, CONFIG_SET_TIMEOUT);
+ } else { // Sending the Collection API message to Overseer via a Zookeeper queue
+ sendToOverseer(rsp, operation, result);
+ }
+ }
}
- protected void sendToZk(SolrQueryResponse rsp, ConfigSetOperation operation, Map<String, Object> result)
+ protected void sendToOverseer(SolrQueryResponse rsp, ConfigSetOperation operation, Map<String, Object> result)
throws KeeperException, InterruptedException {
- if (result != null) {
- // We need to differentiate between collection and configsets actions since they currently
- // use the same underlying queue.
- result.put(QUEUE_OPERATION, CONFIGSETS_ACTION_PREFIX + operation.action.toLower());
- ZkNodeProps props = new ZkNodeProps(result);
- handleResponse(operation.action.toLower(), props, rsp, DEFAULT_ZK_TIMEOUT);
- }
+ // We need to differentiate between collection and configsets actions since they currently
+ // use the same underlying queue.
+ result.put(QUEUE_OPERATION, CONFIGSETS_ACTION_PREFIX + operation.action.toLower());
+ ZkNodeProps props = new ZkNodeProps(result);
+ handleResponse(operation.action.toLower(), props, rsp, CONFIG_SET_TIMEOUT);
}
private void handleConfigUploadRequest(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
@@ -366,7 +373,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
CREATE_OP(CREATE) {
@Override
public Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {
- String baseConfigSetName = req.getParams().get(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
+ String baseConfigSetName = req.getParams().get(ConfigSetCmds.BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
String newConfigSetName = req.getParams().get(NAME);
if (newConfigSetName == null || newConfigSetName.length() == 0) {
throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
@@ -383,13 +390,13 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
}
Map<String, Object> props = CollectionsHandler.copy(req.getParams().required(), null, NAME);
- props.put(BASE_CONFIGSET, baseConfigSetName);
+ props.put(ConfigSetCmds.BASE_CONFIGSET, baseConfigSetName);
if (!DISABLE_CREATE_AUTH_CHECKS &&
!isTrusted(req, h.coreContainer.getAuthenticationPlugin()) &&
isCurrentlyTrusted(h.coreContainer.getConfigSetService(), baseConfigSetName)) {
- throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + BASE_CONFIGSET);
+ throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + ConfigSetCmds.BASE_CONFIGSET);
}
- return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX + ".");
+ return copyPropertiesWithPrefix(req.getParams(), props, ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX);
}
},
DELETE_OP(DELETE) {
@@ -417,6 +424,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
this.action = action;
}
+ public ConfigSetAction getAction() {
+ return action;
+ }
public abstract Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception;
public static ConfigSetOperation get(ConfigSetAction action) {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java
index 239fbecbfa0..45cba5907e6 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java
@@ -37,6 +37,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrQueryRequest;
@@ -403,20 +404,21 @@ class RebalanceLeaders {
private void rejoinElectionQueue(Slice slice, String electionNode, String core, boolean rejoinAtHead)
throws KeeperException, InterruptedException {
Replica replica = slice.getReplica(LeaderElector.getNodeName(electionNode));
+ final CollectionParams.CollectionAction rebalanceleaders = REBALANCELEADERS;
Map<String, Object> propMap = new HashMap<>();
propMap.put(COLLECTION_PROP, collectionName);
propMap.put(SHARD_ID_PROP, slice.getName());
- propMap.put(QUEUE_OPERATION, REBALANCELEADERS.toLower());
+ propMap.put(QUEUE_OPERATION, rebalanceleaders.toLower());
propMap.put(CORE_NAME_PROP, core);
propMap.put(CORE_NODE_NAME_PROP, replica.getName());
propMap.put(ZkStateReader.NODE_NAME_PROP, replica.getNodeName());
propMap.put(REJOIN_AT_HEAD_PROP, Boolean.toString(rejoinAtHead)); // Get ourselves to be first in line.
propMap.put(ELECTION_NODE_PROP, electionNode);
- String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime());
+ String asyncId = rebalanceleaders.toLower() + "_" + core + "_" + Math.abs(System.nanoTime());
propMap.put(ASYNC, asyncId);
asyncRequests.add(asyncId);
- collectionsHandler.sendToOCPQueue(new ZkNodeProps(propMap)); // ignore response; we construct our own
+ collectionsHandler.submitCollectionApiCommand(new ZkNodeProps(propMap), rebalanceleaders); // ignore response; we construct our own
}
// maxWaitSecs - How long are we going to wait? Defaults to 30 seconds.
diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
index 0acb7e89a7a..09d5969f399 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
@@ -552,17 +552,17 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
// Now delete one of the collections, should fail since an alias points to it.
RequestStatusState delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);
// failed because the collection is a part of a compound alias
- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);
CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection("collection_alias_pair");
delResp = delete.processAndWait(cluster.getSolrClient(), 60);
// failed because we tried to delete an alias with followAliases=false
- assertEquals("Should have failed to delete alias: ", delResp, RequestStatusState.FAILED);
+ assertEquals("Should have failed to delete alias: ", RequestStatusState.FAILED, delResp);
delete.setFollowAliases(true);
delResp = delete.processAndWait(cluster.getSolrClient(), 60);
// failed because we tried to delete compound alias
- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);
CollectionAdminRequest.createAlias("collection_alias_one", "collection_one").process(cluster.getSolrClient());
lastVersion = waitForAliasesUpdate(lastVersion, zkStateReader);
@@ -570,13 +570,13 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
delete = CollectionAdminRequest.deleteCollection("collection_one");
delResp = delete.processAndWait(cluster.getSolrClient(), 60);
// failed because we tried to delete collection referenced by multiple aliases
- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);
delete = CollectionAdminRequest.deleteCollection("collection_alias_one");
delete.setFollowAliases(true);
delResp = delete.processAndWait(cluster.getSolrClient(), 60);
// failed because we tried to delete collection referenced by multiple aliases
- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);
+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);
CollectionAdminRequest.deleteAlias("collection_alias_one").process(cluster.getSolrClient());
lastVersion = waitForAliasesUpdate(lastVersion, zkStateReader);
@@ -588,7 +588,7 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
//Delete collection_one.
delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);
- assertEquals("Should not have failed to delete collection, it was removed from the alias: ", delResp, RequestStatusState.COMPLETED);
+ assertEquals("Should not have failed to delete collection, it was removed from the alias: ", RequestStatusState.COMPLETED, delResp);
// Should only see two docs now in second collection
res = cluster.getSolrClient().query("collection_alias_pair", new SolrQuery("*:*"));
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index e3a89164c50..dda0f82b8ff 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -657,7 +657,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("info", "core", "startTime"));
assertNotNull(Utils.toJSONString(rsp), segInfos.get("fieldInfoLegend"));
assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "fields", "id", "flags"));
- assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "ramBytesUsed"));
// test for replicas not active - SOLR-13882
DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(collectionName);
Replica firstReplica = coll.getSlice("shard1").getReplicas().iterator().next();
@@ -1031,6 +1030,10 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
public void testOverseerStatus() throws IOException, SolrServerException {
CollectionAdminResponse response = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
+ // When running with Distributed Collection API, no real data in Overseer status, but the Collection API call above shouldn't fail
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ return;
+ }
assertNotNull("overseer_operations shouldn't be null", response.getResponse().get("overseer_operations"));
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java
new file mode 100644
index 00000000000..ef953aa75dc
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.junit.Test;
+
+public class ConfigSetApiLockingTest extends SolrTestCaseJ4 {
+ private static final String CONFIG_SET_NAME_1 = "configSet1";
+ private static final String CONFIG_SET_NAME_2 = "configSet2";
+ private static final String CONFIG_SET_NAME_3 = "configSet3";
+
+ private static final String BASE_CONFIG_SET_NAME = "baseConfigSet";
+
+ static final int TIMEOUT = 10000;
+
+ /**
+ * Tests the Config Set API locking. All tests run from single test method to save on setup time.
+ */
+ @Test
+ public void monothreadedApiLockTests() throws Exception {
+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));
+ try {
+ server.run();
+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {
+ ConfigSetApiLockFactory apiLockFactory = new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(zkClient, "/apiLockTestRoot"));
+
+ monothreadedTests(apiLockFactory);
+ multithreadedTests(apiLockFactory);
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception {
+ // Config set lock without a base config set
+ DistributedMultiLock cs1Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_1, null);
+ assertTrue("cs1Lock should have been acquired", cs1Lock.isAcquired());
+
+ // This lock does have a base config set
+ DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);
+ assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired());
+
+ // This lock does has the same base config set, but that shouldn't prevent acquiring it
+ DistributedMultiLock cs3Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_3, BASE_CONFIG_SET_NAME);
+ assertTrue("cs3Lock should have been acquired", cs3Lock.isAcquired());
+
+ // But we shouldn't be able to lock at this stage the base config set
+ DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null);
+ assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired());
+
+ cs2Lock.release();
+ assertFalse("csBaseLock should not have been acquired, cs3Lock still there", csBaseLock.isAcquired());
+ cs3Lock.release();
+ assertTrue("csBaseLock should have been acquired", csBaseLock.isAcquired());
+
+ // Acquiring a lock with a locked base config set should not be possible
+ cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);
+ assertFalse("cs2Lock should not have been acquired, csBaseLock still held", cs2Lock.isAcquired());
+
+ csBaseLock.release();
+ assertTrue("cs2Lock should now be acquired, csBaseLock was freed", cs2Lock.isAcquired());
+
+ cs2Lock.release();
+ }
+
+ private void multithreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception {
+ // This lock does have a base config set
+ DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);
+ assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired());
+
+ // But we shouldn't be able to lock at this stage the base config set
+ DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null);
+ assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired());
+
+ // Wait for acquisition of the base config set lock on another thread (and be notified via a latch)
+ final CountDownLatch latch = new CountDownLatch(1);
+ new Thread(() -> {
+ csBaseLock.waitUntilAcquired();
+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind
+ latch.countDown();
+ }).start();
+
+ // Wait for the thread to start and to get blocked in waitUntilAcquired()
+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired
+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,
+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).
+ Thread.sleep(30);
+
+ assertEquals("we should not have been notified that base config set lock was acquired", 1, latch.getCount());
+ assertFalse("base config set lock should not have been acquired", csBaseLock.isAcquired());
+
+ cs2Lock.release();
+ assertTrue("basec config set lock should have been acquired now that other lock was released", csBaseLock.isAcquired());
+
+ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch
+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause
+ int i = 0;
+ while (i < 1000 && latch.getCount() != 0) {
+ Thread.sleep(10);
+ i++;
+ }
+ assertEquals("we should have been notified that the base config set lock was acquired", 0, latch.getCount());
+ }
+}
+
diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java
index 53707136cb7..bcdc867a0c4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java
@@ -67,7 +67,7 @@ public class CreateCollectionCleanupTest extends SolrCloudTestCase {
configureCluster(1)
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.withSolrXml(CLOUD_SOLR_XML_WITH_10S_CREATE_COLL_WAIT)
- .useOtherClusterStateUpdateStrategy()
+ .useOtherCollectionConfigSetExecution()
.configure();
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index de1d19aad23..31c1deb9abc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -77,7 +77,7 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
// these tests need to be isolated, so we dont share the minicluster
configureCluster(4)
.addConfig("conf", configset("cloud-minimal"))
- .useOtherClusterStateUpdateStrategy() // Some tests (this one) use "the other" cluster state update strategy to increase coverage
+ .useOtherCollectionConfigSetExecution() // Some tests (this one) use "the other" cluster Collection API execution strategy to increase coverage
.configure();
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.java
new file mode 100644
index 00000000000..994e1984bfd
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.util.NamedList;
+import org.junit.Test;
+
+/**
+ * Test async id tracking scenarios as used when Collection API is distributed.
+ */
+public class DistributedApiAsyncTrackerTest extends SolrTestCaseJ4 {
+
+ protected ZkTestServer zkServer;
+ protected SolrZkClient zkClient;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ zkServer = new ZkTestServer(createTempDir("zookeeperDir"));
+ zkServer.run();
+ zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+
+ zkServer.shutdown();
+ zkClient.close();
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void testBasic() throws Exception {
+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, "/basic");
+
+ final String asyncId = "mario";
+
+ assertTrue("Could not create async task " + asyncId ,daat.createNewAsyncJobTracker(asyncId));
+ assertFalse("Should not have been able to create duplicate task " + asyncId ,daat.createNewAsyncJobTracker(asyncId));
+
+ daat.cancelAsyncId(asyncId);
+ assertEquals(RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());
+ assertTrue("Could not create async task after cancel " + asyncId ,daat.createNewAsyncJobTracker(asyncId));
+
+ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(asyncId).first());
+ assertFalse("Can't delete a non completed/failed task " + asyncId, daat.deleteSingleAsyncId(asyncId));
+ daat.deleteAllAsyncIds();
+ assertEquals("Task should still be here because couldn't be deleted " + asyncId, RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(asyncId).first());
+
+ daat.setTaskRunning(asyncId);
+ assertEquals(RequestStatusState.RUNNING, daat.getAsyncTaskRequestStatus(asyncId).first());
+
+ NamedList nl = new NamedList();
+ nl.add("MyList", "myValue");
+ daat.setTaskCompleted(asyncId, new OverseerSolrResponse(nl));
+ assertEquals(RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(asyncId).first());
+ assertEquals("Did not retrieve correct completed OverseerSolrResponse " + asyncId, "myValue", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("MyList"));
+
+ assertTrue("Should be able to delete a completed task " + asyncId, daat.deleteSingleAsyncId(asyncId));
+ assertEquals("A completed task should not be found", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());
+ }
+
+ @Test
+ public void testDisconnect() throws Exception {
+ final String TRACKER_ROOT = "/disconnect";
+ DistributedApiAsyncTracker permanentDaat = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);
+ DistributedApiAsyncTracker permanentDaat2 = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);
+
+ final String asycPermanent = "permanentAsync";
+ final String asyncTransient = "transientAsync";
+
+ assertTrue("Could not create async task " + asycPermanent ,permanentDaat.createNewAsyncJobTracker(asycPermanent));
+
+ try (SolrZkClient transientZkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) {
+ DistributedApiAsyncTracker transientDaat = new DistributedApiAsyncTracker(transientZkClient, TRACKER_ROOT);
+ assertTrue("Could not create async task " + asyncTransient ,transientDaat.createNewAsyncJobTracker(asyncTransient));
+
+ assertEquals("permanentDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asycPermanent).first());
+ assertEquals("permanentDaat2 can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asycPermanent).first());
+ assertEquals("transientDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, transientDaat.getAsyncTaskRequestStatus(asycPermanent).first());
+ assertEquals("permanentDaat can't see " + asyncTransient, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asyncTransient).first());
+ assertEquals("permanentDaat2 can't see " + asyncTransient, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asyncTransient).first());
+ assertEquals("transientDaat can't see " + asyncTransient, RequestStatusState.SUBMITTED, transientDaat.getAsyncTaskRequestStatus(asyncTransient).first());
+ }
+
+ // transientDaat connection closed, doesn't change a thing for asycPermanent...
+ assertEquals("permanentDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asycPermanent).first());
+ assertEquals("permanentDaat2 can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asycPermanent).first());
+
+ // ...but asyncTransient is now failed.
+ assertEquals("permanentDaat can't see " + asyncTransient, RequestStatusState.FAILED, permanentDaat.getAsyncTaskRequestStatus(asyncTransient).first());
+ assertEquals("permanentDaat2 can't see " + asyncTransient, RequestStatusState.FAILED, permanentDaat2.getAsyncTaskRequestStatus(asyncTransient).first());
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void testDisconnectAfterCompletion() throws Exception {
+ final String TRACKER_ROOT = "/multiConn";
+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);
+
+ final String asyncId = "theId";
+
+ try (SolrZkClient transientZkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) {
+ DistributedApiAsyncTracker transientDaat = new DistributedApiAsyncTracker(transientZkClient, TRACKER_ROOT);
+ assertTrue("Could not create async task " + asyncId ,transientDaat.createNewAsyncJobTracker(asyncId));
+
+ // The task completes, then connection lost
+ NamedList nl = new NamedList();
+ nl.add("status", "I made it");
+ transientDaat.setTaskCompleted(asyncId, new OverseerSolrResponse(nl));
+
+ assertEquals("transientDaat can't see " + asyncId, RequestStatusState.COMPLETED, transientDaat.getAsyncTaskRequestStatus(asyncId).first());
+ assertEquals("transientDaat can't retrieve correct completed OverseerSolrResponse " + asyncId, "I made it", transientDaat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));
+ assertEquals("daat can't retrieve correct completed OverseerSolrResponse " + asyncId, "I made it", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));
+ }
+
+ // Even though connection was closed, the task should still be completed, and we should be able to retrieve its response
+ assertEquals("daat can't see " + asyncId, RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(asyncId).first());
+ assertEquals("Did not retrieve correct completed OverseerSolrResponse after other connection closes " + asyncId, "I made it", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));
+
+ // And given it completed, it should be deleted
+ daat.deleteAllAsyncIds();
+ assertEquals("task was not deleted " + asyncId, RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());
+ }
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void testIdCleanup() throws Exception {
+ final int maxTasks = 30; // When cleaning up, 3 async id's will be removed
+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, "/manyIds", maxTasks);
+
+ for (int asyncId = 1; asyncId <= maxTasks; asyncId++) {
+ assertTrue(daat.createNewAsyncJobTracker(Integer.toString(asyncId)));
+ }
+
+ // All ids should be tracked
+ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(1)).first());
+ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(maxTasks)).first());
+
+ // Adding one more should trigger cleanup of earlier id's
+ assertTrue(daat.createNewAsyncJobTracker("straw"));
+
+ String cleanedUpId1 = Integer.toString(1);
+
+ assertEquals("Cleanup should have been triggered and removed the first task", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());
+ assertEquals("Last task should not have been cleaned up", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(maxTasks)).first());
+ assertEquals("Task having triggered cleanup should not have been cleaned up", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus("straw").first());
+
+ // Identical to the test 3 lines above but repeated to be considered in context
+ assertEquals("Cleaned up ID (1) should no longer be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());
+ // Creating a new task with same id fails but "revives" the ID if the task is still in progress
+ assertFalse("Should not be able to create a task with same id as task in progress", daat.createNewAsyncJobTracker(cleanedUpId1));
+ assertEquals("Cleaned up ID now visible again", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());
+
+ // 2 is also in progress and was also cleaned up. It should be markable complete without issues (since that's what will happen when it completed).
+ String cleanedUpId2 = Integer.toString(2);
+ assertEquals("Another cleaned up ID (2) should not be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId2).first());
+ NamedList nl = new NamedList();
+ nl.add("code", "da vinci");
+ daat.setTaskCompleted(cleanedUpId2, new OverseerSolrResponse(nl));
+
+ assertEquals("task should now be completed " + cleanedUpId2, RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(cleanedUpId2).first());
+ assertEquals("task should have correct OverseerSolrResponse " + cleanedUpId2, "da vinci", daat.getAsyncTaskRequestStatus(cleanedUpId2).second().getResponse().get("code"));
+
+ // 3 is also in progress and was also cleaned up. It should be markable running without issues, but that doesn't revive it (still not found).
+ // (if we want setTaskRunning to revive a removed task, we'd have to check the persistent node existence each time, which is likely not worth it)
+ String cleanedUpId3 = Integer.toString(3);
+ assertEquals("Another cleaned up ID (3) should not be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId3).first());
+ daat.setTaskRunning(cleanedUpId3);
+ assertEquals("task should now be running " + cleanedUpId3, RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId3).first());
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
index 53c47dfd546..ff96d6cbf3c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
@@ -53,6 +53,7 @@ public class MoveReplicaTest extends SolrCloudTestCase {
// used by MoveReplicaHDFSTest
protected boolean inPlaceMove = true;
+ protected boolean isCollectionApiDistributed = false;
protected String getConfigSet() {
return "cloud-dynamic";
@@ -68,6 +69,11 @@ public class MoveReplicaTest extends SolrCloudTestCase {
.withSolrXml(TEST_PATH().resolve("solr.xml"))
.configure();
+ // If Collection API is distributed let's not wait for Overseer.
+ if (isCollectionApiDistributed = new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ return;
+ }
+
NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
JettySolrRunner overseerJetty = null;
String overseerLeader = (String) overSeerStatus.get("leader");
@@ -267,7 +273,7 @@ public class MoveReplicaTest extends SolrCloudTestCase {
Collections.shuffle(l, random());
String targetNode = null;
for (String node : liveNodes) {
- if (!replica.getNodeName().equals(node) && !overseerLeader.equals(node)) {
+ if (!replica.getNodeName().equals(node) && (isCollectionApiDistributed || !overseerLeader.equals(node))) {
targetNode = node;
break;
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
index c8357afad01..270010ee7ec 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
@@ -23,7 +23,6 @@ import java.util.Random;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.cloud.DistributedQueue;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create;
@@ -32,15 +31,11 @@ import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.Utils;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
import static org.apache.solr.cloud.OverseerTaskProcessor.MAX_PARALLEL_TASKS;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOCK_COLL_TASK;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
/**
* Tests the Multi threaded Collections API.
@@ -69,49 +64,47 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
private void testFillWorkQueue() throws Exception {
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
- DistributedQueue distributedQueue = new ZkDistributedQueue(cloudClient.getZkStateReader().getZkClient(),
- "/overseer/collection-queue-work", new Stats());
- //fill the work queue with blocked tasks by adding more than the no:of parallel tasks
- for (int i = 0; i < MAX_PARALLEL_TASKS + 15; i++) {
- distributedQueue.offer(Utils.toJSON(Utils.makeMap(
- "collection", "A_COLL",
- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),
- ASYNC, Integer.toString(i),
-
- // third task waits for a long time, and thus blocks the queue for all other tasks for A_COLL.
- // Subsequent tasks as well as the first two only wait for 1ms
- "sleep", (i == 2 ? "10000" : "1")
- )));
- log.info("MOCK task added {}", i);
- }
+ // This test does not make sense when the Collection API execution is distributed. There is no queue to fill
+ if (!new CollectionAdminRequest.RequestApiDistributedProcessing().process(client).getIsCollectionApiDistributed()) {
+
+ //fill the work queue with blocked tasks by adding more than the no:of parallel tasks
+ for (int i = 0; i < MAX_PARALLEL_TASKS + 15; i++) {
+
+ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("A_COLL");
+ // third task waits for a long time, and thus blocks the queue for all other tasks for A_COLL.
+ // Subsequent tasks as well as the first two only wait for 1ms
+ mockTask.setSleep(i == 2 ? "5000" : "1");
+ mockTask.processAsync(Integer.toString(i), client);
+
+ log.info("MOCK task added {}", i);
+ }
- // Wait until we see the second A_COLL task getting processed (assuming the first got processed as well)
- Long task1CollA = waitForTaskToCompleted(client, 1);
+ // Wait until we see the second A_COLL task getting processed (assuming the first got processed as well)
+ Long task1CollA = waitForTaskToCompleted(client, 1);
- assertNotNull("Queue did not process first two tasks on A_COLL, can't run test", task1CollA);
+ assertNotNull("Queue did not process first two tasks on A_COLL, can't run test", task1CollA);
- // Make sure the long running task did not finish, otherwise no way the B_COLL task can be tested to run in parallel with it
- assertNull("Long running task finished too early, can't test", checkTaskHasCompleted(client, 2));
+ // Make sure the long running task did not finish, otherwise no way the B_COLL task can be tested to run in parallel with it
+ assertNull("Long running task finished too early, can't test", checkTaskHasCompleted(client, 2));
- // Enqueue a task on another collection not competing with the lock on A_COLL and see that it can be executed right away
- distributedQueue.offer(Utils.toJSON(Utils.makeMap(
- "collection", "B_COLL",
- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),
- ASYNC, "200",
- "sleep", "1"
- )));
+ // Enqueue a task on another collection not competing with the lock on A_COLL and see that it can be executed right away
- // We now check that either the B_COLL task has completed before the third (long running) task on A_COLL,
- // Or if both have completed (if this check got significantly delayed for some reason), we verify B_COLL was first.
- Long taskCollB = waitForTaskToCompleted(client, 200);
+ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("B_COLL");
+ mockTask.setSleep("1");
+ mockTask.processAsync("200", client);
- // We do not wait for the long running task to finish, that would be a waste of time.
- Long task2CollA = checkTaskHasCompleted(client, 2);
+ // We now check that either the B_COLL task has completed before the third (long running) task on A_COLL,
+ // Or if both have completed (if this check got significantly delayed for some reason), we verify B_COLL was first.
+ Long taskCollB = waitForTaskToCompleted(client, 200);
- // Given the wait delay (500 iterations of 100ms), the task has plenty of time to complete, so this is not expected.
- assertNotNull("Task on B_COLL did not complete, can't test", taskCollB);
- // We didn't wait for the 3rd A_COLL task to complete (test can run quickly) but if it did, we expect the B_COLL to have finished first.
- assertTrue("task2CollA: " + task2CollA + " taskCollB: " + taskCollB, task2CollA == null || task2CollA > taskCollB);
+ // We do not wait for the long running task to finish, that would be a waste of time.
+ Long task2CollA = checkTaskHasCompleted(client, 2);
+
+ // Given the wait delay (500 iterations of 100ms), the task has plenty of time to complete, so this is not expected.
+ assertNotNull("Task on B_COLL did not complete, can't test", taskCollB);
+ // We didn't wait for the 3rd A_COLL task to complete (test can run quickly) but if it did, we expect the B_COLL to have finished first.
+ assertTrue("task2CollA: " + task2CollA + " taskCollB: " + taskCollB, task2CollA == null || task2CollA > taskCollB);
+ }
}
}
@@ -141,8 +134,9 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
}
private void testParallelCollectionAPICalls() throws IOException, SolrServerException {
+ final int ASYNC_SHIFT = 10000;
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
- for(int i = 1 ; i <= NUM_COLLECTIONS ; i++) {
+ for(int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++) {
CollectionAdminRequest.createCollection("ocptest" + i,"conf1",3,1).processAsync(String.valueOf(i), client);
}
@@ -150,7 +144,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
int counter = 0;
while(true) {
int numRunningTasks = 0;
- for (int i = 1; i <= NUM_COLLECTIONS; i++)
+ for (int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++)
if (getRequestState(i + "", client) == RequestStatusState.RUNNING) {
numRunningTasks++;
}
@@ -167,7 +161,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
}
}
assertTrue("More than one tasks were supposed to be running in parallel but they weren't.", pass);
- for (int i = 1; i <= NUM_COLLECTIONS; i++) {
+ for (int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++) {
final RequestStatusState state = getRequestStateAfterCompletion(i + "", REQUEST_STATUS_TIMEOUT, client);
assertSame("Task " + i + " did not complete, final state: " + state, RequestStatusState.COMPLETED, state);
}
@@ -175,26 +169,18 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
}
private void testTaskExclusivity() throws Exception, SolrServerException {
-
- DistributedQueue distributedQueue = new ZkDistributedQueue(cloudClient.getZkStateReader().getZkClient(),
- "/overseer/collection-queue-work", new Stats());
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
Create createCollectionRequest = CollectionAdminRequest.createCollection("ocptest_shardsplit","conf1",4,1);
createCollectionRequest.processAsync("1000",client);
- distributedQueue.offer(Utils.toJSON(Utils.makeMap(
- "collection", "ocptest_shardsplit",
- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),
- ASYNC, "1001",
- "sleep", "100"
- )));
- distributedQueue.offer(Utils.toJSON(Utils.makeMap(
- "collection", "ocptest_shardsplit",
- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),
- ASYNC, "1002",
- "sleep", "100"
- )));
+ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("ocptest_shardsplit");
+ mockTask.setSleep("100");
+ mockTask.processAsync("1001", client);
+
+ mockTask = CollectionAdminRequest.mockCollTask("ocptest_shardsplit");
+ mockTask.setSleep("100");
+ mockTask.processAsync("1002", client);
int iterations = 0;
while(true) {
@@ -256,7 +242,6 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
}
private void testLongAndShortRunningParallelApiCalls() throws InterruptedException, IOException, SolrServerException {
-
Thread indexThread = new Thread() {
@Override
public void run() {
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 05974a3afa0..c437a30e3a6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -102,6 +102,10 @@ public class OverseerRolesTest extends SolrCloudTestCase {
@Test
public void testOverseerRole() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ return;
+ }
logOverseerState();
List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
@@ -169,6 +173,10 @@ public class OverseerRolesTest extends SolrCloudTestCase {
@Test
public void testDesignatedOverseerRestarts() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ return;
+ }
logOverseerState();
// Remove the OVERSEER role, in case it was already assigned by another test in this suite
for (String node: OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient())) {
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
index d2d96047658..bf636e78f40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
@@ -44,6 +44,10 @@ public class OverseerStatusTest extends SolrCloudTestCase {
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1).process(cluster.getSolrClient());
NamedList<Object> resp = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient()).getResponse();
+ // When running with Distributed Collection API, no real data in Overseer status, but the Collection API call shouldn't fail
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ return;
+ }
NamedList<Object> collection_operations = (NamedList<Object>) resp.get("collection_operations");
NamedList<Object> overseer_operations = (NamedList<Object>) resp.get("overseer_operations");
SimpleOrderedMap<Object> createcollection
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
index c95b72f38e8..7c5c86476f9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
@@ -751,7 +751,8 @@ public class OverseerTest extends SolrTestCaseJ4 {
httpShardHandlerFactory.init(new PluginInfo("shardHandlerFactory", Collections.emptyMap()));
httpShardHandlerFactorys.add(httpShardHandlerFactory);
Overseer overseer = new Overseer((HttpShardHandler) httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, zkController,
- new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").setUseDistributedClusterStateUpdates(false).build());
+ new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").setUseDistributedClusterStateUpdates(false).
+ setUseDistributedCollectionConfigSetExecution(false).build());
overseers.add(overseer);
ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
server.getZkAddress().replaceAll("/", "_"));
diff --git a/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java b/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
index 8941e0dee7e..de939a7ed83 100644
--- a/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
@@ -48,6 +48,11 @@ public class RollingRestartTest extends AbstractFullDistribZkTestBase {
@Test
//commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
public void test() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cloudClient).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ return;
+ }
+
waitForRecoveriesToFinish(false);
restartWithRolesTest();
diff --git a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
index 197f0352f2d..db96a7672b6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
@@ -162,6 +162,13 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
@Test
public void testBackCompat() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ // TODO once we completely remove Overseer, do we need to move the back compat check to some other place, for example
+ // to when the .system collection is opened?
+ return;
+ }
+
CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();
CloudSolrClient solrClient = cluster.getSolrClient();
CollectionAdminResponse adminResponse = status.process(solrClient);
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java b/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java
index ada8bd358a8..3d893b4cc2f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java
@@ -18,6 +18,7 @@
package org.apache.solr.cloud;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedSet;
@@ -33,9 +34,13 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestSkipOverseerOperations extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
@Before
public void setupCluster() throws Exception {
System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
@@ -53,6 +58,10 @@ public class TestSkipOverseerOperations extends SolrCloudTestCase {
}
public void testSkipLeaderOperations() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ return;
+ }
String overseerLeader = getOverseerLeader();
@@ -136,6 +145,11 @@ public class TestSkipOverseerOperations extends SolrCloudTestCase {
@Test
// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018
public void testSkipDownOperations() throws Exception {
+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {
+ log.info("Skipping test because Collection API is distributed");
+ return;
+ }
+
String overseerLeader = getOverseerLeader();
List<JettySolrRunner> notOverseerNodes = cluster.getJettySolrRunners()
.stream()
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java
new file mode 100644
index 00000000000..1b88c15384e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.CollectionParams;
+import org.junit.Test;
+
+public class ZkDistributedLockTest extends SolrTestCaseJ4 {
+
+ private static final String COLLECTION_NAME = "lockColl";
+ private final String SHARD_NAME = "lockShard";
+ private final String REPLICA_NAME = "lockReplica";
+
+ private static final String CONFIG_SET_NAME = "lockConfigSet";
+
+ static final int TIMEOUT = 10000;
+
+ /**
+ * Tests the obtention of a single read or write lock at a specific hierarchical level, for distributed Collection API
+ * locking as well as distributed Config Set API locking.<br>
+ * Tests the logic with a single thread, then tests multithreaded wait for lock acquire works.
+ * Tests grouped to pay setup only once.<p>
+ *
+ * Higher level locking tests can be found at {@link org.apache.solr.cloud.api.collections.CollectionApiLockingTest} and
+ * {@link org.apache.solr.cloud.ConfigSetApiLockingTest}
+ */
+ @Test
+ public void testSingleLocks() throws Exception {
+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));
+ try {
+ server.run();
+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {
+ DistributedCollectionLockFactory collLockFactory = new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot");
+
+ monothreadedCollectionTests(collLockFactory);
+ multithreadedCollectionTests(collLockFactory);
+
+ DistributedConfigSetLockFactory configSetLockFactory = new ZkDistributedConfigSetLockFactory(zkClient, "/lockTestConfigSetRoot");
+
+ monothreadedConfigSetTests(configSetLockFactory);
+ multithreadedConfigSetTests(configSetLockFactory);
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ private void monothreadedCollectionTests(DistributedCollectionLockFactory factory) {
+ // Collection level locks
+ DistributedLock collRL1 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertTrue("collRL1 should have been acquired", collRL1.isAcquired());
+
+ DistributedLock collRL2 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertTrue("collRL1 should have been acquired", collRL2.isAcquired());
+
+ DistributedLock collWL3 = factory.createLock(true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertFalse("collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired());
+
+ assertTrue("collRL2 should have been acquired, that should not have changed", collRL2.isAcquired());
+
+ collRL1.release();
+ collRL2.release();
+ assertTrue("collWL3 should have been acquired, collRL1 and collRL2 were released", collWL3.isAcquired());
+
+ DistributedLock collRL4 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertFalse("collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired());
+
+ // Collection is write locked by collWL3 and collRL4 read lock waiting behind. Now moving to request shard level locks.
+ // These are totally independent from the Collection level locks so should see no impact.
+ DistributedLock shardWL5 = factory.createLock(true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null);
+ assertTrue("shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired());
+
+ DistributedLock shardWL6 = factory.createLock(true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null);
+ assertFalse("shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired());
+
+ // Get a lock on a Replica. Again this is independent of collection or shard level
+ DistributedLock replicaRL7 = factory.createLock(false, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME);
+ assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired());
+
+ DistributedLock replicaWL8 = factory.createLock(true, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME);
+ assertFalse("replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired());
+
+ replicaRL7.release();
+ assertTrue("replicaWL8 should have been acquired, as replicaRL7 got released", replicaWL8.isAcquired());
+
+
+ collWL3.release();
+ assertTrue("collRL4 should have been acquired given collWL3 released", collRL4.isAcquired());
+ shardWL5.release();
+ assertTrue("shardWL6 should have been acquired, now that shardWL5 was released", shardWL6.isAcquired());
+
+ replicaWL8.release();
+ try {
+ replicaWL8.isAcquired();
+ fail("isAcquired() called after release() on a lock should have thrown exception");
+ } catch (IllegalStateException ise) {
+ // expected
+ }
+
+ // Releasing the collection lock used in the multithreaded phase
+ collRL4.release();
+ }
+
+ private void multithreadedCollectionTests(DistributedCollectionLockFactory factory) throws Exception {
+ // Acquiring right away a read lock
+ DistributedLock readLock = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertTrue("readLock should have been acquired", readLock.isAcquired());
+
+ // And now creating a write lock, that can't be acquired just yet, because of the read lock
+ DistributedLock writeLock = factory.createLock(true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertFalse("writeLock should not have been acquired", writeLock.isAcquired());
+
+ // Wait for acquisition of the write lock on another thread (and be notified via a latch)
+ final CountDownLatch latch = new CountDownLatch(1);
+ new Thread(() -> {
+ writeLock.waitUntilAcquired();
+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind
+ latch.countDown();
+ }).start();
+
+ // Wait for the thread to start and to get blocked in waitUntilAcquired()
+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired
+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,
+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).
+ Thread.sleep(30);
+
+ assertEquals("we should not have been notified that writeLock was acquired", 1, latch.getCount());
+ assertFalse("writeLock should not have been acquired", writeLock.isAcquired());
+
+ readLock.release();
+ assertTrue("writeLock should have been acquired now that readlock was released", writeLock.isAcquired());
+
+ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch
+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause
+ int i = 0;
+ while (i < 1000 && latch.getCount() != 0) {
+ Thread.sleep(10);
+ i++;
+ }
+ assertEquals("we should have been notified that writeLock was acquired", 0, latch.getCount());
+ }
+
+ private void monothreadedConfigSetTests(DistributedConfigSetLockFactory factory) {
+ DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME);
+ assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired());
+
+ DistributedLock configSetRL2 = factory.createLock(false, CONFIG_SET_NAME);
+ assertTrue("configSetRL2 should have been acquired", configSetRL2.isAcquired());
+
+ DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME);
+ assertFalse("configSetWL1 should not have been acquired due to configSetRL1 and configSetRL2", configSetWL1.isAcquired());
+
+ configSetRL1.release();
+ configSetRL2.release();
+ assertTrue("configSetWL1 should have been acquired, configSetRL1 and configSetRL2 were released", configSetWL1.isAcquired());
+
+ DistributedLock configSetRL3 = factory.createLock(false, CONFIG_SET_NAME);
+ assertFalse("configSetRL3 should not have been acquired due to configSetWL1", configSetRL3.isAcquired());
+
+ configSetWL1.release();
+ assertTrue("configSetRL3 should have been acquired, configSetWL1 was released", configSetRL3.isAcquired());
+
+ configSetRL3.release();
+
+ try {
+ configSetRL3.isAcquired();
+ fail("isAcquired() called after release() on a lock should have thrown exception");
+ } catch (IllegalStateException ise) {
+ // expected
+ }
+ }
+
+ private void multithreadedConfigSetTests(DistributedConfigSetLockFactory factory) throws Exception {
+ // Acquiring right away a read lock
+ DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME);
+ assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired());
+
+ // And now creating a write lock, that can't be acquired just yet, because of the read lock
+ DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME);
+ assertFalse("configSetWL1 should not have been acquired due to configSetRL1", configSetWL1.isAcquired());
+
+ // Wait for acquisition of the write lock on another thread (and be notified via a latch)
+ final CountDownLatch latch = new CountDownLatch(1);
+ new Thread(() -> {
+ configSetWL1.waitUntilAcquired();
+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind
+ latch.countDown();
+ }).start();
+
+ // Wait for the thread to start and to get blocked in waitUntilAcquired()
+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired
+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,
+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).
+ Thread.sleep(30);
+
+ assertEquals("we should not have been notified that configSetWL1 was acquired", 1, latch.getCount());
+ assertFalse("configSetWL1 should not have been acquired", configSetWL1.isAcquired());
+
+ configSetRL1.release();
+ assertTrue("configSetWL1 should have been acquired now that configSetRL1 was released", configSetWL1.isAcquired());
+
+ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch
+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause
+ int i = 0;
+ while (i < 1000 && latch.getCount() != 0) {
+ Thread.sleep(10);
+ i++;
+ }
+ assertEquals("we should have been notified that configSetWL1 was acquired", 0, latch.getCount());
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java
new file mode 100644
index 00000000000..e59ec4d0105
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.api.collections;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.DistributedMultiLock;
+import org.apache.solr.cloud.ZkDistributedCollectionLockFactory;
+import org.apache.solr.cloud.ZkTestServer;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.CollectionParams;
+import org.junit.Test;
+
+public class CollectionApiLockingTest extends SolrTestCaseJ4 {
+ private static final String COLLECTION_NAME = "lockColl";
+ final String SHARD1_NAME = "lockShard1";
+ final String SHARD2_NAME = "lockShard2";
+ final String REPLICA_NAME = "lockReplica";
+
+ static final int TIMEOUT = 10000;
+
+ /**
+ * Tests the Collection API locking. All tests run from single test method to save on setup time.
+ */
+ @Test
+ public void monothreadedApiLockTests() throws Exception {
+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));
+ try {
+ server.run();
+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {
+ CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot"));
+
+ monothreadedTests(apiLockFactory);
+ multithreadedTests(apiLockFactory);
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception {
+ // Lock at collection level (which prevents locking + acquiring on any other level of the hierarchy)
+ DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertTrue("Collection should have been acquired", collLock.isAcquired());
+ assertEquals("Lock at collection level expected to need one distributed lock", 1, collLock.getCountInternalLocks());
+
+ // Request a shard lock. Will not be acquired as long as we don't release the collection lock above
+ DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null);
+ assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired());
+ assertEquals("Lock at shard level expected to need two distributed locks", 2, shard1Lock.getCountInternalLocks());
+
+ // Request a lock on another shard. Will not be acquired as long as we don't release the collection lock above
+ DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null);
+ assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired());
+
+ assertTrue("Collection should still be acquired", collLock.isAcquired());
+
+ collLock.release();
+
+ assertTrue("Shard1 should have been acquired now that collection lock released", shard1Lock.isAcquired());
+ assertTrue("Shard2 should have been acquired now that collection lock released", shard2Lock.isAcquired());
+
+ // Request a lock on replica of shard1
+ DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME);
+ assertFalse("replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired());
+
+ // Now ask for a new lock on the collection
+ collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+
+ assertFalse("Collection should not have been acquired, shard1 and shard2 locks preventing it", collLock.isAcquired());
+
+ shard1Lock.release();
+ assertTrue("replicaShard1Lock should have been acquired, as shard1 got released", replicaShard1Lock.isAcquired());
+ assertFalse("Collection should not have been acquired, shard2 lock is preventing it", collLock.isAcquired());
+
+ replicaShard1Lock.release();
+
+ // Request a lock on replica of shard2
+ DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME);
+ assertFalse("replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired());
+
+ shard2Lock.release();
+
+ assertTrue("Collection should have been acquired as shard2 got released and replicaShard2Locks was requested after the collection lock", collLock.isAcquired());
+ assertFalse("replicaShard2Lock should not have been acquired, collLock is locked", replicaShard2Lock.isAcquired());
+
+ collLock.release();
+ assertTrue("replicaShard2Lock should have been acquired, the collection lock got released", replicaShard2Lock.isAcquired());
+
+ // Release remaining lock to allow the multithreaded locking to succeed
+ replicaShard2Lock.release();
+ }
+
+ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception {
+ // Lock on collection...
+ DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);
+ assertTrue("Collection should have been acquired", collLock.isAcquired());
+
+ // ...blocks a lock on replica from being acquired
+ final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME);
+ assertFalse("replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired());
+
+ // Wait for acquisition of the replica lock on another thread (and be notified via a latch)
+ final CountDownLatch latch = new CountDownLatch(1);
+ new Thread(() -> {
+ replicaShard1Lock.waitUntilAcquired();
+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind
+ latch.countDown();
+ }).start();
+
+ // Wait for the thread to start and to get blocked in waitUntilAcquired()
+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired
+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,
+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).
+ Thread.sleep(30);
+
+ assertEquals("we should not have been notified that replica was acquired", 1, latch.getCount());
+ assertFalse("replica lock should not have been acquired", replicaShard1Lock.isAcquired());
+
+ collLock.release();
+ assertTrue("replica lock should have been acquired now that collection lock was released", replicaShard1Lock.isAcquired());
+
+ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch
+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause
+ int i = 0;
+ while (i < 1000 && latch.getCount() != 0) {
+ Thread.sleep(10);
+ i++;
+ }
+ assertEquals("we should have been notified that replica lock was acquired", 0, latch.getCount());
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
index a9fdcaee150..21dd9dbea67 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
@@ -55,7 +55,7 @@ import org.slf4j.LoggerFactory;
public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
private static final int MAX_TIMEOUT_SECONDS = 90;
-
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Before
@@ -65,7 +65,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
}
-
+
@After
public void tearDown() throws Exception {
super.tearDown();
@@ -107,9 +107,9 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
.processAndWait(client, MAX_TIMEOUT_SECONDS);
assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state);
-
+
cluster.waitForActiveCollection(collection, 1, 1);
-
+
//Add a few documents to shard1
int numDocs = TestUtil.nextInt(random(), 10, 100);
List<SolrInputDocument> docs = new ArrayList<>(numDocs);
@@ -135,7 +135,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
assertSame("CreateShard did not complete", RequestStatusState.COMPLETED, state);
client.getZkStateReader().forceUpdateCollection(collection);
-
+
//Add a doc to shard2 to make sure shard2 was created properly
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", numDocs + 1);
@@ -187,7 +187,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
} catch (SolrException e) {
//expected
}
-
+
Slice shard1 = client.getZkStateReader().getClusterState().getCollection(collection).getSlice("shard1");
Replica replica = shard1.getReplicas().iterator().next();
for (String liveNode : client.getZkStateReader().getClusterState().getLiveNodes()) {
@@ -199,7 +199,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
}
}
client.getZkStateReader().forceUpdateCollection(collection);
-
+
shard1 = client.getZkStateReader().getClusterState().getCollection(collection).getSlice("shard1");
String replicaName = shard1.getReplicas().iterator().next().getName();
state = CollectionAdminRequest.deleteReplica(collection, "shard1", replicaName)
@@ -223,17 +223,17 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
.setShards("shard1")
.processAndWait(cluster.getSolrClient(), MAX_TIMEOUT_SECONDS);
assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state);
-
+
int numThreads = 10;
final AtomicInteger numSuccess = new AtomicInteger(0);
final AtomicInteger numFailure = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(numThreads);
-
+
ExecutorService es = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrNamedThreadFactory("testAsyncIdRaceCondition"));
try {
for (int i = 0; i < numThreads; i++) {
es.submit(new Runnable() {
-
+
@Override
public void run() {
CollectionAdminRequest.Reload reloadCollectionRequest = CollectionAdminRequest.reloadCollection("testAsyncIdRaceCondition");
@@ -243,7 +243,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
} catch (InterruptedException e) {
throw new RuntimeException();
}
-
+
try {
if (log.isInfoEnabled()) {
log.info("{} - Reloading Collection.", Thread.currentThread().getName());
@@ -254,7 +254,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
if (log.isInfoEnabled()) {
log.info("Exception during collection reloading, we were waiting for one: ", e);
}
- assertEquals("Task with the same requestid already exists.", e.getMessage());
+ assertEquals("Task with the same requestid already exists. (repeatedId)", e.getMessage());
numFailure.incrementAndGet();
} catch (IOException e) {
throw new RuntimeException();
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
index 884ea222fa0..90e36c5158c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
@@ -62,7 +62,9 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
@Test
@ShardsFixed(num = 2)
public void test() throws Exception {
+ final boolean isDistributedCollectionApi;
try (CloudSolrClient client = createCloudClient(null)) {
+ isDistributedCollectionApi = new CollectionAdminRequest.RequestApiDistributedProcessing().process(client).getIsCollectionApiDistributed();
CollectionAdminRequest.Create req;
if (useTlogReplicas()) {
req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2, 0, 1, 1);
@@ -85,7 +87,9 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {
clusterStatusWithCollectionAndMultipleShards();
clusterStatusWithRouteKey();
clusterStatusAliasTest();
- clusterStatusRolesTest();
+ if (!isDistributedCollectionApi) {
+ clusterStatusRolesTest();
+ }
clusterStatusBadCollectionTest();
replicaPropTest();
clusterStatusZNodeVersion();
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java
index dc49620c01e..97aecf07930 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java
@@ -164,7 +164,7 @@ public class TestRequestStatusCollectionAPI extends BasicDistributedZkTest {
e.printStackTrace();
}
- assertEquals("Task with the same requestid already exists.", r.get("error"));
+ assertEquals("Task with the same requestid already exists. (1002)", r.get("error"));
}
@SuppressWarnings("unchecked")
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java b/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java
index f0ce7ef5a9a..eb97ea7cf11 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java
@@ -42,7 +42,7 @@ public class TestConfigsApi extends SolrTestCaseJ4 {
}
@Override
- protected void sendToZk(SolrQueryResponse rsp,
+ protected void sendToOverseer(SolrQueryResponse rsp,
ConfigSetOperation operation,
Map<String, Object> result) {
result.put(QUEUE_OPERATION, operation.action.toLower());
diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xml
index 0fc94e24473..8c73d838fc1 100644
--- a/solr/server/solr/solr.xml
+++ b/solr/server/solr/solr.xml
@@ -46,6 +46,7 @@
<str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>
<str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>
<bool name="distributedClusterStateUpdates">${distributedClusterStateUpdates:false}</bool>
+ <bool name="distributedCollectionConfigSetExecution">${distributedCollectionConfigSetExecution:false}</bool>
</solrcloud>
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 7d1f977f24d..607a5a8f915 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -848,14 +848,14 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
/**
- * Return a SolrRequest for low-level detailed status of the specified collection.
+ * Return a SolrRequest for low-level detailed status of the specified collection.
* @param collection the collection to get the status of.
*/
public static ColStatus collectionStatus(String collection) {
checkNotNull(CoreAdminParams.COLLECTION, collection);
return new ColStatus(collection);
}
-
+
/**
* Return a SolrRequest for low-level detailed status of all collections on the cluster.
*/
@@ -878,7 +878,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
super(CollectionAction.COLSTATUS);
this.collection = collection;
}
-
+
private ColStatus() {
super(CollectionAction.COLSTATUS);
}
@@ -1340,6 +1340,35 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
+ /**
+ * Returns a SolrRequest to run a mock task. For tests only.
+ */
+ public static MockCollTask mockCollTask(String collection) {
+ return new MockCollTask(collection);
+ }
+
+ // MOCK_COLL_TASK request
+ public static class MockCollTask extends AsyncCollectionAdminRequest {
+ protected final String collection;
+ protected String sleep;
+
+ private MockCollTask(String collection) {
+ super(CollectionAction.MOCK_COLL_TASK);
+ this.collection = checkNotNull(CoreAdminParams.COLLECTION, collection);
+
+ }
+
+ public MockCollTask setSleep(String sleep) { this.sleep = sleep; return this; }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.add(CoreAdminParams.COLLECTION, collection);
+ params.setNonNull("sleep", sleep);
+ return params;
+ }
+ }
+
/**
* Returns a SolrRequest to split a shard in a collection
*/
@@ -2605,6 +2634,30 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
+ // DISTRIBUTEDAPIPROCESSING request for verifying if Collection API execution is distributed
+ public static class RequestApiDistributedProcessing extends CollectionAdminRequest<RequestApiDistributedProcessingResponse> {
+
+ public RequestApiDistributedProcessing() {
+ super(CollectionAction.DISTRIBUTEDAPIPROCESSING);
+ }
+
+ @Override
+ protected RequestApiDistributedProcessingResponse createResponse(SolrClient client) {
+ return new RequestApiDistributedProcessingResponse();
+ }
+ }
+
+ /**
+ * A response object for {@link RequestApiDistributedProcessing} requests
+ */
+ public static class RequestApiDistributedProcessingResponse extends CollectionAdminResponse {
+ public boolean getIsCollectionApiDistributed() {
+ @SuppressWarnings({"rawtypes"})
+ Boolean isDistributedApi = (Boolean) getResponse().get("isDistributedApi");
+ return isDistributedApi;
+ }
+ }
+
/**
* Return a SolrRequest to get the Cluster status
*/
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 1ffa05e7980..b63690fbd8b 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -104,11 +104,13 @@ public interface CollectionParams {
ADDREPLICA(true, LockLevel.SHARD),
MOVEREPLICA(true, LockLevel.SHARD),
OVERSEERSTATUS(false, LockLevel.NONE),
+ // For testing. Could eventually be exposed publicly if needed
+ DISTRIBUTEDAPIPROCESSING(false, LockLevel.NONE),
LIST(false, LockLevel.NONE),
CLUSTERSTATUS(false, LockLevel.NONE),
ADDREPLICAPROP(true, LockLevel.REPLICA),
DELETEREPLICAPROP(true, LockLevel.REPLICA),
- BALANCESHARDUNIQUE(true, LockLevel.SHARD),
+ BALANCESHARDUNIQUE(true, LockLevel.COLLECTION),
REBALANCELEADERS(true, LockLevel.COLLECTION),
MODIFYCOLLECTION(true, LockLevel.COLLECTION),
BACKUP(true, LockLevel.COLLECTION),
@@ -143,7 +145,7 @@ public interface CollectionParams {
CollectionAction(boolean isWrite, LockLevel level) {
this.isWrite = isWrite;
this.lockLevel = level;
- lowerName = toString().toLowerCase(Locale.ROOT);
+ lowerName = name().toLowerCase(Locale.ROOT);
}
public static CollectionAction get(String p) {
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index f14b6f2b1da..79876b6f0ff 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -121,6 +121,7 @@ public class MiniSolrCloudCluster {
" <str name=\"pkiHandlerPrivateKeyPath\">${pkiHandlerPrivateKeyPath:cryptokeys/priv_key512_pkcs8.pem}</str> \n" +
" <str name=\"pkiHandlerPublicKeyPath\">${pkiHandlerPublicKeyPath:cryptokeys/pub_key512.der}</str> \n" +
" <str name=\"distributedClusterStateUpdates\">${solr.distributedClusterStateUpdates:false}</str> \n" +
+ " <str name=\"distributedCollectionConfigSetExecution\">${solr.distributedCollectionConfigSetExecution:false}</str> \n" +
" </solrcloud>\n" +
// NOTE: this turns off the metrics collection unless overriden by a sysprop
" <metrics enabled=\"${metricsEnabled:false}\">\n" +
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index afb6d4dc339..08cce5d0dda 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -109,6 +109,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
private Map<String, Object> clusterProperties = new HashMap<>();
private boolean trackJettyMetrics;
+ private boolean useDistributedCollectionConfigSetExecution;
private boolean useDistributedClusterStateUpdate;
/**
@@ -120,8 +121,11 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
public Builder(int nodeCount, Path baseDir) {
this.nodeCount = nodeCount;
this.baseDir = baseDir;
- // By default the MiniSolrCloudCluster being built will randomly (seed based) decide which cluster update strategy to use
- this.useDistributedClusterStateUpdate = LuceneTestCase.random().nextInt(2) == 0;
+ // By default the MiniSolrCloudCluster being built will randomly (seed based) decide which collection API strategy
+ // to use (distributed or Overseer based) and which cluster update strategy to use (distributed if collection API
+ // is distributed, but Overseer based or distributed randomly chosen if Collection API is Overseer based)
+ this.useDistributedCollectionConfigSetExecution = LuceneTestCase.random().nextInt(2) == 0;
+ this.useDistributedClusterStateUpdate = useDistributedCollectionConfigSetExecution || LuceneTestCase.random().nextInt(2) == 0;
}
/**
@@ -190,21 +194,25 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
}
/**
- * This method makes the MiniSolrCloudCluster use the "other" cluster state update strategy than it normally would.
+ * This method makes the MiniSolrCloudCluster use the "other" Collection API execution strategy than it normally would.
* When some test classes call this method (and some don't) we make sure that a run of multiple tests with a single
- * seed will exercise both code lines (distributed updates and Overseer based updates) so regressions can be spotted
+ * seed will exercise both code lines (distributed and Overseer based Collection API) so regressions can be spotted
* faster.<p>
*
* The real need is for a few tests covering reasonable use cases to call this method. If you're adding a new test,
* you don't have to call it (but it's ok if you do).
*/
- public Builder useOtherClusterStateUpdateStrategy() {
- useDistributedClusterStateUpdate = !useDistributedClusterStateUpdate;
+ public Builder useOtherCollectionConfigSetExecution() {
+ // Switch from Overseer to distributed Collection execution and vice versa
+ useDistributedCollectionConfigSetExecution = !useDistributedCollectionConfigSetExecution;
+ // Reverse distributed cluster state updates as well if possible (state can't be Overseer based if Collections API is distributed)
+ useDistributedClusterStateUpdate = !useDistributedClusterStateUpdate || useDistributedCollectionConfigSetExecution;
return this;
}
/**
- * Force the cluster state update strategy to be either Overseer based or distributed. <b>This method can be useful when
+ * Force the cluster Collection and config state API execution as well as the cluster state update strategy to be
+ * either Overseer based or distributed. <b>This method can be useful when
* debugging tests</b> failing in only one of the two modes to have all local runs exhibit the issue, as well obviously for
* tests that are not compatible with one of the two modes.
* <p>
@@ -215,19 +223,24 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
* <solrcloud>
* ....
* <str name="distributedClusterStateUpdates">${solr.distributedClusterStateUpdates:false}</str>
+ * <str name="distributedCollectionConfigSetExecution">${solr.distributedCollectionConfigSetExecution:false}</str>
* ....
* </solrcloud>
* }</pre>
* For an example of a configuration supporting this setting, see {@link MiniSolrCloudCluster#DEFAULT_CLOUD_SOLR_XML}.
* When a test sets a different {@code solr.xml} config (using {@link #withSolrXml}), if the config does not contain
- * the placeholder, the strategy will be defined by the value assigned to {@code useDistributedClusterStateUpdates}
- * in {@link org.apache.solr.core.CloudConfig.CloudConfigBuilder}.
+ * the placeholder, the strategy will be defined by the values assigned to {@code useDistributedClusterStateUpdates}
+ * and {@code useDistributedCollectionConfigSetExecution} in {@link org.apache.solr.core.CloudConfig.CloudConfigBuilder}.
*
- * @param distributed When {@code true}, cluster state updates are handled in a distributed way by nodes. When
- * {@code false}, cluster state updates are handled by Overseer.
+ * @param distributedCollectionConfigSetApi When {@code true}, Collection and Config Set API commands are executed in
+ * a distributed way by nodes. When {@code false}, they are executed by Overseer.
+ * @param distributedClusterStateUpdates When {@code true}, cluster state updates are handled in a distributed way by nodes. When
+ * {@code false}, cluster state updates are handled by Overseer.<p>
+ * If {@code distributedCollectionConfigSetApi} is {@code true} then this parameter must be {@code true}.
*/
- public Builder withDistributedClusterStateUpdates(boolean distributed) {
- useDistributedClusterStateUpdate = distributed;
+ public Builder withDistributedClusterStateUpdates(boolean distributedCollectionConfigSetApi, boolean distributedClusterStateUpdates) {
+ useDistributedCollectionConfigSetExecution = distributedCollectionConfigSetApi;
+ useDistributedClusterStateUpdate = distributedClusterStateUpdates;
return this;
}
@@ -262,12 +275,13 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
* @throws Exception if an error occurs on startup
*/
public MiniSolrCloudCluster build() throws Exception {
- // This will have an impact on how the MiniSolrCloudCluster and therefore the test run if the config being
- // used does have the appropriate placeholder.
- // It is a good place to hard code true or false instead of useDistributedClusterStateUpdate to run all qualifying
- // tests with a given cluster state update strategy (non qualifying tests will use the default value assigned to
- // useDistributedClusterStateUpdates in org.apache.solr.core.CloudConfig.CloudConfigBuilder, so if you really want
- // ALL tests to run with a given strategy, patch it there too (and revert before commit!)
+ // Two lines below will have an impact on how the MiniSolrCloudCluster and therefore the test run if the config being
+ // used in the test does have the appropriate placeholders. See for example DEFAULT_CLOUD_SOLR_XML in MiniSolrCloudCluster.
+ // Hard coding values here will impact such tests.
+ // To hard code behavior for tests not having these placeholders - and for SolrCloud as well for that matter! -
+ // change the values assigned to useDistributedClusterStateUpdates and useDistributedCollectionConfigSetExecution in
+ // org.apache.solr.core.CloudConfig.CloudConfigBuilder. Do not forget then to revert before commit!
+ System.setProperty("solr.distributedCollectionConfigSetExecution", Boolean.toString(useDistributedCollectionConfigSetExecution));
System.setProperty("solr.distributedClusterStateUpdates", Boolean.toString(useDistributedClusterStateUpdate));
JettyConfig jettyConfig = jettyConfigBuilder.build();