Apache Solr open-source search software
SOLR-15146: Collection and Config Set API can run distributed on all nodes (#70)
Ilan Ginzburg4a78b458047c
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex e56a5c48165..afe4751addd 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -260,6 +260,9 @@ Other Changes* SOLR-15341: Lucene has removed CodecReader#ramBytesUsed in LUCENE-9387, so ramBytesUsed will no longer be reportedin SegmentsInfo handler (janhoy)+* SOLR-15146: Allow Collection API and Config Set API to be done in a distributed fashion without going through Overseer (Ilan Ginzburg)++Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order executiondiff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.javanew file mode 100644index 00000000000..7200969cbc8--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetApiLockFactory.java@@ -0,0 +1,54 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;+import java.util.ArrayList;+import java.util.List;++/**+ * This class implements a higher level locking abstraction for the Config Set API using lower level read and write locks.+ */+public class ConfigSetApiLockFactory {++ private final DistributedConfigSetLockFactory lockFactory;++ public ConfigSetApiLockFactory(DistributedConfigSetLockFactory lockFactory) {+ this.lockFactory = lockFactory;+ }++ /**+ * For the {@link org.apache.solr.common.params.CollectionParams.LockLevel} of the passed {@code action}, obtains the+ * required locks (if any) and returns.<p>+ *+ * This method obtains a write lock on {@code configSetName} as well as (when not {@code null}), a read lock on {@code baseConfigSetName}.+ *+ * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding Config Set API command+ * can execute safely.+ * The returned lock <b>MUST</b> be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would+ * prevent other threads from locking.+ */+ public DistributedMultiLock createConfigSetApiLock(String configSetName, String baseConfigSetName) {+ List<DistributedLock> locks = new ArrayList<>(2);++ locks.add(lockFactory.createLock(true, configSetName));+ if (baseConfigSetName != null) {+ locks.add(lockFactory.createLock(false, baseConfigSetName));+ }++ return new DistributedMultiLock(locks);+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.javanew file mode 100644index 00000000000..5d5983a5f0c--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ConfigSetCmds.java@@ -0,0 +1,218 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.io.InputStreamReader;+import java.lang.invoke.MethodHandles;+import java.nio.charset.StandardCharsets;+import java.util.HashMap;+import java.util.Iterator;+import java.util.Map;++import com.jayway.jsonpath.internal.Utils;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.ZkNodeProps;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.ConfigSetParams;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.core.ConfigSetProperties;+import org.apache.solr.core.ConfigSetService;+import org.apache.solr.core.CoreContainer;+import org.apache.zookeeper.KeeperException;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import static org.apache.solr.common.params.CommonParams.NAME;+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;+import static org.apache.solr.common.util.Utils.toJSONString;+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;++/**+ * This class contains methods dealing with Config Sets and called for Config Set API execution, called+ * from the {@link OverseerConfigSetMessageHandler} or from+ * {@link org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner#runConfigSetCommand} depending+ * on whether Collection and Config Set APIs are Overseer based or distributed.+ */+public class ConfigSetCmds {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ /**+ * Name of the ConfigSet to copy from for CREATE+ */+ public static final String BASE_CONFIGSET = "baseConfigSet";++ /**+ * Prefix for properties that should be applied to the ConfigSet for CREATE+ */+ public static final String CONFIG_SET_PROPERTY_PREFIX = "configSetProp.";++ public static String getBaseConfigSetName(ConfigSetParams.ConfigSetAction action, String baseConfigSetName) {+ if (action == CREATE) {+ return Utils.isEmpty(baseConfigSetName) ? DEFAULT_CONFIGSET_NAME : baseConfigSetName;+ }+ return null;+ }+++ @SuppressWarnings({"rawtypes"})+ private static NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {+ byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);+ if (oldPropsData != null) {+ InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);+ try {+ return ConfigSetProperties.readFromInputStream(reader);+ } finally {+ reader.close();+ }+ }+ return null;+ }++ private static Map<String, Object> getNewProperties(ZkNodeProps message) {+ Map<String, Object> properties = null;+ for (Map.Entry<String, Object> entry : message.getProperties().entrySet()) {+ if (entry.getKey().startsWith(CONFIG_SET_PROPERTY_PREFIX)) {+ if (properties == null) {+ properties = new HashMap<String, Object>();+ }+ properties.put(entry.getKey().substring((CONFIG_SET_PROPERTY_PREFIX).length()),+ entry.getValue());+ }+ }+ return properties;+ }++ private static void mergeOldProperties(Map<String, Object> newProps, @SuppressWarnings({"rawtypes"})NamedList oldProps) {+ @SuppressWarnings({"unchecked"})+ Iterator<Map.Entry<String, Object>> it = oldProps.iterator();+ while (it.hasNext()) {+ Map.Entry<String, Object> oldEntry = it.next();+ if (!newProps.containsKey(oldEntry.getKey())) {+ newProps.put(oldEntry.getKey(), oldEntry.getValue());+ }+ }+ }++ private static byte[] getPropertyData(Map<String, Object> newProps) {+ if (newProps != null) {+ String propertyDataStr = toJSONString(newProps);+ if (propertyDataStr == null) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid property specification");+ }+ return propertyDataStr.getBytes(StandardCharsets.UTF_8);+ }+ return null;+ }++ public static void createConfigSet(ZkNodeProps message, CoreContainer coreContainer) throws IOException {+ String configSetName = message.getStr(NAME);+ if (configSetName == null || configSetName.length() == 0) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet name not specified");+ }++ String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);++ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);+ }++ // is there a base config that already exists+ if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,+ "Base ConfigSet does not exist: " + baseConfigSetName);+ }++ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;+ Map<String, Object> props = getNewProperties(message);+ if (props != null) {+ // read the old config properties and do a merge, if necessary+ @SuppressWarnings({"rawtypes"})+ NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);+ if (oldProps != null) {+ mergeOldProperties(props, oldProps);+ }+ }+ byte[] propertyData = getPropertyData(props);++ try {+ coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);+ if (propertyData != null) {+ coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);+ }+ } catch (Exception e) {+ // copying the config dir or writing the properties file may have failed.+ // we should delete the ConfigSet because it may be invalid,+ // assuming we actually wrote something. E.g. could be+ // the entire baseConfig set with the old properties, including immutable,+ // that would make it impossible for the user to delete.+ try {+ if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {+ deleteConfigSet(configSetName, true, coreContainer);+ }+ } catch (IOException ioe) {+ log.error("Error while trying to delete partially created ConfigSet", ioe);+ }+ throw e;+ }+ }++ public static void deleteConfigSet(ZkNodeProps message, CoreContainer coreContainer) throws IOException {+ String configSetName = message.getStr(NAME);+ if (configSetName == null || configSetName.length() == 0) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet name not specified");+ }++ deleteConfigSet(configSetName, false, coreContainer);+ }++ private static void deleteConfigSet(String configSetName, boolean force, CoreContainer coreContainer) throws IOException {+ if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);+ }++ ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();++ for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {+ String configName = null;+ try {+ configName = zkStateReader.readConfigName(entry.getKey());+ } catch (KeeperException ex) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,+ "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");+ }+ if (configSetName.equals(configName))+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,+ "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");+ }++ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;+ @SuppressWarnings({"rawtypes"})+ NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);+ if (properties != null) {+ Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);+ boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;+ if (!force && isImmutableConfigSet) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);+ }+ }+ coreContainer.getConfigSetService().deleteConfig(configSetName);+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.javanew file mode 100644index 00000000000..cc007fd18de--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedApiAsyncTracker.java@@ -0,0 +1,355 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.lang.invoke.MethodHandles;+import java.nio.charset.StandardCharsets;+import java.util.Collection;++import com.google.common.annotations.VisibleForTesting;+import org.apache.solr.client.solrj.response.RequestStatusState;+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Pair;+import org.apache.solr.common.util.SimpleOrderedMap;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;++/**+ * Class holding the implementation required for tracking asynchronous Collection API (or other) tasks when the Collection+ * API is distributed.<p>+ *+ * This replaces the features provided by the distributed maps on ZK paths /overseer/collection-map-completed,+ * /overseer/collection-map-failure and /overseer/async_ids when the Collection API commands are handled by the Overseer.<p>+ *+ * It works by using two Zookeeper directories, one for persistent nodes for each new async id and one for ephemeral nodes+ * for each async id currently being processed (in flight).<br>+ * A persistent async node has either no data, or has a serialized OverseerSolrResponse as content.+ * An ephemeral async node has two possible states (content): 'S' or 'R'.<p>+ *+ * The actual state of an async task is built from a combination of the two nodes:+ * <pre>+ * +===================+=========================================+=================================================+====================++ * | | persistent=success OverseerSolrResponse | persistent=null or failed OverseerSolrResponse | No persistent node |+ * +===================+=========================================+=================================================+====================++ * | ephemeral="S" | Task completed successfully | Task submitted | Unknown task |+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------++ * | ephemeral="R" | Task completed successfully | Task running | Unknown task |+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------++ * | No ephemeral node | Task completed successfully | Task failed (see response or null=node failure) | Unknown task |+ * +-------------------+-----------------------------------------+-------------------------------------------------+--------------------++ * </pre>+ */+public class DistributedApiAsyncTracker {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ /**+ * Corresponds to Overseer.NUM_RESPONSES_TO_STORE. The size of the persistent store of async ID's put in Zookeeper.+ * This is the max total tracked async request ID's over all nodes running in the distributed Collection API.+ */+ public static int MAX_TRACKED_ASYNC_TASKS = 10000;++ private static final String ZK_ASYNC_PERSISTENT = "/persistent";+ private static final String ZK_ASYNC_INFLIGHT = "/inflight";++ private final String persistentIdsPath;+ private final String inFlightIdsPath;++ /**+ * Persistent storage in Zookeeper under path {@link #ZK_ASYNC_PERSISTENT} of all currently known (in flight, completed+ * with success or error) async request id's.+ */+ private final SizeLimitedDistributedMap trackedAsyncTasks;++ private final InFlightJobs inFlightAsyncTasks;++ public DistributedApiAsyncTracker(SolrZkClient zkClient, String rootPath) {+ this(zkClient, rootPath, MAX_TRACKED_ASYNC_TASKS);+ }++ @VisibleForTesting+ DistributedApiAsyncTracker(SolrZkClient zkClient, String rootPath, int maxTrackedTasks) {+ persistentIdsPath = rootPath + ZK_ASYNC_PERSISTENT;+ inFlightIdsPath = rootPath + ZK_ASYNC_INFLIGHT;++ trackedAsyncTasks = new SizeLimitedDistributedMap(zkClient, persistentIdsPath, maxTrackedTasks, null);+ inFlightAsyncTasks = new InFlightJobs(zkClient, inFlightIdsPath);+ }++ /**+ * After a successful call to this method, caller MUST eventually call {@link #setTaskCompleted} or {@link #cancelAsyncId}+ * otherwise the task will forever be considered as in progress.+ * @param asyncId if {@code null} this method will do nothing.+ * @return {@code true} if the asyncId was not already in use (or is {@code null}) and {@code false} if it is already+ * in use and can't be allocated again.+ */+ public boolean createNewAsyncJobTracker(String asyncId) {+ if (asyncId == null) {+ return true;+ }+ try {+ // First create the persistent node, with no content. If that fails, it means the asyncId has been previously used+ // and not yet cleared...+ if (!trackedAsyncTasks.putIfAbsent(asyncId, null)) {+ return false;+ }++ // ...then create the transient node. If the corresponding ephemeral node already exists, it means the persistent node+ // was removed (maybe trackedAsyncTasks grew too large? It has a max size then evicts). We cannot then track the new+ // provided asyncId, and have simply "revived" its persistent node...+ try {+ inFlightAsyncTasks.createNewInFlightTask(asyncId);+ return true;+ } catch (KeeperException.NodeExistsException nee) {+ log.warn("Async id {} was not found in trackedAsyncTasks but was still present in inFlightAsyncTasks", asyncId);+ return false;+ }+ } catch (KeeperException ke) {+ throw new SolrException(SERVER_ERROR, "Error creating new async job tracking " + asyncId, ke);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new SolrException(SERVER_ERROR, "Interrupted creating new async job tracking " + asyncId, ie);+ }+ }++ /**+ * Initially an async task is submitted. Just before it actually starts execution it is set to running.+ */+ public void setTaskRunning(String asyncId) {+ if (asyncId == null) {+ return;+ }+ try {+ inFlightAsyncTasks.setTaskRunning(asyncId);+ } catch (KeeperException ke) {+ throw new SolrException(SERVER_ERROR, "Error setting async task as running " + asyncId, ke);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new SolrException(SERVER_ERROR, "Interrupted setting async task as running " + asyncId, ie);+ }+ }++ /**+ * Mark the completion (success or error) of an async task. The success or error is judged by the contents+ * of the {@link OverseerSolrResponse}.+ */+ public void setTaskCompleted(String asyncId, OverseerSolrResponse solrResponse) {+ if (asyncId == null) {+ return;+ }+ // First update the persistent node with the execution result, only then remove the transient node+ // (otherwise a status check might report the task in error)+ try {+ try {+ trackedAsyncTasks.put(asyncId, OverseerSolrResponseSerializer.serialize(solrResponse));+ } finally {+ inFlightAsyncTasks.deleteInFlightTask(asyncId);+ }+ } catch (KeeperException ke) {+ throw new SolrException(SERVER_ERROR, "Error setting async task as completed " + asyncId, ke);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new SolrException(SERVER_ERROR, "Interrupted setting async task as completed " + asyncId, ie);+ }+ }++ /**+ * Cancels the tracking of an asyncId, if the corresponding command could not be executed.+ */+ public void cancelAsyncId(String asyncId) {+ if (asyncId == null) {+ return;+ }+ try {+ try {+ trackedAsyncTasks.remove(asyncId);+ } finally {+ inFlightAsyncTasks.deleteInFlightTask(asyncId);+ }+ } catch (KeeperException ke) {+ throw new SolrException(SERVER_ERROR, "Error canceling async task " + asyncId, ke);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new SolrException(SERVER_ERROR, "Interrupted canceling async task " + asyncId, ie);+ }+ }++ /**+ * This method implements the logic described in the class Javadoc table ({@link DistributedApiAsyncTracker}), using the+ * two sets of tracking info to build the actual state of an async task.<p>+ *+ * Returns the status of an async task, and when relevant the corresponding response from the command execution. The+ * returned {@link OverseerSolrResponse} will not be {@code null} when the returned+ * {@link RequestStatusState} is {@link RequestStatusState#COMPLETED} or {@link RequestStatusState#FAILED} (and will be+ * {@code null} in all other cases).+ */+ @SuppressWarnings("unchecked")+ public Pair<RequestStatusState, OverseerSolrResponse> getAsyncTaskRequestStatus(String asyncId) throws Exception {+ if (asyncId == null || !trackedAsyncTasks.contains(asyncId)) {+ // This return addresses the whole "No persistent node" column from the table+ return new Pair<>(RequestStatusState.NOT_FOUND, null);+ }++ byte[] data = trackedAsyncTasks.get(asyncId);+ OverseerSolrResponse response = data != null ? OverseerSolrResponseSerializer.deserialize(data) : null;++ if (response != null && response.getResponse().get("failure") == null && response.getResponse().get("exception") == null) {+ // This return addresses the whole "persistent=success OverseerSolrResponse" column from the table+ return new Pair<>(RequestStatusState.COMPLETED, response);+ }++ // Now dealing with the middle column "persistent=null or failed OverseerSolrResponse"+ InFlightJobs.State ephemeralState = inFlightAsyncTasks.getInFlightState(asyncId);+ if (ephemeralState == InFlightJobs.State.SUBMITTED) {+ return new Pair<>(RequestStatusState.SUBMITTED, null);+ } else if (ephemeralState == InFlightJobs.State.RUNNING) {+ return new Pair<>(RequestStatusState.RUNNING, null);+ }++ // The task has failed, but there are two options: if response is null, it has failed because the node on which it was+ // running has crashed. If it is not null, it has failed because the execution has failed. Because caller expects a non+ // null response in any case, let's make up one if needed...+ if (response == null) {+ // Node crash has removed the ephemeral node, but the command did not complete execution (or didn't even start it, who+ // knows). We have a failure to report though so let's create a reasonable return response.+ @SuppressWarnings({"rawtypes"})+ NamedList results = new NamedList();+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();+ nl.add("msg", "Operation (asyncId: " + asyncId + ") failed due to server restart. Please resubmit.");+ nl.add("rspCode", SERVER_ERROR.code);+ results.add("exception", nl);+ response = new OverseerSolrResponse(results);+ }++ return new Pair<>(RequestStatusState.FAILED, response);+ }+++ /**+ * Deletes a single async tracking ID if the corresponding job has completed or failed.+ * @return {@code true} if the {@code asyncId} was found to be of a completed or failed job and was successfully removed,+ * {@code false} if the id was not found or was found for a submitted or running job (these are not removed).+ */+ public boolean deleteSingleAsyncId(String asyncId) throws Exception {+ return inFlightAsyncTasks.getInFlightState(asyncId) == InFlightJobs.State.NOT_FOUND && trackedAsyncTasks.remove(asyncId);+ }++ /**+ * Deletes all async id's for completed or failed async jobs. Does not touch id's for submitted or running jobs.+ */+ public void deleteAllAsyncIds() throws Exception {+ Collection<String> allTracked = trackedAsyncTasks.keys();++ for (String asyncId : allTracked) {+ deleteSingleAsyncId(asyncId);+ }+ }++ /**+ * Manages the ephemeral nodes for tasks currently being processed (running or waiting for a lock) by a node.+ */+ private static class InFlightJobs {+ enum State {+ SUBMITTED("S"),+ RUNNING("R"),+ NOT_FOUND(null);++ private final String shorthand;++ State(String shorthand) {+ this.shorthand = shorthand;+ }+ }++ private final SolrZkClient zkClient;+ private final String rootNodePath;++ InFlightJobs(SolrZkClient zkClient, String rootNodePath) {+ this.zkClient = zkClient;+ this.rootNodePath = rootNodePath;++ try {+ if (!zkClient.exists(rootNodePath, true)) {+ zkClient.makePath(rootNodePath, new byte[0], CreateMode.PERSISTENT, true);+ }+ } catch (KeeperException.NodeExistsException nee) {+ // Some other thread (on this or another JVM) beat us to create the node, that's ok, the node exists.+ } catch (KeeperException ke) {+ throw new SolrException(SERVER_ERROR, "Error creating root node " + rootNodePath, ke);+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ throw new SolrException(SERVER_ERROR, "Interrupted creating root node " + rootNodePath, ie);+ }+ }++ void createNewInFlightTask(String asyncId) throws KeeperException, InterruptedException {+ zkClient.create(getPath(asyncId), State.SUBMITTED.shorthand.getBytes(StandardCharsets.UTF_8), CreateMode.EPHEMERAL, true);+ }++ void setTaskRunning(String asyncId) throws KeeperException, InterruptedException {+ zkClient.setData(getPath(asyncId), State.RUNNING.shorthand.getBytes(StandardCharsets.UTF_8), true);+ }++ void deleteInFlightTask(String asyncId) throws KeeperException, InterruptedException {+ zkClient.delete(getPath(asyncId), -1, true);+ }++ State getInFlightState(String asyncId) throws KeeperException, InterruptedException {+ if (!zkClient.exists(getPath(asyncId), true)) {+ return State.NOT_FOUND;+ }++ final byte[] bytes;+ try {+ bytes = zkClient.getData(getPath(asyncId), null, null, true);+ } catch (KeeperException.NoNodeException nne) {+ // Unlikely race, but not impossible...+ if (log.isInfoEnabled()) {+ log.info("AsyncId ephemeral node " + getPath(asyncId) + " vanished from underneath us. Funny."); // nowarn+ }+ return State.NOT_FOUND;+ }++ if (bytes == null) {+ // This is not expected. The ephemeral nodes are always created with content.+ log.error("AsyncId ephemeral node " + getPath(asyncId) + " has null content. This is unexpected (bug)."); // nowarn+ return State.NOT_FOUND;+ }++ String content = new String(bytes, StandardCharsets.UTF_8);+ if (State.RUNNING.shorthand.equals(content)) {+ return State.RUNNING;+ } else if (State.SUBMITTED.shorthand.equals(content)) {+ return State.SUBMITTED;+ } else {+ log.error("AsyncId ephemeral node " + getPath(asyncId) + " has unexpected content \"" + content + "\". This is unexpected (bug)."); // nowarn+ return State.NOT_FOUND;+ }+ }++ private String getPath(String asyncId) {+ return rootNodePath + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + asyncId;+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.javanew file mode 100644index 00000000000..d87dafc956a--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedCollectionLockFactory.java@@ -0,0 +1,53 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import org.apache.solr.cloud.api.collections.CollectionApiLockFactory;+import org.apache.solr.common.params.CollectionParams;++public interface DistributedCollectionLockFactory {+ /**+ * Create a new lock of the specified type (read or write) entering the "competition" for actually getting the lock at+ * the given level for the given path i.e. a lock at {@code collName} or a lock at {@code collName/shardId} or a lock+ * at {@code collName/shardId/replicaName}, depending on the passed {@code level}.<p>+ *+ * The paths are used to define which locks compete with each other (locks of equal paths compete).<p>+ *+ * Upon return from this call, the lock <b>has not been acquired</b> but the it had entered the lock acquiring "competition",+ * and the caller can decide to wait until the lock is granted by calling {@link DistributedLock#waitUntilAcquired()}.<br>+ * Separating the lock creation from lock acquisition allows a more deterministic release of the locks when/if they can't be+ * acquired.<p>+ *+ * Locks at different paths are independent of each other, multiple {@link DistributedLock} are therefore requested for+ * a single operation and are packaged together and returned as an {@link DistributedMultiLock},+ * see {@link CollectionApiLockFactory#createCollectionApiLock}.+ * @param isWriteLock {@code true} if requesting a write lock, {@code false} for a read lock.+ * @param level The requested locking level. Can be one of:+ * <ul><li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}</li>+ * <li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD}</li>+ * <li>{@link org.apache.solr.common.params.CollectionParams.LockLevel#REPLICA}</li></ul>+ * @param collName the collection name, can never be {@code null} as is needed for all locks.+ * @param shardId is ignored and can be {@code null} if {@code level} is {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}+ * @param replicaName is ignored and can be {@code null} if {@code level} is {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION}+ * or {@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD}+ * @return a lock instance that must be {@link DistributedLock#release()}'ed in a {@code finally},+ * regardless of the lock having been acquired or not.+ */+ DistributedLock createLock(boolean isWriteLock, CollectionParams.LockLevel level, String collName, String shardId,+ String replicaName);+}diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.javanew file mode 100644index 00000000000..4f2752740b6--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedConfigSetLockFactory.java@@ -0,0 +1,41 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++public interface DistributedConfigSetLockFactory {+ /**+ * Create a new lock of the specified type (read or write) entering the "competition" for actually getting the lock for+ * the {@code configSetName}<p>+ *+ * Upon return from this call, the lock <b>has not been acquired</b> but the it had entered the lock acquiring "competition",+ * and the caller can decide to wait until the lock is granted by calling {@link DistributedLock#waitUntilAcquired()}.<br>+ * Separating the lock creation from lock acquisition allows a more deterministic release of the locks when/if they can't be+ * acquired.<p>+ *+ * Locks at different paths are independent of each other, multiple {@link DistributedLock} are therefore requested for+ * a single operation and are packaged together and returned as an {@link DistributedMultiLock},+ * see {@link org.apache.solr.cloud.api.collections.CollectionApiLockFactory#createCollectionApiLock} or+ * {@link org.apache.solr.cloud.ConfigSetApiLockFactory#createConfigSetApiLock}.+ *+ * @param isWriteLock {@code true} if requesting a write lock, {@code false} for a read lock.+ * @param configSetName the config set name, can never be {@code null}.+ * @return a lock instance that must be {@link DistributedLock#release()}'ed in a {@code finally},+ * regardless of the lock having been acquired or not.+ */+ DistributedLock createLock(boolean isWriteLock, String configSetName);+}diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.javanew file mode 100644index 00000000000..de618e7618d--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedLock.java@@ -0,0 +1,27 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++/**+ * A lock that can be used across cluster nodes.+ */+public interface DistributedLock {+ void waitUntilAcquired();+ void release();+ boolean isAcquired();+}diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.javanew file mode 100644index 00000000000..d0fe596e5f5--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMultiLock.java@@ -0,0 +1,76 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.lang.invoke.MethodHandles;+import java.util.List;++import com.google.common.annotations.VisibleForTesting;+import org.apache.solr.common.SolrException;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * A lock as acquired for running a single API command (Collection or Config Set or anything else in the future).+ * Internally it is composed of multiple {@link DistributedLock}'s.+ */+public class DistributedMultiLock {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ private final List<DistributedLock> locks;+ private volatile boolean isReleased = false;++ public DistributedMultiLock(List<DistributedLock> locks) {+ this.locks = locks;+ }++ public void waitUntilAcquired() {+ if (isReleased) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Released lock can't be waited upon");+ }++ for (DistributedLock lock : locks) {+ log.debug("DistributedMultiLock.waitUntilAcquired. About to wait on lock {}", lock);+ lock.waitUntilAcquired();+ log.debug("DistributedMultiLock.waitUntilAcquired. Acquired lock {}", lock);+ }+ }++ public void release() {+ isReleased = true;+ for (DistributedLock lock : locks) {+ lock.release();+ }+ }++ public boolean isAcquired() {+ if (isReleased) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Released lock can't be tested");+ }+ for (DistributedLock lock : locks) {+ if (!lock.isAcquired()) {+ return false;+ }+ }+ return true;+ }++ @VisibleForTesting+ public int getCountInternalLocks() {+ return locks.size();+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.javaindex 3a6434e03d7..2596b296ed5 100644--- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java@@ -16,36 +16,22 @@*/package org.apache.solr.cloud;-import java.io.ByteArrayInputStream;-import java.io.IOException;-import java.io.InputStreamReader;import java.lang.invoke.MethodHandles;-import java.nio.charset.StandardCharsets;-import java.util.HashMap;import java.util.HashSet;-import java.util.Iterator;-import java.util.Map;import java.util.Set;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;-import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ConfigSetParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.core.ConfigSetProperties;-import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;-import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import static org.apache.solr.common.params.CommonParams.NAME;-import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;-import static org.apache.solr.common.util.Utils.toJSONString;-import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;/*** A {@link OverseerMessageHandler} that handles ConfigSets API related@@ -58,16 +44,6 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {*/public static final String CONFIGSETS_ACTION_PREFIX = "configsets:";- /**- * Name of the ConfigSet to copy from for CREATE- */- public static final String BASE_CONFIGSET = "baseConfigSet";-- /**- * Prefix for properties that should be applied to the ConfigSet for CREATE- */- public static final String PROPERTY_PREFIX = "configSetProp";-private ZkStateReader zkStateReader;private CoreContainer coreContainer;@@ -112,10 +88,10 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {}switch (action) {case CREATE:- createConfigSet(message);+ ConfigSetCmds.createConfigSet(message, coreContainer);break;case DELETE:- deleteConfigSet(message);+ ConfigSetCmds.deleteConfigSet(message, coreContainer);break;default:throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"@@ -213,160 +189,12 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {private String getBaseConfigSetIfCreate(ZkNodeProps message) {- String operation = message.getStr(Overseer.QUEUE_OPERATION);- if (operation != null) {- operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());- ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);- if (action == CREATE) {- String baseConfigSetName = message.getStr(BASE_CONFIGSET);- if (baseConfigSetName == null || baseConfigSetName.length() == 0) {- baseConfigSetName = DEFAULT_CONFIGSET_NAME;- }- return baseConfigSetName;- }- }- return null;- }-- @SuppressWarnings({"rawtypes"})- private NamedList getConfigSetProperties(ConfigSetService configSetService, String configName, String propertyPath) throws IOException {- byte[] oldPropsData = configSetService.downloadFileFromConfig(configName, propertyPath);- if (oldPropsData != null) {- InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);- try {- return ConfigSetProperties.readFromInputStream(reader);- } finally {- reader.close();- }- }- return null;- }-- private Map<String, Object> getNewProperties(ZkNodeProps message) {- Map<String, Object> properties = null;- for (Map.Entry<String, Object> entry : message.getProperties().entrySet()) {- if (entry.getKey().startsWith(PROPERTY_PREFIX + ".")) {- if (properties == null) {- properties = new HashMap<String, Object>();- }- properties.put(entry.getKey().substring((PROPERTY_PREFIX + ".").length()),- entry.getValue());- }- }- return properties;- }-- private void mergeOldProperties(Map<String, Object> newProps, @SuppressWarnings({"rawtypes"})NamedList oldProps) {- @SuppressWarnings({"unchecked"})- Iterator<Map.Entry<String, Object>> it = oldProps.iterator();- while (it.hasNext()) {- Map.Entry<String, Object> oldEntry = it.next();- if (!newProps.containsKey(oldEntry.getKey())) {- newProps.put(oldEntry.getKey(), oldEntry.getValue());- }- }- }-- private byte[] getPropertyData(Map<String, Object> newProps) {- if (newProps != null) {- String propertyDataStr = toJSONString(newProps);- if (propertyDataStr == null) {- throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid property specification");- }- return propertyDataStr.getBytes(StandardCharsets.UTF_8);- }- return null;- }-- private void createConfigSet(ZkNodeProps message) throws IOException {- String configSetName = getTaskKey(message);- if (configSetName == null || configSetName.length() == 0) {- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");- }-- String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);-- if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);- }-- // is there a base config that already exists- if (!coreContainer.getConfigSetService().checkConfigExists(baseConfigSetName)) {- throw new SolrException(ErrorCode.BAD_REQUEST,- "Base ConfigSet does not exist: " + baseConfigSetName);- }-- String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;- Map<String, Object> props = getNewProperties(message);- if (props != null) {- // read the old config properties and do a merge, if necessary- @SuppressWarnings({"rawtypes"})- NamedList oldProps = getConfigSetProperties(coreContainer.getConfigSetService(), baseConfigSetName, propertyPath);- if (oldProps != null) {- mergeOldProperties(props, oldProps);- }- }- byte[] propertyData = getPropertyData(props);-- try {- coreContainer.getConfigSetService().copyConfig(baseConfigSetName, configSetName);- if (propertyData != null) {- coreContainer.getConfigSetService().uploadFileToConfig(configSetName, propertyPath, propertyData, true);- }- } catch (Exception e) {- // copying the config dir or writing the properties file may have failed.- // we should delete the ConfigSet because it may be invalid,- // assuming we actually wrote something. E.g. could be- // the entire baseConfig set with the old properties, including immutable,- // that would make it impossible for the user to delete.- try {- if (coreContainer.getConfigSetService().checkConfigExists(configSetName)) {- deleteConfigSet(configSetName, true);- }- } catch (IOException ioe) {- log.error("Error while trying to delete partially created ConfigSet", ioe);- }- throw e;- }+ String operation = message.getStr(Overseer.QUEUE_OPERATION).substring(CONFIGSETS_ACTION_PREFIX.length());+ ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);+ return ConfigSetCmds.getBaseConfigSetName(action, message.getStr(ConfigSetCmds.BASE_CONFIGSET));}- private void deleteConfigSet(ZkNodeProps message) throws IOException {- String configSetName = getTaskKey(message);- if (configSetName == null || configSetName.length() == 0) {- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");- }- deleteConfigSet(configSetName, false);- }- private void deleteConfigSet(String configSetName, boolean force) throws IOException {- if (!coreContainer.getConfigSetService().checkConfigExists(configSetName)) {- throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);- }- for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {- String configName = null;- try {- configName = zkStateReader.readConfigName(entry.getKey());- } catch (KeeperException ex) {- throw new SolrException(ErrorCode.BAD_REQUEST,- "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");- }- if (configSetName.equals(configName))- throw new SolrException(ErrorCode.BAD_REQUEST,- "Can not delete ConfigSet as it is currently being used by collection [" + entry.getKey() + "]");- }-- String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;- @SuppressWarnings({"rawtypes"})- NamedList properties = getConfigSetProperties(coreContainer.getConfigSetService(), configSetName, propertyPath);- if (properties != null) {- Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);- boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;- if (!force && isImmutableConfigSet) {- throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);- }- }- coreContainer.getConfigSetService().deleteConfig(configSetName);- }}diff --git a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.javaindex 0cb6cbe8e87..853db094f26 100644--- a/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java+++ b/solr/core/src/java/org/apache/solr/cloud/SizeLimitedDistributedMap.java@@ -17,6 +17,9 @@package org.apache.solr.cloud;import java.util.List;+import java.util.Map;++import com.google.common.collect.Maps;import org.apache.lucene.util.PriorityQueue;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.zookeeper.KeeperException;@@ -50,6 +53,17 @@ public class SizeLimitedDistributedMap extends DistributedMap {@Overridepublic void put(String trackingId, byte[] data) throws KeeperException, InterruptedException {+ shrinkIfNeeded();+ super.put(trackingId, data);+ }++ @Override+ public boolean putIfAbsent(String trackingId, byte[] data) throws KeeperException, InterruptedException {+ shrinkIfNeeded();+ return super.putIfAbsent(trackingId, data);+ }++ private void shrinkIfNeeded() throws KeeperException, InterruptedException {if (this.size() >= maxSize) {// Bring down the sizeList<String> children = zookeeper.getChildren(dir, null, true);@@ -63,23 +77,25 @@ public class SizeLimitedDistributedMap extends DistributedMap {}};+ Map<String, Long> childToModificationZxid = Maps.newHashMapWithExpectedSize(children.size());for (String child : children) {Stat stat = zookeeper.exists(dir + "/" + child, null, true);- priorityQueue.insertWithOverflow(stat.getMzxid());+ if (stat != null) {+ priorityQueue.insertWithOverflow(stat.getMzxid());+ childToModificationZxid.put(child, stat.getMzxid());+ }}- long topElementMzxId = priorityQueue.top();+ long topElementMzxId = priorityQueue.top(); // can be null, but highly unlikelyfor (String child : children) {- Stat stat = zookeeper.exists(dir + "/" + child, null, true);- if (stat.getMzxid() <= topElementMzxId) {+ Long id = childToModificationZxid.get(child);+ if (id != null && id <= topElementMzxId) {zookeeper.delete(dir + "/" + child, -1, true);if (onOverflowObserver != null) onOverflowObserver.onChildDelete(child.substring(PREFIX.length()));}}}-- super.put(trackingId, data);}interface OnOverflowObserver {diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.javanew file mode 100644index 00000000000..d3d00cb5436--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedCollectionLockFactory.java@@ -0,0 +1,100 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import com.google.common.base.Preconditions;+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.params.CollectionParams;++/**+ * A distributed lock implementation using Zookeeper "directory" nodes created within a collection znode hierarchy, for+ * use with the distributed Collection API implementation.+ * The locks are implemented using ephemeral nodes placed below the "directory" nodes.+ *+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>+ */+public class ZkDistributedCollectionLockFactory extends ZkDistributedLockFactory implements DistributedCollectionLockFactory {++ public ZkDistributedCollectionLockFactory(SolrZkClient zkClient, String rootPath) {+ super(zkClient, rootPath);+ }++ public DistributedLock createLock(boolean isWriteLock, CollectionParams.LockLevel level, String collName, String shardId,+ String replicaName) {+ Preconditions.checkArgument(collName != null, "collName can't be null");+ Preconditions.checkArgument(level == CollectionParams.LockLevel.COLLECTION || shardId != null,+ "shardId can't be null when getting lock for shard or replica");+ Preconditions.checkArgument(level != CollectionParams.LockLevel.REPLICA || replicaName != null,+ "replicaName can't be null when getting lock for replica");++ String lockPath = getLockPath(level, collName, shardId, replicaName);+ return doCreateLock(isWriteLock, lockPath);+ }++ /**+ * Returns the Zookeeper path to the lock, creating missing nodes if needed.+ * Note that the complete lock hierarchy (/locks and below) can be deleted if SolrCloud is stopped.+ *+ * The tree of lock directories for a given collection {@code collName} is as follows:+ * <pre>+ * rootPath/+ * collName/+ * Locks <-- EPHEMERAL collection level locks go here+ * _ShardName1/+ * Locks <-- EPHEMERAL shard level locks go here+ * _replicaNameS1R1 <-- EPHEMERAL replica level locks go here+ * _replicaNameS1R2 <-- EPHEMERAL replica level locks go here+ * _ShardName2/+ * Locks <-- EPHEMERAL shard level locks go here+ * _replicaNameS2R1 <-- EPHEMERAL replica level locks go here+ * _replicaNameS2R2 <-- EPHEMERAL replica level locks go here+ * </pre>+ * This method will create the path where the {@code EPHEMERAL} lock nodes should go. That path is:+ * <ul>+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#COLLECTION} -+ * {@code /locks/collections/collName/Locks}</li>+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#SHARD} -+ * {@code /locks/collections/collName/_shardName/Locks}</li>+ * <li>For {@link org.apache.solr.common.params.CollectionParams.LockLevel#REPLICA} -+ * {@code /locks/collections/collName/_shardName/_replicaName}. There is no {@code Locks} subnode here because replicas do+ * not have children so no need to separate {@code EPHEMERAL} lock nodes from children nodes as is the case for shards+ * and collections</li>+ * </ul>+ * Note the {@code _} prefixing shards and replica names is to support shards or replicas called "{@code Locks}".+ * Also note the returned path does not contain the separator ({@code "/"}) at the end.+ */+ private String getLockPath(CollectionParams.LockLevel level, String collName, String shardId, String replicaName) {+ StringBuilder path = getPathPrefix();+ path.append(collName).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR);++ final String LOCK_NODENAME = "Locks"; // Should not start with SUBNODE_PREFIX :)+ final String SUBNODE_PREFIX = "_";++ if (level == CollectionParams.LockLevel.COLLECTION) {+ return path.append(LOCK_NODENAME).toString();+ } else if (level == CollectionParams.LockLevel.SHARD) {+ return path.append(SUBNODE_PREFIX).append(shardId).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR).append(LOCK_NODENAME).toString();+ } else if (level == CollectionParams.LockLevel.REPLICA) {+ return path.append(SUBNODE_PREFIX).append(shardId).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR).append(SUBNODE_PREFIX).append(replicaName).toString();+ } else {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported lock level " + level);+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.javanew file mode 100644index 00000000000..b1386a7b88b--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedConfigSetLockFactory.java@@ -0,0 +1,60 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import com.google.common.base.Preconditions;+import org.apache.solr.common.cloud.SolrZkClient;++/**+ * A distributed lock implementation using Zookeeper "directory" nodes created within a collection znode hierarchy, for+ * use with the distributed Collection API implementation.+ * The locks are implemented using ephemeral nodes placed below the "directory" nodes.+ *+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>+ */+public class ZkDistributedConfigSetLockFactory extends ZkDistributedLockFactory implements DistributedConfigSetLockFactory {++ public ZkDistributedConfigSetLockFactory(SolrZkClient zkClient, String rootPath) {+ super(zkClient, rootPath);+ }++ public DistributedLock createLock(boolean isWriteLock, String configSetName) {+ Preconditions.checkArgument(configSetName != null, "configSetName can't be null");++ String lockPath = getLockPath(configSetName);+ return doCreateLock(isWriteLock, lockPath);+ }++ /**+ * Returns the Zookeeper path to the lock, creating missing nodes if needed.+ * Note that the complete lock hierarchy (/rootPath and below) can be deleted if SolrCloud is stopped.+ *+ * The tree of lock directories is very flat, given there's no real structure to what's being locked in a config set:+ * <pre>+ * rootPath/+ * configSet1/ <-- EPHEMERAL config set locks go here+ * configSet2/ <-- EPHEMERAL config set locks go here+ * etc...+ * </pre>+ * This method will create the path where the {@code EPHEMERAL} lock nodes should go.<p>+ * The returned path does not contain the separator ({@code "/"}) at the end.+ */+ private String getLockPath(String configSetName) {+ return getPathPrefix().append(configSetName).toString();+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.javanew file mode 100644index 00000000000..6094929fa49--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLock.java@@ -0,0 +1,234 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.util.List;+import java.util.concurrent.CountDownLatch;++import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException;+import org.apache.zookeeper.WatchedEvent;+import org.apache.zookeeper.Watcher;++import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;++/**+ * A lock to be used across cluster nodes based on Zookeeper. Used for the distributed Collection API and distributed+ * Config Set API implementations.+ * @see <a href="https://zookeeper.apache.org/doc/current/recipes.html#sc_recipes_Locks">Zookeeper lock recipe</a>+ */+abstract class ZkDistributedLock implements DistributedLock {+ /**+ * End of the lock node name prefix before the sequential part+ */+ static final char LOCK_PREFIX_SUFFIX = '_';+ /**+ * Prefix of EPHEMERAL read lock node names+ */+ static final String READ_LOCK_PREFIX = "R" + LOCK_PREFIX_SUFFIX;+ /**+ * Prefix of EPHEMERAL write lock node names+ */+ static final String WRITE_LOCK_PREFIX = "W" + LOCK_PREFIX_SUFFIX;++ /**+ * Read lock.+ */+ static class Read extends ZkDistributedLock {+ protected Read(SolrZkClient zkClient, String lockPath) throws KeeperException, InterruptedException {+ super(zkClient, lockPath, READ_LOCK_PREFIX);+ }++ @Override+ boolean isBlockedByNodeType(String otherLockName) {+ // A read lock is only blocked by a lower numbered write lock+ // Lower numbered read locks are ok, they can coexist.+ return otherLockName.startsWith(WRITE_LOCK_PREFIX);+ }+ }++ /**+ * Write lock.+ */+ static class Write extends ZkDistributedLock {+ protected Write(SolrZkClient zkClient, String lockPath) throws KeeperException, InterruptedException {+ super(zkClient, lockPath, WRITE_LOCK_PREFIX);+ }++ @Override+ boolean isBlockedByNodeType(String otherLockName) {+ // A write lock is blocked by another read or write lock with a lower sequence number+ return true;+ }+ }++ private final SolrZkClient zkClient;+ private final String lockDir;+ private final String lockNode;+ protected final long sequence;+ protected volatile boolean released = false;++ protected ZkDistributedLock(SolrZkClient zkClient, String lockDir, String lockNodePrefix) throws KeeperException, InterruptedException {+ this.zkClient = zkClient;+ this.lockDir = lockDir;++ // Create the SEQUENTIAL EPHEMERAL node. We enter the locking rat race here. We MUST eventually call release() or we block others.+ lockNode = zkClient.create(lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + lockNodePrefix, null,+ CreateMode.EPHEMERAL_SEQUENTIAL, false);++ sequence = getSequenceFromNodename(lockNode);+ }++ private static class DeletedNodeWatcher implements Watcher {+ final CountDownLatch latch;+ final String nodeBeingWatched;+ String errorMessage = null; // non null means error++ DeletedNodeWatcher(String nodeBeingWatched) {+ this.latch = new CountDownLatch(1);+ this.nodeBeingWatched = nodeBeingWatched;+ }++ @Override+ public void process(WatchedEvent event) {+ if (event.getType() == Event.EventType.None) {+ return;+ } else if (event.getType() != Event.EventType.NodeDeleted) {+ synchronized (this) {+ errorMessage = "Received unexpected watch event " + event.getType() + " on " + nodeBeingWatched;+ }+ }+ latch.countDown();+ }++ void await() throws InterruptedException {+ latch.await();+ synchronized (this) {+ if (errorMessage != null) {+ throw new SolrException(SERVER_ERROR, errorMessage);+ }+ }+ }+ }++ @Override+ public void waitUntilAcquired() {+ try {+ if (released) {+ throw new IllegalStateException("Bug. waitUntilAcquired() should not be called after release(). " + lockNode);+ }+ String nodeToWatch = nodeToWatch();+ while (nodeToWatch != null) {+ final DeletedNodeWatcher watcher = new DeletedNodeWatcher(nodeToWatch);+ if (zkClient.exists(nodeToWatch, watcher, true) != null) {+ watcher.await();+ }+ nodeToWatch = nodeToWatch();+ }+ } catch (KeeperException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }++ @Override+ public void release() {+ try {+ zkClient.delete(lockNode, -1, true);+ released = true;+ } catch (KeeperException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }++ @Override+ public boolean isAcquired() {+ try {+ if (released) {+ throw new IllegalStateException("Bug. isAcquired() should not be called after release(). " + lockNode);+ }+ return nodeToWatch() == null;+ } catch (KeeperException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }++ /**+ * @return Another lock node (complete path) that must go away for us to acquire the lock, or {@code null} if the lock is ours.+ */+ String nodeToWatch() throws KeeperException, InterruptedException {+ List<String> locks = zkClient.getChildren(lockDir, null, true);+ boolean foundSelf = false; // For finding bugs or ZK bad behavior+ // We deviate from the ZK recipe here: we do not sort the list of nodes, and we stop waiting on the first one we find+ // that blocks us. This is done in O(n), whereas sorting is more expensive. And we iterate only once over the set of children.+ // Might cause more wakeups than needed though (multiple locks watching the same one rather than one another) but lock+ // contention expected low (how many concurrent and conflicting Collection API requests a reasonable user can issue?)+ // and the code below is simpler. Changing to a sorted "optimal" approach implies only changes in this method.+ for (String lock : locks) {+ long seq = getSequenceFromNodename(lock);+ if (seq == sequence) {+ foundSelf = true;+ } else if (seq < sequence && isBlockedByNodeType(lock)) {+ // Return the full path to the node to watch+ return lockDir + DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR + lock;+ }+ // seq is bigger than sequence, can't block us. If the iteration was sorted we could avoid iterating over those.+ }+ if (!foundSelf) {+ // If this basic assumption doesn't hold with Zookeeper, we're in deep trouble. And not only here.+ throw new SolrException(SERVER_ERROR, "Missing lock node " + lockNode);+ }++ // Didn't return early on any other blocking lock, means we own it+ return null;+ }++ /**+ * @param otherLockName name of a competing lock node. Used to find out the type of the lock (read or write)+ * @return {@code true} if a lock node of the given type with a lower sequence number blocks us from acquiring the lock.+ */+ abstract boolean isBlockedByNodeType(String otherLockName);++ static long getSequenceFromNodename(String lockNode) {+ // Javadoc of ZooKeeper.create() specifies "The sequence number is always fixed length of 10 digits, 0 padded"+ // for sequential nodes+ final int SEQUENCE_LENGTH = 10;+ // Before the sequence we have our specific char+ if (lockNode.charAt(lockNode.length() - SEQUENCE_LENGTH - 1) != LOCK_PREFIX_SUFFIX) {+ throw new SolrException(SERVER_ERROR, "Unexpected lock node path " + lockNode);+ }++ return Long.parseLong(lockNode.substring(lockNode.length() - SEQUENCE_LENGTH));+ }++ @Override+ public String toString() {+ return lockNode;+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.javanew file mode 100644index 00000000000..4ddffc368f5--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedLockFactory.java@@ -0,0 +1,66 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.zookeeper.CreateMode;+import org.apache.zookeeper.KeeperException;++abstract class ZkDistributedLockFactory {++ private final SolrZkClient zkClient;+ private final String rootPath;++ ZkDistributedLockFactory(SolrZkClient zkClient, String rootPath) {+ this.zkClient = zkClient;+ this.rootPath = rootPath;+ }++ protected DistributedLock doCreateLock(boolean isWriteLock, String lockPath) {+ try {+ // TODO optimize by first attempting to create the ZkDistributedLock without calling makeLockPath() and only call it+ // if the lock creation fails. This will be less costly on high contention (and slightly more on low contention)+ makeLockPath(lockPath);++ return isWriteLock ? new ZkDistributedLock.Write(zkClient, lockPath) : new ZkDistributedLock.Read(zkClient, lockPath);+ } catch (KeeperException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);+ }+ }++ protected StringBuilder getPathPrefix() {+ StringBuilder path = new StringBuilder(100);+ path.append(rootPath).append(DistributedCollectionConfigSetCommandRunner.ZK_PATH_SEPARATOR);+ return path;+ }++ private void makeLockPath(String lockNodePath) throws KeeperException, InterruptedException {+ try {+ if (!zkClient.exists(lockNodePath, true)) {+ zkClient.makePath(lockNodePath, new byte[0], CreateMode.PERSISTENT, true);+ }+ } catch (KeeperException.NodeExistsException nee) {+ // Some other thread (on this or another JVM) beat us to create the node, that's ok.+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.javaindex 8acb7024c09..164ee63c4e6 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollApiCmds.java@@ -24,9 +24,11 @@ import java.util.Map;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;+import com.google.common.collect.ImmutableMap;import org.apache.commons.lang3.StringUtils;import org.apache.solr.cloud.DistributedClusterStateUpdater;import org.apache.solr.cloud.Overseer;+import org.apache.solr.cloud.OverseerNodePrioritizer;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.Replica;@@ -35,6 +37,7 @@ import org.apache.solr.common.cloud.UrlScheme;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionAdminParams;+import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;@@ -55,16 +58,15 @@ import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;+import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonParams.NAME;/**- * This class contains "smaller" Collection API commands implementation as well as the interface implemented by all commands.- * Previously these implementations in {@link OverseerCollectionMessageHandler} were relying on methods implementing the- * functional interface.+ * This class contains "smaller" Collection API commands implementation, the interface implemented by all commands and the+ * class mapping a collection action to the actual command.+ * Previously the "smaller" command implementations in {@link OverseerCollectionMessageHandler} were relying on methods+ * implementing the functional interface.*/public class CollApiCmds {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@@ -76,6 +78,75 @@ public class CollApiCmds {void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"}) NamedList results) throws Exception;}+ /**+ * Map {@link org.apache.solr.common.params.CollectionParams.CollectionAction} to instances of {@link CollApiCmds.CollectionApiCommand} and+ * being usable by both {@link OverseerCollectionMessageHandler} and {@link DistributedCollectionConfigSetCommandRunner} so that+ * the mappings do not have to be maintained in two places.+ */+ protected static class CommandMap {+ final private Map<CollectionParams.CollectionAction, CollApiCmds.CollectionApiCommand> commandMap;++ /**+ * Constructor used when Collection API is run on the Overseer. Called by {@link OverseerCollectionMessageHandler}+ */+ CommandMap(CollectionCommandContext ccc, OverseerNodePrioritizer overseerPrioritizer) {+ this(overseerPrioritizer, ccc);+ assert !ccc.isDistributedCollectionAPI();+ }++ /**+ * Constructor used when Collection API execution is distributed+ */+ CommandMap(CollectionCommandContext ccc) {+ // Overseer prioritizer will not be used. Running in distributed Collection API mode+ this(null, ccc);+ assert ccc.isDistributedCollectionAPI();+ }++ private CommandMap(OverseerNodePrioritizer overseerPrioritizer, CollectionCommandContext ccc) {+ commandMap = new ImmutableMap.Builder<CollectionParams.CollectionAction, CollApiCmds.CollectionApiCommand>()+ .put(REPLACENODE, new ReplaceNodeCmd(ccc))+ .put(DELETENODE, new DeleteNodeCmd(ccc))+ .put(BACKUP, new BackupCmd(ccc))+ .put(RESTORE, new RestoreCmd(ccc))+ .put(DELETEBACKUP, new DeleteBackupCmd(ccc))+ .put(CREATESNAPSHOT, new CreateSnapshotCmd(ccc))+ .put(DELETESNAPSHOT, new DeleteSnapshotCmd(ccc))+ .put(SPLITSHARD, new SplitShardCmd(ccc))+ .put(ADDROLE, new OverseerRoleCmd(ccc, ADDROLE, overseerPrioritizer))+ .put(REMOVEROLE, new OverseerRoleCmd(ccc, REMOVEROLE, overseerPrioritizer))+ .put(MOCK_COLL_TASK, new CollApiCmds.MockOperationCmd())+ .put(MOCK_SHARD_TASK, new CollApiCmds.MockOperationCmd())+ .put(MOCK_REPLICA_TASK, new CollApiCmds.MockOperationCmd())+ .put(CREATESHARD, new CreateShardCmd(ccc))+ .put(MIGRATE, new MigrateCmd(ccc))+ .put(CREATE, new CreateCollectionCmd(ccc))+ .put(MODIFYCOLLECTION, new CollApiCmds.ModifyCollectionCmd(ccc))+ .put(ADDREPLICAPROP, new CollApiCmds.AddReplicaPropCmd(ccc))+ .put(DELETEREPLICAPROP, new CollApiCmds.DeleteReplicaPropCmd(ccc))+ .put(BALANCESHARDUNIQUE, new CollApiCmds.BalanceShardsUniqueCmd(ccc))+ .put(REBALANCELEADERS, new CollApiCmds.RebalanceLeadersCmd(ccc))+ .put(RELOAD, new CollApiCmds.ReloadCollectionCmd(ccc))+ .put(DELETE, new DeleteCollectionCmd(ccc))+ .put(CREATEALIAS, new CreateAliasCmd(ccc))+ .put(DELETEALIAS, new DeleteAliasCmd(ccc))+ .put(ALIASPROP, new SetAliasPropCmd(ccc))+ .put(MAINTAINROUTEDALIAS, new MaintainRoutedAliasCmd(ccc))+ .put(OVERSEERSTATUS, new OverseerStatusCmd(ccc))+ .put(DELETESHARD, new DeleteShardCmd(ccc))+ .put(DELETEREPLICA, new DeleteReplicaCmd(ccc))+ .put(ADDREPLICA, new AddReplicaCmd(ccc))+ .put(MOVEREPLICA, new MoveReplicaCmd(ccc))+ .put(REINDEXCOLLECTION, new ReindexCollectionCmd(ccc))+ .put(RENAME, new RenameCmd(ccc))+ .build();+ }++ CollApiCmds.CollectionApiCommand getActionCommand(CollectionParams.CollectionAction action) {+ return commandMap.get(action);+ }+ }+static public class MockOperationCmd implements CollectionApiCommand {@SuppressForbidden(reason = "Needs currentTimeMillis for mock requests")@SuppressWarnings({"unchecked"})diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.javanew file mode 100644index 00000000000..70c7cf785bd--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionApiLockFactory.java@@ -0,0 +1,111 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud.api.collections;++import java.util.ArrayList;+import java.util.List;++import org.apache.solr.cloud.DistributedCollectionLockFactory;+import org.apache.solr.cloud.DistributedLock;+import org.apache.solr.cloud.DistributedMultiLock;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.params.CollectionParams;++/**+ * This class implements a higher level locking abstraction for the Collection API using lower level read and write locks.+ */+public class CollectionApiLockFactory {++ private final DistributedCollectionLockFactory lockFactory;++ CollectionApiLockFactory(DistributedCollectionLockFactory lockFactory) {+ this.lockFactory = lockFactory;+ }++ /**+ * For the {@link org.apache.solr.common.params.CollectionParams.LockLevel} of the passed {@code action}, obtains the+ * required locks (if any) and returns.<p>+ *+ * This method obtains a write lock at the actual level and path of the action, and also obtains read locks on "lower"+ * lock levels. For example for a lock at the shard level, a write lock will be requested at the corresponding shard path+ * and a read lock on the corresponding collection path (in order to prevent an operation locking at the collection level+ * from executing concurrently with an operation on one of the shards of the collection).+ * See documentation linked to SOLR-14840 regarding Collection API locking.+ *+ * @return a lock that once {@link DistributedMultiLock#isAcquired()} guarantees the corresponding Collection+ * API command can execute safely.+ * The returned lock <b>MUST</b> be {@link DistributedMultiLock#release()} no matter what once no longer needed as otherwise it would+ * prevent other threads from locking.+ */+ DistributedMultiLock createCollectionApiLock(CollectionParams.LockLevel lockLevel, String collName, String shardId, String replicaName) {+ if (lockLevel == CollectionParams.LockLevel.NONE) {+ return new DistributedMultiLock(List.of());+ }++ if (lockLevel == CollectionParams.LockLevel.CLUSTER) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug. Not expecting locking at cluster level.");+ }++ if (collName == null) {+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug. collName can't be null");+ }++ // There is a bug in the way locks are managed in the Overseer, it sometimes locks at a given level (for example shard)+ // without knowing on which shard to lock. This can happen for example for SPLITSHARD using a split.key (details below)+ // or DELETEREPLICA when specifying a count of replicas to delete but no shard.+ //+ // When doing a splitshard using a split.key (and not specifying the actual shard name), when the OverseerCollectionMessageHandler+ // grabs a lock for the Collection API it does not have the shard name and the lock is obtained for a null key.+ // This means that this lock does not prevent another operation specifically targeting the shard from executing concurrently+ // (can even be another split if it is specified with a shard name). The null shard lock does prevent another split.key split+ // for another shard of the same collection at the same time (because trying to obtain another lock on the null key below the+ // collection will fail).+ // The splitshard request is built in CollectionAdminRequest.splitShard(). The case of concern is when SplitShard.setSplitKey()+ // is then called (see test CollectionsAPISolrJTest.testSplitShard()). CollectionOperation.SPLITSHARD_OP in CollectionsHandler+ // verifies that when the key is specified no shard name is specified.+ // The locking happens from OverseerTaskProcessor.run() calling OverseerCollectionMessageHandler.lockTask().+ // The ArrayList passed to LockTree.Session.lock() contains the collection name (index 0) but not shard name (value null+ // at index 1) yet the action.lockLevel is SHARD. Note that there ends up being a single shard name associated with the+ // split key. It is computed in SplitShardCmd.getParentSlice().+ //+ // We deal with the issue to trigger the same (wrong) behavior for compatibility of distributed Collection API vs Overseer+ // based collection API. We make up a shard name that is highly unlikely to otherwise exist and lock it instead.+ // Another option would be to bump up the locking level from SHARD to COLLECTION when no shard is specified (would be+ // "more correct" from the perspective of preventing competing Collection API commands from running concurrently, but+ // would restrict concurrency a bit more than existing solution, and if some installations depend on current behavior...).+ if (lockLevel == CollectionParams.LockLevel.SHARD && shardId == null) {+ shardId = "MadeUpShardNameWhenNoShardNameWasProvided";+ // Bumping locking level to collection would be instead: lockLevel = CollectionParams.LockLevel.COLLECTION;+ }++ // The first requested lock is a write one (on the target object for the action, depending on lock level), then requesting+ // read locks on "higher" levels (collection > shard > replica here for the level. Note LockLevel "height" is other way around).+ boolean requestWriteLock = true;+ final CollectionParams.LockLevel[] iterationOrder = { CollectionParams.LockLevel.REPLICA, CollectionParams.LockLevel.SHARD, CollectionParams.LockLevel.COLLECTION };+ List<DistributedLock> locks = new ArrayList<>(iterationOrder.length);+ for (CollectionParams.LockLevel level : iterationOrder) {+ // This comparison is based on the LockLevel height value that classifies replica > shard > collection.+ if (lockLevel.isHigherOrEqual(level)) {+ locks.add(lockFactory.createLock(requestWriteLock, level, collName, shardId, replicaName));+ requestWriteLock = false;+ }+ }++ return new DistributedMultiLock(locks);+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.javaindex b9fe9bd34d2..92a226b6448 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CollectionCommandContext.java@@ -37,6 +37,11 @@ import java.util.concurrent.ExecutorService;* distributed way, unrelated to and not depending upon Overseer abstractions (including overseer collection message handling).*/public interface CollectionCommandContext {+ /**+ * When this method returns {@code true}, Overseer specific actions do not make sense and commands should not be doing them.+ */+ boolean isDistributedCollectionAPI();+ShardHandler newShardHandler();SolrCloudManager getSolrCloudManager();@@ -61,11 +66,12 @@ public interface CollectionCommandContext {ExecutorService getExecutorService();/**- * This method enables the commands to enqueue to the overseer cluster state update. This should only be used when the command- * is running in the Overseer (and will throw an exception if called when Collection API is distributed)+ * This method enables the commands to enqueue to the overseer cluster state update. This should only be used when the+ * cluster state update is running in the Overseer (and will throw an exception if called when Collection API is distributed+ * given that distributed Collection API implies distributed Cluster State Update)*/default void offerStateUpdate(byte[] data) throws KeeperException, InterruptedException {- throw new IllegalStateException("Bug! offerStateUpdate() should not be called when distributed state updates are enabled");+ throw new IllegalStateException("Bug! offerStateUpdate() should not be called when distributed cluster state updates are enabled");}default String getOverseerId() {diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.javaindex f5e8c1e159b..4dab2e8295a 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java@@ -54,7 +54,6 @@ import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.common.util.TimeSource;import org.apache.solr.common.util.Utils;import org.apache.solr.core.ConfigSetService;import org.apache.solr.core.CoreContainer;@@ -83,13 +82,9 @@ import static org.apache.solr.handler.admin.ConfigSetsHandler.getSuffixedNameForpublic class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());private final CollectionCommandContext ccc;- private final TimeSource timeSource;- private final DistribStateManager stateManager;public CreateCollectionCmd(CollectionCommandContext ccc) {this.ccc = ccc;- this.stateManager = ccc.getSolrCloudManager().getDistribStateManager();- this.timeSource = ccc.getSolrCloudManager().getTimeSource();}@Override@@ -133,7 +128,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {ZkStateReader zkStateReader = ccc.getZkStateReader();// this also creates the collection zk node as a side-effect- CollectionHandlingUtils.createConfNode(stateManager, configName, collectionName);+ CollectionHandlingUtils.createConfNode(ccc.getSolrCloudManager().getDistribStateManager(), configName, collectionName);Map<String,String> collectionParams = new HashMap<>();Map<String,Object> collectionProps = message.getProperties();@@ -144,7 +139,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}}- createCollectionZkNode(stateManager, collectionName, collectionParams, ccc.getCoreContainer().getConfigSetService());+ createCollectionZkNode(ccc.getSolrCloudManager().getDistribStateManager(), collectionName, collectionParams, ccc.getCoreContainer().getConfigSetService());// Note that in code below there are two main execution paths: Overseer based cluster state updates and distributed// cluster state updates (look for isDistributedStateUpdate() conditions).@@ -169,6 +164,8 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {// When cluster state updates are handled by Overseer, ask it to load that collection it doesn't know about.// When cluster state updates are distributed, ZK is the source of truth for all nodes so no reload needed.if (!ccc.getDistributedClusterStateUpdater().isDistributedStateUpdate()) {+ // If cluster state update is not distributed and we execute here, the Collection API is not distributed either+ // and this execution happens on the Overseer node, so direct memory access as done below is ok.ccc.submitIntraProcessMessage(new RefreshCollectionMessage(collectionName));}} else {@@ -182,7 +179,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {}// wait for a while until we see the collection- TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, timeSource);+ TimeOut waitUntil = new TimeOut(30, TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource());boolean created = false;while (!waitUntil.hasTimedOut()) {waitUntil.sleep(100);@@ -341,7 +338,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {@SuppressWarnings({"rawtypes"})boolean failure = results.get("failure") != null && ((SimpleOrderedMap)results.get("failure")).size() > 0;if (isPRS) {- TimeOut timeout = new TimeOut(Integer.getInteger("solr.waitToSeeReplicasInStateTimeoutSeconds", 120), TimeUnit.SECONDS, timeSource); // could be a big cluster+ TimeOut timeout = new TimeOut(Integer.getInteger("solr.waitToSeeReplicasInStateTimeoutSeconds", 120), TimeUnit.SECONDS, ccc.getSolrCloudManager().getTimeSource()); // could be a big clusterPerReplicaStates prs = PerReplicaStates.fetch(collectionPath, ccc.getZkStateReader().getZkClient(), null);while (!timeout.hasTimedOut()) {if(prs.allActive()) break;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.javaindex 6612deda1fb..df7895c06e7 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java@@ -175,7 +175,7 @@ public class DeleteCollectionCmd implements CollApiCmds.CollectionApiCommand {boolean configSetIsUsedByOtherCollection = false;// make sure the configSet is not shared with other collections- // Similar to what happens in: OverseerConfigSetMessageHandler::deleteConfigSet+ // Similar to what happens in: ConfigSetCmds::deleteConfigSetfor (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {String otherConfigSetName = null;try {diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.javanew file mode 100644index 00000000000..e63c5e067a2--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionCommandContext.java@@ -0,0 +1,87 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud.api.collections;++import java.util.concurrent.ExecutorService;++import org.apache.solr.client.solrj.cloud.SolrCloudManager;+import org.apache.solr.cloud.DistributedClusterStateUpdater;+import org.apache.solr.common.SolrCloseable;+import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.handler.component.ShardHandler;++public class DistributedCollectionCommandContext implements CollectionCommandContext {+ private final CoreContainer coreContainer;+ private final DistributedClusterStateUpdater getDistributedClusterStateUpdater;+ private final ExecutorService executorService;++ private final SolrCloudManager solrCloudManager;+ private final ZkStateReader zkStateReader;++ public DistributedCollectionCommandContext(CoreContainer coreContainer, ExecutorService executorService) {+ this.coreContainer = coreContainer;+ this.getDistributedClusterStateUpdater = new DistributedClusterStateUpdater(coreContainer.getConfig().getCloudConfig().getDistributedClusterStateUpdates());;+ this.executorService = executorService;++ solrCloudManager = this.coreContainer.getZkController().getSolrCloudManager();+ zkStateReader = this.coreContainer.getZkController().getZkStateReader();+ }++ @Override+ public boolean isDistributedCollectionAPI() {+ return true;+ }++ @Override+ public ShardHandler newShardHandler() {+ // This method builds a new shard handler! A given shard handler can't be reused because it can only serve a single thread.+ return this.coreContainer.getShardHandlerFactory().getShardHandler();+ }++ @Override+ public SolrCloudManager getSolrCloudManager() {+ return solrCloudManager;+ }++ @Override+ public CoreContainer getCoreContainer() {+ return this.coreContainer;+ }++ @Override+ public ZkStateReader getZkStateReader() {+ return zkStateReader;+ }++ @Override+ public DistributedClusterStateUpdater getDistributedClusterStateUpdater() {+ return this.getDistributedClusterStateUpdater;+ }++ @Override+ public SolrCloseable getCloseableToLatchOn() {+ // Debatable: SolrCloudManager instance is tracked for closing to interrupt some command execution. Could very well monitor something else.+ return getSolrCloudManager();+ }++ @Override+ public ExecutorService getExecutorService() {+ return this.executorService;+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.javanew file mode 100644index 00000000000..57ad7635370--- /dev/null+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DistributedCollectionConfigSetCommandRunner.java@@ -0,0 +1,471 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud.api.collections;++import java.io.IOException;+import java.lang.invoke.MethodHandles;+import java.util.Map;+import java.util.concurrent.Callable;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Future;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.SynchronousQueue;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;++import org.apache.solr.client.solrj.response.RequestStatusState;+import org.apache.solr.cloud.ConfigSetApiLockFactory;+import org.apache.solr.cloud.ConfigSetCmds;+import org.apache.solr.cloud.DistributedApiAsyncTracker;+import org.apache.solr.cloud.DistributedMultiLock;+import org.apache.solr.cloud.OverseerSolrResponse;+import org.apache.solr.cloud.ZkDistributedCollectionLockFactory;+import org.apache.solr.cloud.ZkDistributedConfigSetLockFactory;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.ZkNodeProps;+import org.apache.solr.common.params.CollectionParams;+import org.apache.solr.common.params.ConfigSetParams;+import org.apache.solr.common.params.CoreAdminParams;+import org.apache.solr.common.util.ExecutorUtil;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Pair;+import org.apache.solr.common.util.SimpleOrderedMap;+import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.handler.admin.ConfigSetsHandler;+import org.apache.solr.logging.MDCLoggingContext;+import org.apache.solr.response.SolrQueryResponse;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;+import static org.apache.solr.common.params.CommonParams.NAME;++/**+ * Class for execution Collection API and Config Set API commands in a distributed way, without going through Overseer and+ * {@link OverseerCollectionMessageHandler} or {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler}.<p>+ *+ * This class is only called when Collection and Config Set API calls are configured to be distributed, which implies+ * cluster state updates are distributed as well.+ */+public class DistributedCollectionConfigSetCommandRunner {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ public static final String ZK_PATH_SEPARATOR = "/";++ private static final String ZK_DISTRIBUTED_API_ROOT = "/distributedapi";++ /**+ * Zookeeper node below which the locking hierarchy is anchored+ */+ private static final String ZK_COLLECTION_LOCKS = ZK_DISTRIBUTED_API_ROOT + "/collectionlocks";+ private static final String ZK_CONFIG_SET_LOCKS = ZK_DISTRIBUTED_API_ROOT + "/configsetlocks";++ /**+ * Zookeeper node below which the async id (requestId) tracking is done (for async requests)+ */+ private static final String ZK_ASYNC_ROOT = ZK_DISTRIBUTED_API_ROOT + "/async";++ private final ExecutorService distributedCollectionApiExecutorService;+ /**+ * All Collection API commands are executed as if they are asynchronous to stick to the same behavior as the Overseer+ * based Collection API execution. The difference between sync and async is twofold:+ * 1. Non async execution does wait for a while to see if the command has completed and if so, returns success (and if+ * not, returns failure)+ * 2. There is no way to query the status of a non async command that is still running, or to kill it (short of shutting+ * down the node on which it's running). This is not the best design, but at this stage that's what happens with+ * Overseer based Collection API, so implementing the same.+ * <p>+ * The common aspects between sync and async is that actual command execution is given an infinite time to eventually+ * complete (or fail). Again this is not great, but that's how it is for now. Likely some cleanup required later (once+ * Overseer is actually remove? Like Solr 10 or 11?).+ */+ private final ExecutorService commandsExecutor;+ private final CoreContainer coreContainer;+ final private CollApiCmds.CommandMap commandMapper;+ private final CollectionCommandContext ccc;+ private final DistributedApiAsyncTracker asyncTaskTracker;++ private volatile boolean shuttingDown = false;++ public DistributedCollectionConfigSetCommandRunner(CoreContainer coreContainer) {+ this.coreContainer = coreContainer;++ if (log.isInfoEnabled()) {+ // Note is it hard to print a log when Collection API is handled by Overseer because Overseer is started regardless+ // of how Collection API is handled, so it doesn't really know...+ log.info("Creating DistributedCollectionConfigSetCommandRunner. Collection and ConfigSet APIs are running distributed (not Overseer based)");+ }++ // TODO we should look at how everything is getting closed when the node is shutdown.+ // But it seems that CollectionsHandler (that creates instances of this class) is not really closed, so maybe it doesn't matter?+ // With distributed Collection API execution, each node will have such an executor but given how thread pools work,+ // threads will only be created if needed (including the corePoolSize threads).+ distributedCollectionApiExecutorService = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 10, 0L, TimeUnit.MILLISECONDS,+ new SynchronousQueue<>(),+ new SolrNamedThreadFactory("DistributedCollectionApiExecutorService"));++ commandsExecutor = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 20, 0L, TimeUnit.MILLISECONDS,+ new SynchronousQueue<>(),+ new SolrNamedThreadFactory("DistributedCollectionApiCommandExecutor"));++ ccc = new DistributedCollectionCommandContext(this.coreContainer, this.distributedCollectionApiExecutorService);+ commandMapper = new CollApiCmds.CommandMap(ccc);+ asyncTaskTracker = new DistributedApiAsyncTracker(ccc.getZkStateReader().getZkClient(), ZK_ASYNC_ROOT);+ }++ /**+ * See {@link DistributedApiAsyncTracker#getAsyncTaskRequestStatus(String)}+ */+ public Pair<RequestStatusState, OverseerSolrResponse> getAsyncTaskRequestStatus(String asyncId) throws Exception {+ return asyncTaskTracker.getAsyncTaskRequestStatus(asyncId);+ }++ /**+ * See {@link DistributedApiAsyncTracker#deleteSingleAsyncId(String)}+ */+ public boolean deleteSingleAsyncId(String asyncId) throws Exception {+ return asyncTaskTracker.deleteSingleAsyncId(asyncId);+ }++ /**+ * See {@link DistributedApiAsyncTracker#deleteAllAsyncIds()}+ */+ public void deleteAllAsyncIds() throws Exception {+ asyncTaskTracker.deleteAllAsyncIds();+ }+++ /**+ * When {@link org.apache.solr.handler.admin.CollectionsHandler#invokeAction} does not enqueue to overseer queue and+ * instead calls this method, this method is expected to do the equivalent of what Overseer does in+ * {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler#processMessage}.+ * <p>+ * The steps leading to that call in the Overseer execution path are (and the equivalent is done here):+ * <ul>+ * <li>{@link org.apache.solr.cloud.OverseerTaskProcessor#run()} gets the message from the ZK queue, grabs the+ * corresponding locks (write lock on the config set target of the API command and a read lock on the base config set+ * if any - the case for config set creation) then executes the command using an executor service (it also checks the+ * asyncId if any is specified but async calls are not supported for Config Set API calls).</li>+ * <li>In {@link org.apache.solr.cloud.OverseerTaskProcessor}.{@code Runner.run()} (run on an executor thread) a call is made to+ * {@link org.apache.solr.cloud.OverseerConfigSetMessageHandler#processMessage} which does a few checks and calls the+ * appropriate Config Set method.+ * </ul>+ */+ public void runConfigSetCommand(SolrQueryResponse rsp, ConfigSetsHandler.ConfigSetOperation operation, Map<String, Object> result, long timeoutMs) throws Exception {+ // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it through this method earlier).+ // See stopAndWaitForPendingTasksToComplete() below+ if (shuttingDown) {+ throw new SolrException(SolrException.ErrorCode.CONFLICT, "Solr is shutting down, no more Config Set API tasks may be executed");+ }++ ConfigSetParams.ConfigSetAction action = operation.getAction();++ // never null+ String configSetName = (String) result.get(NAME);+ // baseConfigSetName will be null if we're not creating a new config set+ String baseConfigSetName = ConfigSetCmds.getBaseConfigSetName(action, (String) result.get(ConfigSetCmds.BASE_CONFIGSET));++ if (log.isInfoEnabled()) {+ log.info("Running Config Set API locally for " + action + " " + configSetName); // nowarn+ }++ ConfigSetCommandRunner commandRunner = new ConfigSetCommandRunner(new ZkNodeProps(result), action, configSetName, baseConfigSetName);+ final Future<Void> taskFuture;+ try {+ taskFuture = commandsExecutor.submit(commandRunner);+ } catch (RejectedExecutionException ree) {+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree);+ }++ // Wait for a while... Just like Overseer based Config Set API (wait can timeout but actual command execution does not)+ try {+ taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS);+ } catch (TimeoutException te) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " " + configSetName + " timed out after " + timeoutMs + "ms");+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " " + configSetName + " interrupted", e);+ }+ }++ /**+ * When {@link org.apache.solr.handler.admin.CollectionsHandler#invokeAction} does not enqueue to overseer queue and+ * instead calls this method, this method is expected to do the equivalent of what Overseer does in+ * {@link org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler#processMessage}.+ * <p>+ * The steps leading to that call in the Overseer execution path are (and the equivalent is done here):+ * <ul>+ * <li>{@link org.apache.solr.cloud.OverseerTaskProcessor#run()} gets the message from the ZK queue, grabs the+ * corresponding lock (Collection API calls do locking to prevent non compatible concurrent modifications of a collection),+ * marks the async id of the task as running then executes the command using an executor service</li>+ * <li>In {@link org.apache.solr.cloud.OverseerTaskProcessor}.{@code Runner.run()} (run on an executor thread) a call is made to+ * {@link org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler#processMessage} which sets the logging+ * context, calls {@link CollApiCmds.CollectionApiCommand#call}+ * </ul>+ */+ public OverseerSolrResponse runCollectionCommand(ZkNodeProps message, CollectionParams.CollectionAction action, long timeoutMs) {+ // We refuse new tasks, but will wait for already submitted ones (i.e. those that made it through this method earlier).+ // See stopAndWaitForPendingTasksToComplete() below+ if (shuttingDown) {+ throw new SolrException(SolrException.ErrorCode.CONFLICT, "Solr is shutting down, no more Collection API tasks may be executed");+ }++ final String asyncId = message.getStr(ASYNC);++ if (log.isInfoEnabled()) {+ log.info("Running Collection API locally for " + action.name() + " asyncId=" + asyncId); // nowarn+ }++ // Following the call below returning true, we must eventually cancel or complete the task. Happens either in the+ // CollectionCommandRunner below or in the catch when the runner would not execute.+ if (!asyncTaskTracker.createNewAsyncJobTracker(asyncId)) {+ NamedList<String> resp = new NamedList<>();+ resp.add("error", "Task with the same requestid already exists. (" + asyncId + ")");+ resp.add(CoreAdminParams.REQUESTID, asyncId);+ return new OverseerSolrResponse(resp);+ }++ CollectionCommandRunner commandRunner = new CollectionCommandRunner(message, action, asyncId);+ final Future<OverseerSolrResponse> taskFuture;+ try {+ taskFuture = commandsExecutor.submit(commandRunner);+ } catch (RejectedExecutionException ree) {+ // The command will not run, need to cancel the async ID so it can be reused on a subsequent attempt by the client+ asyncTaskTracker.cancelAsyncId(asyncId);+ throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Too many executing commands", ree);+ }++ if (asyncId == null) {+ // Non async calls wait for a while in case the command completes. If they time out, there's no way to track the+ // job progress (improvement suggestion: decorrelate having a task ID from the fact of waiting for the job to complete)+ try {+ return taskFuture.get(timeoutMs, TimeUnit.MILLISECONDS);+ } catch (TimeoutException te) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " timed out after " + timeoutMs + "ms");+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " interrupted", e);+ } catch (Exception e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, action + " failed", e);+ }+ } else {+ // Async calls do not wait for the command to finish but get instead back the async id (that they just sent...)+ NamedList<String> resp = new NamedList<>();+ resp.add(CoreAdminParams.REQUESTID, asyncId);+ return new OverseerSolrResponse(resp);+ }+ }++ /**+ * Best effort wait for termination of all tasks for a (short) while, then return. This method is called when the JVM+ * shuts down so tasks that did not complete are expected to be stopped mid processing.+ */+ public void stopAndWaitForPendingTasksToComplete() {+ shuttingDown = true;++ commandsExecutor.shutdown();+ distributedCollectionApiExecutorService.shutdown();++ // Duration we are willing to wait for threads to terminate, total.+ final long TOTAL_WAIT_NS = 10L * 1000 * 1000 * 1000;++ long start = System.nanoTime();+ try {+ commandsExecutor.awaitTermination(TOTAL_WAIT_NS, TimeUnit.NANOSECONDS);+ long remaining = TOTAL_WAIT_NS - (System.nanoTime() - start);+ if (remaining > 0L) {+ distributedCollectionApiExecutorService.awaitTermination(remaining, TimeUnit.NANOSECONDS);+ }+ } catch (InterruptedException ie) {+ Thread.currentThread().interrupt();+ }+ }++ /**+ * Collection name can be found in either of two message parameters (why??). Return it from where it's defined.+ * (see also parameter {@code collectionNameParamName} of {@link org.apache.solr.cloud.DistributedClusterStateUpdater.MutatingCommand#MutatingCommand(CollectionParams.CollectionAction, String)})+ */+ public static String getCollectionName(ZkNodeProps message) {+ return message.containsKey(COLLECTION_PROP) ?+ message.getStr(COLLECTION_PROP) : message.getStr(NAME);+ }++ /**+ * All Collection API commands are executed in separate threads so that the command can run to completion even if the+ * client request times out, or for async tasks, the client request returns and the command is executed (which really+ * is very similar).+ * This provides a behavior very similar to the one provided by Overseer based Collection API execution.+ */+ private class CollectionCommandRunner implements Callable<OverseerSolrResponse> {+ private final ZkNodeProps message;+ private final CollectionParams.CollectionAction action;+ private final String asyncId;++ private CollectionCommandRunner(ZkNodeProps message, CollectionParams.CollectionAction action, String asyncId) {+ this.message = message;+ this.action = action;+ this.asyncId = asyncId;+ }++ /**+ * This method does always free the Collection API lock it grabs and updates the tracking of the {@code asyncId}+ * (initiated before this method got called) on all execution paths out of this method!+ */+ @Override+ @SuppressWarnings("unchecked")+ public OverseerSolrResponse call() {+ final String collName = getCollectionName(message);+ final String shardId = message.getStr(SHARD_ID_PROP);+ final String replicaName = message.getStr(REPLICA_PROP);++ MDCLoggingContext.setCollection(collName);+ MDCLoggingContext.setShard(shardId);+ MDCLoggingContext.setReplica(replicaName);++ @SuppressWarnings({"rawtypes"})+ NamedList results = new NamedList();+ try {+ // Create API lock for executing the command. This call is non blocking (not blocked on waiting for a lock to be acquired anyway,+ // might be blocked on access to ZK etc)+ // We create a new CollectionApiLockFactory using a new ZkDistributedCollectionLockFactory because earlier (in the constructor of this class)+ // the ZkStateReader was not yet available, due to how CoreContainer is built in part in the constructor and in part in its+ // load() method. And this class is built from the CoreContainer constructor...+ // The cost of these creations is low, and these classes do not hold state but only serve as an interface to Zookeeper.+ // Note that after this call, we MUST execute the lock.release(); in the finally below+ DistributedMultiLock lock = new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(ccc.getZkStateReader().getZkClient(), ZK_COLLECTION_LOCKS)).createCollectionApiLock(action.lockLevel, collName, shardId, replicaName);++ try {+ log.debug("CollectionCommandRunner about to acquire lock for action {} lock level {}. {}/{}/{}",+ action, action.lockLevel, collName, shardId, replicaName);++ // Block this thread until all required locks are acquired.+ lock.waitUntilAcquired();++ // Got the lock so moving from submitted to running if we run for an async task (if asyncId is null the asyncTaskTracker+ // calls do nothing).+ asyncTaskTracker.setTaskRunning(asyncId);++ log.debug("DistributedCollectionConfigSetCommandRunner.runCollectionCommand. Lock acquired. Calling: {}, {}", action, message);++ CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action);+ if (command != null) {+ command.call(ccc.getSolrCloudManager().getClusterStateProvider().getClusterState(), message, results);+ } else {+ asyncTaskTracker.cancelAsyncId(asyncId);+ // Seeing this is a bug, not bad user data+ String message = "Bug: Unknown operation " + action;+ log.error(message);+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message);+ }+ } finally {+ try {+ // TODO If the Collection API command failed because the collection does not exist, we've just created some lock directory+ // structure for a non existent collection... Maybe try to remove it here? No big deal for now as leftover nodes in the+ // lock hierarchy do no harm, and there shouldn't be too many of those.+ lock.release();+ } catch (SolrException se) {+ log.error("Error when releasing collection locks for operation " + action, se); // nowarn+ }+ }+ } catch (Exception e) {+ if (e instanceof InterruptedException) {+ Thread.currentThread().interrupt();+ }+ // Output some error logs+ if (collName == null) {+ SolrException.log(log, "Operation " + action + " failed", e);+ } else {+ SolrException.log(log, "Collection " + collName + ", operation " + action + " failed", e);+ }++ results.add("Operation " + action + " caused exception:", e);+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();+ nl.add("msg", e.getMessage());+ nl.add("rspCode", e instanceof SolrException ? ((SolrException)e).code() : -1);+ results.add("exception", nl);+ }++ OverseerSolrResponse res = new OverseerSolrResponse(results);+ // Following call marks success or failure depending on the contents of res+ asyncTaskTracker.setTaskCompleted(asyncId, res);+ return res;+ }+ }++ /**+ * All Config Set API commands are executed in separate threads so that the command can run to completion even if the+ * client request times out.<br>+ * This provides a behavior very similar to the one provided by Overseer based Config Set API execution.<p>+ *+ * The logic of this class is similar to (and simpler than) the one of {@link CollectionCommandRunner}, more detailed+ * comments there.+ */+ private class ConfigSetCommandRunner implements Callable<Void> {+ private final ZkNodeProps message;+ private final ConfigSetParams.ConfigSetAction action;+ private final String configSetName;+ private final String baseConfigSetName;++ private ConfigSetCommandRunner(ZkNodeProps message, ConfigSetParams.ConfigSetAction action, String configSetName, String baseConfigSetName) {+ this.message = message;+ this.action = action;+ this.configSetName = configSetName;+ this.baseConfigSetName = baseConfigSetName;+ }++ /**+ * This method does always free the Collection API lock it grabs.+ */+ @Override+ public Void call() throws IOException {+ // After this call, we MUST execute the lock.release(); in the finally below+ DistributedMultiLock lock =+ new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(ccc.getZkStateReader().getZkClient(), ZK_CONFIG_SET_LOCKS))+ .createConfigSetApiLock(configSetName, baseConfigSetName);++ try {+ log.debug("ConfigSetCommandRunner about to acquire lock for action {} config set {} base config set {}",+ action, configSetName, baseConfigSetName);++ // Block this thread until all required locks are acquired.+ lock.waitUntilAcquired();++ log.debug("ConfigSetCommandRunner. Lock acquired. Calling: {}, {}", action, message);++ switch (action) {+ case CREATE:+ ConfigSetCmds.createConfigSet(message, coreContainer);+ break;+ case DELETE:+ ConfigSetCmds.deleteConfigSet(message, coreContainer);+ break;+ default:+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bug! Unknown Config Set action: " + action);+ }+ } finally {+ lock.release();+ }++ return null;+ }+ }+}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.javaindex 280eb407cca..0df599a6cf0 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java@@ -58,12 +58,12 @@ public class MaintainRoutedAliasCmd extends AliasCmd {*/static void remoteInvoke(CollectionsHandler collHandler, String aliasName, String targetCol)throws Exception {- final String operation = CollectionParams.CollectionAction.MAINTAINROUTEDALIAS.toLower();+ final CollectionParams.CollectionAction maintainroutedalias = CollectionParams.CollectionAction.MAINTAINROUTEDALIAS;Map<String, Object> msg = new HashMap<>();- msg.put(Overseer.QUEUE_OPERATION, operation);+ msg.put(Overseer.QUEUE_OPERATION, maintainroutedalias.toLower());msg.put(CollectionParams.NAME, aliasName);msg.put(MaintainRoutedAliasCmd.ROUTED_ALIAS_TARGET_COL, targetCol);- final SolrResponse rsp = collHandler.sendToOCPQueue(new ZkNodeProps(msg));+ final SolrResponse rsp = collHandler.submitCollectionApiCommand(new ZkNodeProps(msg), maintainroutedalias);if (rsp.getException() != null) {throw rsp.getException();}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.javaindex 582c4738c8a..4416f236309 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OcmhCollectionCommandContext.java@@ -39,6 +39,11 @@ public class OcmhCollectionCommandContext implements CollectionCommandContext {this.ocmh = ocmh;}+ @Override+ public boolean isDistributedCollectionAPI() {+ return false;+ }+@Overridepublic ShardHandler newShardHandler() {return ocmh.shardHandlerFactory.getShardHandler();diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.javaindex fa79e5a3829..44c8f3d6d44 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java@@ -16,7 +16,6 @@*/package org.apache.solr.cloud.api.collections;-import com.google.common.collect.ImmutableMap;import org.apache.solr.client.solrj.cloud.SolrCloudManager;import org.apache.solr.cloud.LockTree;import org.apache.solr.cloud.Overseer;@@ -43,20 +42,19 @@ import org.slf4j.LoggerFactory;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.Arrays;-import java.util.Map;import java.util.concurrent.ExecutorService;import java.util.concurrent.SynchronousQueue;import java.util.concurrent.TimeUnit;import static org.apache.solr.common.cloud.ZkStateReader.*;import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;import static org.apache.solr.common.params.CommonParams.NAME;/*** A {@link OverseerMessageHandler} that handles Collections API related overseer messages.<p>*- * A lot of the content that was in this class got moved to {@link CollectionHandlingUtils} and {@link CollApiCmds}.+ * A lot of the content that was in this class got moved to {@link CollectionHandlingUtils} and {@link CollApiCmds}.<p>+ * The equivalent of this class for distributed Collection API command execution is {@link DistributedCollectionConfigSetCommandRunner}.*/public class OverseerCollectionMessageHandler implements OverseerMessageHandler, SolrCloseable {@@ -71,7 +69,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,String myId;Stats stats;TimeSource timeSource;- private final CollectionCommandContext ccc;// Set that tracks collections that are currently being processed by a running task.// This is used for handling mutual exclusion of the tasks.@@ -81,7 +78,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,new SynchronousQueue<>(),new SolrNamedThreadFactory("OverseerCollectionMessageHandlerThreadFactory"));- final private Map<CollectionAction, CollApiCmds.CollectionApiCommand> commandMap;+ final private CollApiCmds.CommandMap commandMapper;private volatile boolean isClosed;@@ -100,44 +97,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,this.cloudManager = overseer.getSolrCloudManager();this.timeSource = cloudManager.getTimeSource();this.isClosed = false;- ccc = new OcmhCollectionCommandContext(this);- commandMap = new ImmutableMap.Builder<CollectionAction, CollApiCmds.CollectionApiCommand>()- .put(REPLACENODE, new ReplaceNodeCmd(ccc))- .put(DELETENODE, new DeleteNodeCmd(ccc))- .put(BACKUP, new BackupCmd(ccc))- .put(RESTORE, new RestoreCmd(ccc))- .put(DELETEBACKUP, new DeleteBackupCmd(ccc))- .put(CREATESNAPSHOT, new CreateSnapshotCmd(ccc))- .put(DELETESNAPSHOT, new DeleteSnapshotCmd(ccc))- .put(SPLITSHARD, new SplitShardCmd(ccc))- .put(ADDROLE, new OverseerRoleCmd(ccc, ADDROLE, overseerPrioritizer))- .put(REMOVEROLE, new OverseerRoleCmd(ccc, REMOVEROLE, overseerPrioritizer))- .put(MOCK_COLL_TASK, new CollApiCmds.MockOperationCmd())- .put(MOCK_SHARD_TASK, new CollApiCmds.MockOperationCmd())- .put(MOCK_REPLICA_TASK, new CollApiCmds.MockOperationCmd())- .put(CREATESHARD, new CreateShardCmd(ccc))- .put(MIGRATE, new MigrateCmd(ccc))- .put(CREATE, new CreateCollectionCmd(ccc))- .put(MODIFYCOLLECTION, new CollApiCmds.ModifyCollectionCmd(ccc))- .put(ADDREPLICAPROP, new CollApiCmds.AddReplicaPropCmd(ccc))- .put(DELETEREPLICAPROP, new CollApiCmds.DeleteReplicaPropCmd(ccc))- .put(BALANCESHARDUNIQUE, new CollApiCmds.BalanceShardsUniqueCmd(ccc))- .put(REBALANCELEADERS, new CollApiCmds.RebalanceLeadersCmd(ccc))- .put(RELOAD, new CollApiCmds.ReloadCollectionCmd(ccc))- .put(DELETE, new DeleteCollectionCmd(ccc))- .put(CREATEALIAS, new CreateAliasCmd(ccc))- .put(DELETEALIAS, new DeleteAliasCmd(ccc))- .put(ALIASPROP, new SetAliasPropCmd(ccc))- .put(MAINTAINROUTEDALIAS, new MaintainRoutedAliasCmd(ccc))- .put(OVERSEERSTATUS, new OverseerStatusCmd(ccc))- .put(DELETESHARD, new DeleteShardCmd(ccc))- .put(DELETEREPLICA, new DeleteReplicaCmd(ccc))- .put(ADDREPLICA, new AddReplicaCmd(ccc))- .put(MOVEREPLICA, new MoveReplicaCmd(ccc))- .put(REINDEXCOLLECTION, new ReindexCollectionCmd(ccc))- .put(RENAME, new RenameCmd(ccc))- .build()- ;+ commandMapper = new CollApiCmds.CommandMap(new OcmhCollectionCommandContext(this), overseerPrioritizer);}@Override@@ -152,7 +112,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,NamedList results = new NamedList();try {CollectionAction action = getCollectionAction(operation);- CollApiCmds.CollectionApiCommand command = commandMap.get(action);+ CollApiCmds.CollectionApiCommand command = commandMapper.getActionCommand(action);if (command != null) {command.call(cloudManager.getClusterStateProvider().getClusterState(), message, results);} else {diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.javaindex d071e392390..c8bb3e45b0b 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java@@ -56,6 +56,11 @@ public class OverseerRoleCmd implements CollApiCmds.CollectionApiCommand {@Override@SuppressWarnings({"unchecked", "rawtypes"})public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {+ if (ccc.isDistributedCollectionAPI()) {+ // No Overseer (not accessible from Collection API command execution in any case) so this command can't be run...+ log.error("Cluster is running with distributed Collection API execution. Ignoring Collection API operation " + operation); // nowarn+ return;+ }ZkStateReader zkStateReader = ccc.getZkStateReader();SolrZkClient zkClient = zkStateReader.getZkClient();Map roles = null;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.javaindex c7efe4b8ae3..6cc6b9bdfe5 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerStatusCmd.java@@ -41,6 +41,18 @@ import org.slf4j.LoggerFactory;* <b>within the current Overseer node</b> (this is important because distributed operations occurring on other nodes* are <b>not included</b> in these stats, for example distributed cluster state updates or Per Replica States updates).<p>*+ * More fundamentally, when the Collection API command execution is distributed, this specific command is <b>not</b> being+ * run on the Overseer anyway (but then not much is running on the overseer as cluster state updates are distributed as well)+ * so Overseer stats and status can't be returned and actually do not even make sense. Zookeeper based queue metrics do not+ * make sense either because Zookeeper queues are then not used.<p>+ *+ * The {@link Stats} instance returned by {@link CollectionCommandContext#getOverseerStats()} when running in the Overseer+ * is created in Overseer.start() and passed to the cluster state updater from where it is also propagated to the various+ * Zookeeper queues to register various events. This class is the only place where it is used in the Collection API implementation,+ * and only to return results.+ *+ * TODO: create a new command returning node specific Collection API/Config set API/cluster state updates stats such as success and failures?<p>+ ** The structure of the returned results is as follows:* <ul>* <li><b>{@code leader}:</b> {@code ID} of the current overseer leader node</li>@@ -59,8 +71,8 @@ import org.slf4j.LoggerFactory;* (not all of them!) and {@link org.apache.solr.cloud.overseer.OverseerAction} (the complete list: {@code create},* {@code delete}, {@code createshard}, {@code deleteshard}, {@code addreplica}, {@code addreplicaprop}, {@code deletereplicaprop},* {@code balanceshardunique}, {@code modifycollection}, {@code state}, {@code leader}, {@code deletecore}, {@code addroutingrule},- * {@code removeroutingrule}, {@code updateshardstate}, {@code downnode} and {@code quit} with this like one unlikely- * to be observed since the Overseer is existing right away)</li>+ * {@code removeroutingrule}, {@code updateshardstate}, {@code downnode} and {@code quit} with this last one unlikely+ * to be observed since the Overseer is exiting right away)</li>* <li>{@code update_state} (when Overseer cluster state updater persists changes in Zookeeper)</li>* </ul>* For each key, the value is a map composed of:@@ -137,6 +149,13 @@ public class OverseerStatusCmd implements CollApiCmds.CollectionApiCommand {@Override@SuppressWarnings("unchecked")public void call(ClusterState state, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {+ // If Collection API execution is distributed, we're not running on the Overseer node so can't return any Overseer stats.+ if (ccc.getCoreContainer().getDistributedCollectionCommandRunner().isPresent()) {+ // TODO: introduce a per node status command allowing insight into how Cluster state updates, Collection API and+ // config set API execution went on that node...+ return;+ }+ZkStateReader zkStateReader = ccc.getZkStateReader();String leaderNode = OverseerTaskProcessor.getLeaderNode(zkStateReader.getZkClient());results.add("leader", leaderNode);diff --git a/solr/core/src/java/org/apache/solr/core/CloudConfig.java b/solr/core/src/java/org/apache/solr/core/CloudConfig.javaindex 440f1288fe7..fdfa9a4bd87 100644--- a/solr/core/src/java/org/apache/solr/core/CloudConfig.java+++ b/solr/core/src/java/org/apache/solr/core/CloudConfig.java@@ -50,10 +50,12 @@ public class CloudConfig {private final boolean useDistributedClusterStateUpdates;+ private final boolean useDistributedCollectionConfigSetExecution;+CloudConfig(String zkHost, int zkClientTimeout, int hostPort, String hostName, String hostContext, boolean useGenericCoreNames,int leaderVoteWait, int leaderConflictResolveWait, String zkCredentialsProviderClass, String zkACLProviderClass,int createCollectionWaitTimeTillActive, boolean createCollectionCheckLeaderActive, String pkiHandlerPrivateKeyPath,- String pkiHandlerPublicKeyPath, boolean useDistributedClusterStateUpdates) {+ String pkiHandlerPublicKeyPath, boolean useDistributedClusterStateUpdates, boolean useDistributedCollectionConfigSetExecution) {this.zkHost = zkHost;this.zkClientTimeout = zkClientTimeout;this.hostPort = hostPort;@@ -69,6 +71,11 @@ public class CloudConfig {this.pkiHandlerPrivateKeyPath = pkiHandlerPrivateKeyPath;this.pkiHandlerPublicKeyPath = pkiHandlerPublicKeyPath;this.useDistributedClusterStateUpdates = useDistributedClusterStateUpdates;+ this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;++ if (useDistributedCollectionConfigSetExecution && !useDistributedClusterStateUpdates) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'useDistributedCollectionConfigSetExecution' can't be true if useDistributedClusterStateUpdates is false");+ }if (this.hostPort == -1)throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'hostPort' must be configured to run SolrCloud");@@ -136,6 +143,10 @@ public class CloudConfig {return useDistributedClusterStateUpdates;}+ public boolean getDistributedCollectionConfigSetExecution() {+ return useDistributedCollectionConfigSetExecution;+ }+public static class CloudConfigBuilder {private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 45000;@@ -159,6 +170,7 @@ public class CloudConfig {private String pkiHandlerPrivateKeyPath;private String pkiHandlerPublicKeyPath;private boolean useDistributedClusterStateUpdates = false;+ private boolean useDistributedCollectionConfigSetExecution = false;public CloudConfigBuilder(String hostName, int hostPort) {this(hostName, hostPort, null);@@ -230,10 +242,16 @@ public class CloudConfig {return this;}+ public CloudConfigBuilder setUseDistributedCollectionConfigSetExecution(boolean useDistributedCollectionConfigSetExecution) {+ this.useDistributedCollectionConfigSetExecution = useDistributedCollectionConfigSetExecution;+ return this;+ }+public CloudConfig build() {return new CloudConfig(zkHost, zkClientTimeout, hostPort, hostName, hostContext, useGenericCoreNames, leaderVoteWait,leaderConflictResolveWait, zkCredentialsProviderClass, zkACLProviderClass, createCollectionWaitTimeTillActive,- createCollectionCheckLeaderActive, pkiHandlerPrivateKeyPath, pkiHandlerPublicKeyPath, useDistributedClusterStateUpdates);+ createCollectionCheckLeaderActive, pkiHandlerPrivateKeyPath, pkiHandlerPublicKeyPath,+ useDistributedClusterStateUpdates, useDistributedCollectionConfigSetExecution);}}}diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 88c71f5d9db..21ffc7497b9 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -72,6 +72,7 @@ import org.apache.solr.cloud.CloudDescriptor;import org.apache.solr.cloud.ClusterSingleton;import org.apache.solr.cloud.OverseerTaskQueue;import org.apache.solr.cloud.ZkController;+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;import org.apache.solr.cluster.events.ClusterEventProducer;import org.apache.solr.cluster.events.impl.ClusterEventProducerFactory;import org.apache.solr.cluster.placement.PlacementPluginConfig;@@ -276,6 +277,12 @@ public class CoreContainer {private ExecutorService coreContainerAsyncTaskExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("Core Container Async Task");+ /**+ * Non empty if the Collection API is executed in a distributed way and not on Overseer, once the CoreContainer has been initialized+ * properly, i.e. method {@link #load()} called. Until then it is null, and it is not expected to be read.+ */+ private volatile Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionCommandRunner;+private enum CoreInitFailedAction {fromleader, none}/**@@ -353,9 +360,10 @@ public class CoreContainer {}public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrCoreLoad) {+ this.cfg = requireNonNull(config);this.loader = config.getSolrResourceLoader();this.solrHome = config.getSolrHome();- this.cfg = requireNonNull(config);+try {containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(cfg.getCloudConfig()));} catch (IOException | InvalidKeySpecException e) {@@ -594,6 +602,7 @@ public class CoreContainer {cfg = null;containerProperties = null;replayUpdatesExecutor = null;+ distributedCollectionCommandRunner = Optional.empty();}public static CoreContainer createAndLoad(Path solrHome) {@@ -759,6 +768,15 @@ public class CoreContainer {coreConfigService = ConfigSetService.createConfigSetService(this);createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);++ // CoreContainer is initialized enough at this stage so we can set distributedCollectionCommandRunner (the+ // construction of DistributedCollectionConfigSetCommandRunner uses Zookeeper so can't be done from the CoreContainer constructor+ // because there Zookeeper is not yet ready). Given this is used in the CollectionsHandler created next line, this is+ // the latest point where distributedCollectionCommandRunner can be initialized without refactoring this method...+ // TODO: manage to completely build CoreContainer in the constructor and not in the load() method... Requires some test refactoring.+ this.distributedCollectionCommandRunner = isZooKeeperAware() && cfg.getCloudConfig().getDistributedCollectionConfigSetExecution() ?+ Optional.of(new DistributedCollectionConfigSetCommandRunner(this)) : Optional.empty();+collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);final CollectionsAPI collectionsAPI = new CollectionsAPI(collectionsHandler);containerHandlers.getApiBag().registerObject(collectionsAPI);@@ -938,7 +956,9 @@ public class CoreContainer {});clusterSingletons.setReady();- zkSys.getZkController().checkOverseerDesignate();+ if (!distributedCollectionCommandRunner.isPresent()) {+ zkSys.getZkController().checkOverseerDesignate();+ }}// This is a bit redundant but these are two distinct concepts for all they're accomplished at the same time.@@ -1038,8 +1058,14 @@ public class CoreContainer {ZkController zkController = getZkController();if (zkController != null) {- OverseerTaskQueue overseerCollectionQueue = zkController.getOverseerCollectionQueue();- overseerCollectionQueue.allowOverseerPendingTasksToComplete();+ if (distributedCollectionCommandRunner.isPresent()) {+ // Local (i.e. distributed) Collection API processing+ distributedCollectionCommandRunner.get().stopAndWaitForPendingTasksToComplete();+ } else {+ // Overseer based processing+ OverseerTaskQueue overseerCollectionQueue = zkController.getOverseerCollectionQueue();+ overseerCollectionQueue.allowOverseerPendingTasksToComplete();+ }}if (log.isInfoEnabled()) {log.info("Shutting down CoreContainer instance={}", System.identityHashCode(this));@@ -2217,6 +2243,10 @@ public class CoreContainer {return placementPluginFactory;}+ public Optional<DistributedCollectionConfigSetCommandRunner> getDistributedCollectionCommandRunner() {+ return this.distributedCollectionCommandRunner;+ }+static {ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());}diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.javaindex 7c941eeee74..d40c18afaa8 100644--- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java+++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java@@ -491,6 +491,9 @@ public class SolrXmlConfig {case "distributedClusterStateUpdates":builder.setUseDistributedClusterStateUpdates(Boolean.parseBoolean(value));break;+ case "distributedCollectionConfigSetExecution":+ builder.setUseDistributedCollectionConfigSetExecution(Boolean.parseBoolean(value));+ break;default:throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown configuration parameter in <solrcloud> section of solr.xml: " + name);}diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.javaindex e8ef968287e..09fb2a3ae21 100644--- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java+++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java@@ -26,7 +26,7 @@ import org.apache.solr.api.PayloadObj;import org.apache.solr.client.solrj.request.beans.ClusterPropPayload;import org.apache.solr.client.solrj.request.beans.CreateConfigPayload;import org.apache.solr.client.solrj.request.beans.RateLimiterPayload;-import org.apache.solr.cloud.OverseerConfigSetMessageHandler;+import org.apache.solr.cloud.ConfigSetCmds;import org.apache.solr.common.SolrException;import org.apache.solr.common.annotation.JsonProperty;import org.apache.solr.common.cloud.ClusterProperties;@@ -131,7 +131,7 @@ public class ClusterAPI {Map<String, Object> mapVals = obj.get().toMap(new HashMap<>());Map<String,Object> customProps = (Map<String, Object>) mapVals.remove("properties");if(customProps!= null) {- customProps.forEach((k, o) -> mapVals.put(OverseerConfigSetMessageHandler.PROPERTY_PREFIX+"."+ k, o));+ customProps.forEach((k, o) -> mapVals.put(ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX + k, o));}mapVals.put("action", ConfigSetParams.ConfigSetAction.CREATE.toString());configSetsHandler.handleRequestBody(wrapParams(obj.getRequest(), mapVals), obj.getResponse());diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex 1d689d0105d..5c5499278ba 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -28,7 +28,6 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard;import org.apache.solr.client.solrj.response.RequestStatusState;import org.apache.solr.client.solrj.util.SolrIdentifierValidator;-import org.apache.solr.cloud.DistributedClusterStateUpdater;import org.apache.solr.cloud.OverseerSolrResponse;import org.apache.solr.cloud.OverseerSolrResponseSerializer;import org.apache.solr.cloud.OverseerTaskQueue;@@ -36,6 +35,7 @@ import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;import org.apache.solr.cloud.ZkController;import org.apache.solr.cloud.ZkController.NotInClusterStateException;import org.apache.solr.cloud.ZkShardTerms;+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;import org.apache.solr.cloud.api.collections.ReindexCollectionCmd;import org.apache.solr.cloud.api.collections.RoutedAlias;import org.apache.solr.cloud.overseer.SliceMutator;@@ -63,6 +63,7 @@ import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Pair;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.Utils;import org.apache.solr.core.CloudConfig;@@ -99,6 +100,7 @@ import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;import java.util.Map;+import java.util.Optional;import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;@@ -137,7 +139,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permissionprotected final CoreContainer coreContainer;private final CollectionHandlerApi v2Handler;- private final DistributedClusterStateUpdater distributedClusterStateUpdater;+ private final Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionConfigSetCommandRunner;public CollectionsHandler() {// Unlike most request handlers, CoreContainer initialization@@ -154,17 +156,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permissionpublic CollectionsHandler(final CoreContainer coreContainer) {this.coreContainer = coreContainer;v2Handler = new CollectionHandlerApi(this);- // Get the state change factory to know if need to enqueue to Overseer or process distributed.- // Some SolrCloud tests do not need Zookeeper and end up with a null cloudConfig in NodeConfig (because- // TestHarness.buildTestNodeConfig() uses the zkHost to decide it's SolrCloud).- // These tests do not use Zookeeper and do not do state updates (see subclasses of TestBaseStatsCacheCloud).- // Some non SolrCloud tests do not even pass a config at all, so let be cautious here (code is not pretty).- // We do want to initialize here and not do it lazy to not deal with synchronization for actual prod code.- if (coreContainer == null || coreContainer.getConfig() == null || coreContainer.getConfig().getCloudConfig() == null) {- distributedClusterStateUpdater = null;- } else {- distributedClusterStateUpdater = new DistributedClusterStateUpdater(coreContainer.getConfig().getCloudConfig().getDistributedClusterStateUpdates());- }+ distributedCollectionConfigSetCommandRunner = coreContainer != null ? coreContainer.getDistributedCollectionCommandRunner() : Optional.empty();}@Override@@ -179,7 +171,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission}@Override- final public void init(@SuppressWarnings({"rawtypes"})NamedList args) {+ final public void init(@SuppressWarnings({"rawtypes"}) NamedList args) {}@@ -259,7 +251,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permissionprops.put(QUEUE_OPERATION, operation.action.toLower());ZkNodeProps zkProps = new ZkNodeProps(props);- SolrResponse overseerResponse = sendToOCPQueue(zkProps, operation.timeOut);+ final SolrResponse overseerResponse;++ overseerResponse = submitCollectionApiCommand(zkProps, operation.action, operation.timeOut);+rsp.getValues().addAll(overseerResponse.getResponse());Exception exp = overseerResponse.getException();if (exp != null) {@@ -279,70 +274,75 @@ public class CollectionsHandler extends RequestHandlerBase implements Permissionpublic static long DEFAULT_COLLECTION_OP_TIMEOUT = 180 * 1000;- public SolrResponse sendToOCPQueue(ZkNodeProps m) throws KeeperException, InterruptedException {- return sendToOCPQueue(m, DEFAULT_COLLECTION_OP_TIMEOUT);+ public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action) throws KeeperException, InterruptedException {+ return submitCollectionApiCommand(m, action, DEFAULT_COLLECTION_OP_TIMEOUT);}- public SolrResponse sendToOCPQueue(ZkNodeProps m, long timeout) throws KeeperException, InterruptedException {- String operation = m.getStr(QUEUE_OPERATION);- if (operation == null) {- throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION);- }- if (m.get(ASYNC) != null) {-- String asyncId = m.getStr(ASYNC);-- if (asyncId.equals("-1")) {- throw new SolrException(ErrorCode.BAD_REQUEST, "requestid can not be -1. It is reserved for cleanup purposes.");+ public SolrResponse submitCollectionApiCommand(ZkNodeProps m, CollectionAction action, long timeout) throws KeeperException, InterruptedException {+ // Collection API messages are either sent to Overseer and processed there, or processed locally.+ // Distributing Collection API implies we're also distributing Cluster State Updates. Indeed collection creation+ // with non distributed cluster state updates requires for "Per Replica States" that the Collection API be running+ // on Overseer, which means that it is not possible to distributed Collection API while keeping cluster state updates+ // on Overseer. See the call to CollectionCommandContext.submitIntraProcessMessage() in CreateCollectionCmd.call() which+ // can only be done if the Collection API command runs on the same JVM as the Overseer based cluster state update...+ // The configuration handling includes these checks to not allow distributing collection API without distributing+ // cluster state updates (but the other way around is ok). See constructor of CloudConfig.+ if (distributedCollectionConfigSetCommandRunner.isPresent()) {+ return distributedCollectionConfigSetCommandRunner.get().runCollectionCommand(m, action, timeout);+ } else { // Sending the Collection API message to Overseer via a Zookeeper queue+ String operation = m.getStr(QUEUE_OPERATION);+ if (operation == null) {+ throw new SolrException(ErrorCode.BAD_REQUEST, "missing key " + QUEUE_OPERATION);}-- NamedList<String> r = new NamedList<>();--- if (coreContainer.getZkController().claimAsyncId(asyncId)) {- boolean success = false;- try {- coreContainer.getZkController().getOverseerCollectionQueue()- .offer(Utils.toJSON(m));- success = true;- } finally {- if (!success) {- try {- coreContainer.getZkController().clearAsyncId(asyncId);- } catch (Exception e) {- // let the original exception bubble up- log.error("Unable to release async ID={}", asyncId, e);- SolrZkClient.checkInterrupted(e);+ if (m.get(ASYNC) != null) {+ String asyncId = m.getStr(ASYNC);+ NamedList<String> r = new NamedList<>();++ if (coreContainer.getZkController().claimAsyncId(asyncId)) {+ boolean success = false;+ try {+ coreContainer.getZkController().getOverseerCollectionQueue()+ .offer(Utils.toJSON(m));+ success = true;+ } finally {+ if (!success) {+ try {+ coreContainer.getZkController().clearAsyncId(asyncId);+ } catch (Exception e) {+ // let the original exception bubble up+ log.error("Unable to release async ID={}", asyncId, e);+ SolrZkClient.checkInterrupted(e);+ }}}+ } else {+ r.add("error", "Task with the same requestid already exists. (" + asyncId + ")");}- } else {- r.add("error", "Task with the same requestid already exists.");- }- r.add(CoreAdminParams.REQUESTID, (String) m.get(ASYNC));+ r.add(CoreAdminParams.REQUESTID, (String) m.get(ASYNC));- return new OverseerSolrResponse(r);- }+ return new OverseerSolrResponse(r);+ }- long time = System.nanoTime();- QueueEvent event = coreContainer.getZkController()- .getOverseerCollectionQueue()- .offer(Utils.toJSON(m), timeout);- if (event.getBytes() != null) {- return OverseerSolrResponseSerializer.deserialize(event.getBytes());- } else {- if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {- throw new SolrException(ErrorCode.SERVER_ERROR, operation- + " the collection time out:" + timeout / 1000 + "s");- } else if (event.getWatchedEvent() != null) {- throw new SolrException(ErrorCode.SERVER_ERROR, operation- + " the collection error [Watcher fired on path: "- + event.getWatchedEvent().getPath() + " state: "- + event.getWatchedEvent().getState() + " type "- + event.getWatchedEvent().getType() + "]");+ long time = System.nanoTime();+ QueueEvent event = coreContainer.getZkController()+ .getOverseerCollectionQueue()+ .offer(Utils.toJSON(m), timeout);+ if (event.getBytes() != null) {+ return OverseerSolrResponseSerializer.deserialize(event.getBytes());} else {- throw new SolrException(ErrorCode.SERVER_ERROR, operation- + " the collection unknown case");+ if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {+ throw new SolrException(ErrorCode.SERVER_ERROR, operation+ + " the collection time out:" + timeout / 1000 + "s");+ } else if (event.getWatchedEvent() != null) {+ throw new SolrException(ErrorCode.SERVER_ERROR, operation+ + " the collection error [Watcher fired on path: "+ + event.getWatchedEvent().getPath() + " state: "+ + event.getWatchedEvent().getState() + " type "+ + event.getWatchedEvent().getType() + "]");+ } else {+ throw new SolrException(ErrorCode.SERVER_ERROR, operation+ + " the collection unknown case");+ }}}}@@ -813,29 +813,53 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionREQUESTSTATUS_OP(REQUESTSTATUS, (req, rsp, h) -> {req.getParams().required().check(REQUESTID);- final CoreContainer coreContainer1 = h.coreContainer;+ final CoreContainer coreContainer = h.coreContainer;final String requestId = req.getParams().get(REQUESTID);- final ZkController zkController = coreContainer1.getZkController();-- final NamedList<Object> results = new NamedList<>();- if (zkController.getOverseerCompletedMap().contains(requestId)) {- final byte[] mapEntry = zkController.getOverseerCompletedMap().get(requestId);- rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());- addStatusToResponse(results, COMPLETED, "found [" + requestId + "] in completed tasks");- } else if (zkController.getOverseerFailureMap().contains(requestId)) {- final byte[] mapEntry = zkController.getOverseerFailureMap().get(requestId);- rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());- addStatusToResponse(results, FAILED, "found [" + requestId + "] in failed tasks");- } else if (zkController.getOverseerRunningMap().contains(requestId)) {- addStatusToResponse(results, RUNNING, "found [" + requestId + "] in running tasks");- } else if (h.overseerCollectionQueueContains(requestId)) {- addStatusToResponse(results, SUBMITTED, "found [" + requestId + "] in submitted tasks");+ final ZkController zkController = coreContainer.getZkController();++ final NamedList<Object> status = new NamedList<>();+ if (coreContainer.getDistributedCollectionCommandRunner().isEmpty()) {+ if (zkController.getOverseerCompletedMap().contains(requestId)) {+ final byte[] mapEntry = zkController.getOverseerCompletedMap().get(requestId);+ rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());+ addStatusToResponse(status, COMPLETED, "found [" + requestId + "] in completed tasks");+ } else if (zkController.getOverseerFailureMap().contains(requestId)) {+ final byte[] mapEntry = zkController.getOverseerFailureMap().get(requestId);+ rsp.getValues().addAll(OverseerSolrResponseSerializer.deserialize(mapEntry).getResponse());+ addStatusToResponse(status, FAILED, "found [" + requestId + "] in failed tasks");+ } else if (zkController.getOverseerRunningMap().contains(requestId)) {+ addStatusToResponse(status, RUNNING, "found [" + requestId + "] in running tasks");+ } else if (h.overseerCollectionQueueContains(requestId)) {+ addStatusToResponse(status, SUBMITTED, "found [" + requestId + "] in submitted tasks");+ } else {+ addStatusToResponse(status, NOT_FOUND, "Did not find [" + requestId + "] in any tasks queue");+ }} else {- addStatusToResponse(results, NOT_FOUND, "Did not find [" + requestId + "] in any tasks queue");+ Pair<RequestStatusState, OverseerSolrResponse> sr =+ coreContainer.getDistributedCollectionCommandRunner().get().getAsyncTaskRequestStatus(requestId);+ final String message;+ switch (sr.first()) {+ case COMPLETED:+ message = "found [" + requestId + "] in completed tasks";+ rsp.getValues().addAll(sr.second().getResponse());+ break;+ case FAILED:+ message = "found [" + requestId + "] in failed tasks";+ rsp.getValues().addAll(sr.second().getResponse());+ break;+ case RUNNING:+ message = "found [" + requestId + "] in running tasks";+ break;+ case SUBMITTED:+ message = "found [" + requestId + "] in submitted tasks";+ break;+ default:+ message = "Did not find [" + requestId + "] in any tasks queue";+ }+ addStatusToResponse(status, sr.first(), message);}- final SolrResponse response = new OverseerSolrResponse(results);- rsp.getValues().addAll(response.getResponse());+ rsp.getValues().addAll(status);return null;}),DELETESTATUS_OP(DELETESTATUS, new CollectionOp() {@@ -856,30 +880,42 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission"Both requestid and flush parameters can not be specified together.");}- if (flush) {- Collection<String> completed = zkController.getOverseerCompletedMap().keys();- Collection<String> failed = zkController.getOverseerFailureMap().keys();- for (String asyncId : completed) {- zkController.getOverseerCompletedMap().remove(asyncId);- zkController.clearAsyncId(asyncId);- }- for (String asyncId : failed) {- zkController.getOverseerFailureMap().remove(asyncId);- zkController.clearAsyncId(asyncId);+ if (coreContainer.getDistributedCollectionCommandRunner().isEmpty()) {+ if (flush) {+ Collection<String> completed = zkController.getOverseerCompletedMap().keys();+ Collection<String> failed = zkController.getOverseerFailureMap().keys();+ for (String asyncId : completed) {+ zkController.getOverseerCompletedMap().remove(asyncId);+ zkController.clearAsyncId(asyncId);+ }+ for (String asyncId : failed) {+ zkController.getOverseerFailureMap().remove(asyncId);+ zkController.clearAsyncId(asyncId);+ }+ rsp.getValues().add("status", "successfully cleared stored collection api responses");+ } else {+ // Request to cleanup+ if (zkController.getOverseerCompletedMap().remove(requestId)) {+ zkController.clearAsyncId(requestId);+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");+ } else if (zkController.getOverseerFailureMap().remove(requestId)) {+ zkController.clearAsyncId(requestId);+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");+ } else {+ rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");+ // Don't call zkController.clearAsyncId for this, since it could be a running/pending task+ }}- rsp.getValues().add("status", "successfully cleared stored collection api responses");- return null;} else {- // Request to cleanup- if (zkController.getOverseerCompletedMap().remove(requestId)) {- zkController.clearAsyncId(requestId);- rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");- } else if (zkController.getOverseerFailureMap().remove(requestId)) {- zkController.clearAsyncId(requestId);- rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");+ if (flush) {+ coreContainer.getDistributedCollectionCommandRunner().get().deleteAllAsyncIds();+ rsp.getValues().add("status", "successfully cleared stored collection api responses");} else {- rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");- // Don't call zkController.clearAsyncId for this, since it could be a running/pending task+ if (coreContainer.getDistributedCollectionCommandRunner().get().deleteSingleAsyncId(requestId)) {+ rsp.getValues().add("status", "successfully removed stored response for [" + requestId + "]");+ } else {+ rsp.getValues().add("status", "[" + requestId + "] not found in stored responses");+ }}}return null;@@ -906,7 +942,14 @@ public class CollectionsHandler extends RequestHandlerBase implements Permissionreturn copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX);}),OVERSEERSTATUS_OP(OVERSEERSTATUS, (req, rsp, h) -> new LinkedHashMap<>()),-+ @SuppressWarnings({"unchecked"})+ DISTRIBUTEDAPIPROCESSING_OP(DISTRIBUTEDAPIPROCESSING, (req, rsp, h) -> {+ NamedList<Object> results = new NamedList<>();+ boolean isDistributedApi = h.coreContainer.getDistributedCollectionCommandRunner().isPresent();+ results.add("isDistributedApi", isDistributedApi);+ rsp.getValues().addAll(results);+ return null;+ }),/*** Handle list collection request.* Do list collection request to zk host@@ -1313,7 +1356,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission"shard",FOLLOW_ALIASES);}),- DELETENODE_OP(DELETENODE, (req, rsp, h) -> copy(req.getParams().required(), null, "node"));+ DELETENODE_OP(DELETENODE, (req, rsp, h) -> copy(req.getParams().required(), null, "node")),+ MOCK_COLL_TASK_OP(MOCK_COLL_TASK, (req, rsp, h) -> {+ Map<String, Object> map = copy(req.getParams().required(), null, COLLECTION_PROP);+ return copy(req.getParams(), map, "sleep");});/*** Places all prefixed properties in the sink map (or a new map) using the prefix as the key and a map ofdiff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.javaindex 39608a0d13c..75b040e519d 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java@@ -24,6 +24,7 @@ import java.util.Collections;import java.util.Iterator;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.concurrent.TimeUnit;import java.util.zip.ZipEntry;import java.util.zip.ZipInputStream;@@ -31,9 +32,11 @@ import java.util.zip.ZipInputStream;import org.apache.commons.io.IOUtils;import org.apache.commons.lang3.StringUtils;import org.apache.solr.client.solrj.SolrResponse;+import org.apache.solr.cloud.ConfigSetCmds;import org.apache.solr.cloud.OverseerSolrResponse;import org.apache.solr.cloud.OverseerSolrResponseSerializer;import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrException.ErrorCode;import org.apache.solr.common.cloud.ZkNodeProps;@@ -57,9 +60,7 @@ import org.slf4j.Logger;import org.slf4j.LoggerFactory;import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;-import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.BASE_CONFIGSET;import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.CONFIGSETS_ACTION_PREFIX;-import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.PROPERTY_PREFIX;import static org.apache.solr.common.params.CommonParams.NAME;import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE;@@ -75,7 +76,8 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNfinal public static String AUTOCREATED_CONFIGSET_SUFFIX = ".AUTOCREATED";private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());protected final CoreContainer coreContainer;- public static long DEFAULT_ZK_TIMEOUT = 300 * 1000;+ private final Optional<DistributedCollectionConfigSetCommandRunner> distributedCollectionConfigSetCommandRunner;+ public static long CONFIG_SET_TIMEOUT = 300 * 1000;/*** Overloaded ctor to inject CoreContainer into the handler.*@@ -83,6 +85,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN*/public ConfigSetsHandler(final CoreContainer coreContainer) {this.coreContainer = coreContainer;+ distributedCollectionConfigSetCommandRunner = coreContainer != null ? coreContainer.getDistributedCollectionCommandRunner() : Optional.empty();}public static String getSuffixedNameForAutoGeneratedConfigSet(String configName) {@@ -136,18 +139,22 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNlog.info("Invoked ConfigSet Action :{} with params {} ", action.toLower(), req.getParamString());}Map<String, Object> result = operation.call(req, rsp, this);- sendToZk(rsp, operation, result);+ if (result != null) {+ if (distributedCollectionConfigSetCommandRunner.isPresent()) {+ distributedCollectionConfigSetCommandRunner.get().runConfigSetCommand(rsp, operation, result, CONFIG_SET_TIMEOUT);+ } else { // Sending the Collection API message to Overseer via a Zookeeper queue+ sendToOverseer(rsp, operation, result);+ }+ }}- protected void sendToZk(SolrQueryResponse rsp, ConfigSetOperation operation, Map<String, Object> result)+ protected void sendToOverseer(SolrQueryResponse rsp, ConfigSetOperation operation, Map<String, Object> result)throws KeeperException, InterruptedException {- if (result != null) {- // We need to differentiate between collection and configsets actions since they currently- // use the same underlying queue.- result.put(QUEUE_OPERATION, CONFIGSETS_ACTION_PREFIX + operation.action.toLower());- ZkNodeProps props = new ZkNodeProps(result);- handleResponse(operation.action.toLower(), props, rsp, DEFAULT_ZK_TIMEOUT);- }+ // We need to differentiate between collection and configsets actions since they currently+ // use the same underlying queue.+ result.put(QUEUE_OPERATION, CONFIGSETS_ACTION_PREFIX + operation.action.toLower());+ ZkNodeProps props = new ZkNodeProps(result);+ handleResponse(operation.action.toLower(), props, rsp, CONFIG_SET_TIMEOUT);}private void handleConfigUploadRequest(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {@@ -366,7 +373,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNCREATE_OP(CREATE) {@Overridepublic Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {- String baseConfigSetName = req.getParams().get(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);+ String baseConfigSetName = req.getParams().get(ConfigSetCmds.BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);String newConfigSetName = req.getParams().get(NAME);if (newConfigSetName == null || newConfigSetName.length() == 0) {throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");@@ -383,13 +390,13 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN}Map<String, Object> props = CollectionsHandler.copy(req.getParams().required(), null, NAME);- props.put(BASE_CONFIGSET, baseConfigSetName);+ props.put(ConfigSetCmds.BASE_CONFIGSET, baseConfigSetName);if (!DISABLE_CREATE_AUTH_CHECKS &&!isTrusted(req, h.coreContainer.getAuthenticationPlugin()) &&isCurrentlyTrusted(h.coreContainer.getConfigSetService(), baseConfigSetName)) {- throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + BASE_CONFIGSET);+ throw new SolrException(ErrorCode.UNAUTHORIZED, "Can't create a configset with an unauthenticated request from a trusted " + ConfigSetCmds.BASE_CONFIGSET);}- return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX + ".");+ return copyPropertiesWithPrefix(req.getParams(), props, ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX);}},DELETE_OP(DELETE) {@@ -417,6 +424,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNthis.action = action;}+ public ConfigSetAction getAction() {+ return action;+ }public abstract Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception;public static ConfigSetOperation get(ConfigSetAction action) {diff --git a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.javaindex 239fbecbfa0..45cba5907e6 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java@@ -37,6 +37,7 @@ import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;+import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.core.CoreContainer;import org.apache.solr.request.SolrQueryRequest;@@ -403,20 +404,21 @@ class RebalanceLeaders {private void rejoinElectionQueue(Slice slice, String electionNode, String core, boolean rejoinAtHead)throws KeeperException, InterruptedException {Replica replica = slice.getReplica(LeaderElector.getNodeName(electionNode));+ final CollectionParams.CollectionAction rebalanceleaders = REBALANCELEADERS;Map<String, Object> propMap = new HashMap<>();propMap.put(COLLECTION_PROP, collectionName);propMap.put(SHARD_ID_PROP, slice.getName());- propMap.put(QUEUE_OPERATION, REBALANCELEADERS.toLower());+ propMap.put(QUEUE_OPERATION, rebalanceleaders.toLower());propMap.put(CORE_NAME_PROP, core);propMap.put(CORE_NODE_NAME_PROP, replica.getName());propMap.put(ZkStateReader.NODE_NAME_PROP, replica.getNodeName());propMap.put(REJOIN_AT_HEAD_PROP, Boolean.toString(rejoinAtHead)); // Get ourselves to be first in line.propMap.put(ELECTION_NODE_PROP, electionNode);- String asyncId = REBALANCELEADERS.toLower() + "_" + core + "_" + Math.abs(System.nanoTime());+ String asyncId = rebalanceleaders.toLower() + "_" + core + "_" + Math.abs(System.nanoTime());propMap.put(ASYNC, asyncId);asyncRequests.add(asyncId);- collectionsHandler.sendToOCPQueue(new ZkNodeProps(propMap)); // ignore response; we construct our own+ collectionsHandler.submitCollectionApiCommand(new ZkNodeProps(propMap), rebalanceleaders); // ignore response; we construct our own}// maxWaitSecs - How long are we going to wait? Defaults to 30 seconds.diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.javaindex 0acb7e89a7a..09d5969f399 100644--- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java@@ -552,17 +552,17 @@ public class AliasIntegrationTest extends SolrCloudTestCase {// Now delete one of the collections, should fail since an alias points to it.RequestStatusState delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);// failed because the collection is a part of a compound alias- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection("collection_alias_pair");delResp = delete.processAndWait(cluster.getSolrClient(), 60);// failed because we tried to delete an alias with followAliases=false- assertEquals("Should have failed to delete alias: ", delResp, RequestStatusState.FAILED);+ assertEquals("Should have failed to delete alias: ", RequestStatusState.FAILED, delResp);delete.setFollowAliases(true);delResp = delete.processAndWait(cluster.getSolrClient(), 60);// failed because we tried to delete compound alias- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);CollectionAdminRequest.createAlias("collection_alias_one", "collection_one").process(cluster.getSolrClient());lastVersion = waitForAliasesUpdate(lastVersion, zkStateReader);@@ -570,13 +570,13 @@ public class AliasIntegrationTest extends SolrCloudTestCase {delete = CollectionAdminRequest.deleteCollection("collection_one");delResp = delete.processAndWait(cluster.getSolrClient(), 60);// failed because we tried to delete collection referenced by multiple aliases- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);delete = CollectionAdminRequest.deleteCollection("collection_alias_one");delete.setFollowAliases(true);delResp = delete.processAndWait(cluster.getSolrClient(), 60);// failed because we tried to delete collection referenced by multiple aliases- assertEquals("Should have failed to delete collection: ", delResp, RequestStatusState.FAILED);+ assertEquals("Should have failed to delete collection: ", RequestStatusState.FAILED, delResp);CollectionAdminRequest.deleteAlias("collection_alias_one").process(cluster.getSolrClient());lastVersion = waitForAliasesUpdate(lastVersion, zkStateReader);@@ -588,7 +588,7 @@ public class AliasIntegrationTest extends SolrCloudTestCase {//Delete collection_one.delResp = CollectionAdminRequest.deleteCollection("collection_one").processAndWait(cluster.getSolrClient(), 60);- assertEquals("Should not have failed to delete collection, it was removed from the alias: ", delResp, RequestStatusState.COMPLETED);+ assertEquals("Should not have failed to delete collection, it was removed from the alias: ", RequestStatusState.COMPLETED, delResp);// Should only see two docs now in second collectionres = cluster.getSolrClient().query("collection_alias_pair", new SolrQuery("*:*"));diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.javaindex e3a89164c50..dda0f82b8ff 100644--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java@@ -657,7 +657,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("info", "core", "startTime"));assertNotNull(Utils.toJSONString(rsp), segInfos.get("fieldInfoLegend"));assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "fields", "id", "flags"));- assertNotNull(Utils.toJSONString(rsp), segInfos.findRecursive("segments", "_0", "ramBytesUsed"));// test for replicas not active - SOLR-13882DocCollection coll = cluster.getSolrClient().getClusterStateProvider().getClusterState().getCollection(collectionName);Replica firstReplica = coll.getSlice("shard1").getReplicas().iterator().next();@@ -1031,6 +1030,10 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {public void testOverseerStatus() throws IOException, SolrServerException {CollectionAdminResponse response = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient());assertEquals(0, response.getStatus());+ // When running with Distributed Collection API, no real data in Overseer status, but the Collection API call above shouldn't fail+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ return;+ }assertNotNull("overseer_operations shouldn't be null", response.getResponse().get("overseer_operations"));}diff --git a/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.javanew file mode 100644index 00000000000..ef953aa75dc--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/ConfigSetApiLockingTest.java@@ -0,0 +1,124 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;+import java.util.concurrent.CountDownLatch;++import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.cloud.SolrZkClient;+import org.junit.Test;++public class ConfigSetApiLockingTest extends SolrTestCaseJ4 {+ private static final String CONFIG_SET_NAME_1 = "configSet1";+ private static final String CONFIG_SET_NAME_2 = "configSet2";+ private static final String CONFIG_SET_NAME_3 = "configSet3";++ private static final String BASE_CONFIG_SET_NAME = "baseConfigSet";++ static final int TIMEOUT = 10000;++ /**+ * Tests the Config Set API locking. All tests run from single test method to save on setup time.+ */+ @Test+ public void monothreadedApiLockTests() throws Exception {+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));+ try {+ server.run();+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {+ ConfigSetApiLockFactory apiLockFactory = new ConfigSetApiLockFactory(new ZkDistributedConfigSetLockFactory(zkClient, "/apiLockTestRoot"));++ monothreadedTests(apiLockFactory);+ multithreadedTests(apiLockFactory);+ }+ } finally {+ server.shutdown();+ }+ }++ private void monothreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception {+ // Config set lock without a base config set+ DistributedMultiLock cs1Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_1, null);+ assertTrue("cs1Lock should have been acquired", cs1Lock.isAcquired());++ // This lock does have a base config set+ DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);+ assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired());++ // This lock does has the same base config set, but that shouldn't prevent acquiring it+ DistributedMultiLock cs3Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_3, BASE_CONFIG_SET_NAME);+ assertTrue("cs3Lock should have been acquired", cs3Lock.isAcquired());++ // But we shouldn't be able to lock at this stage the base config set+ DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null);+ assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired());++ cs2Lock.release();+ assertFalse("csBaseLock should not have been acquired, cs3Lock still there", csBaseLock.isAcquired());+ cs3Lock.release();+ assertTrue("csBaseLock should have been acquired", csBaseLock.isAcquired());++ // Acquiring a lock with a locked base config set should not be possible+ cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);+ assertFalse("cs2Lock should not have been acquired, csBaseLock still held", cs2Lock.isAcquired());++ csBaseLock.release();+ assertTrue("cs2Lock should now be acquired, csBaseLock was freed", cs2Lock.isAcquired());++ cs2Lock.release();+ }++ private void multithreadedTests(ConfigSetApiLockFactory apiLockingHelper) throws Exception {+ // This lock does have a base config set+ DistributedMultiLock cs2Lock = apiLockingHelper.createConfigSetApiLock(CONFIG_SET_NAME_2, BASE_CONFIG_SET_NAME);+ assertTrue("cs2Lock should have been acquired", cs2Lock.isAcquired());++ // But we shouldn't be able to lock at this stage the base config set+ DistributedMultiLock csBaseLock = apiLockingHelper.createConfigSetApiLock(BASE_CONFIG_SET_NAME, null);+ assertFalse("csBaseLock should not have been acquired", csBaseLock.isAcquired());++ // Wait for acquisition of the base config set lock on another thread (and be notified via a latch)+ final CountDownLatch latch = new CountDownLatch(1);+ new Thread(() -> {+ csBaseLock.waitUntilAcquired();+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind+ latch.countDown();+ }).start();++ // Wait for the thread to start and to get blocked in waitUntilAcquired()+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).+ Thread.sleep(30);++ assertEquals("we should not have been notified that base config set lock was acquired", 1, latch.getCount());+ assertFalse("base config set lock should not have been acquired", csBaseLock.isAcquired());++ cs2Lock.release();+ assertTrue("basec config set lock should have been acquired now that other lock was released", csBaseLock.isAcquired());++ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause+ int i = 0;+ while (i < 1000 && latch.getCount() != 0) {+ Thread.sleep(10);+ i++;+ }+ assertEquals("we should have been notified that the base config set lock was acquired", 0, latch.getCount());+ }+}+diff --git a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.javaindex 53707136cb7..bcdc867a0c4 100644--- a/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/CreateCollectionCleanupTest.java@@ -67,7 +67,7 @@ public class CreateCollectionCleanupTest extends SolrCloudTestCase {configureCluster(1).addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")).withSolrXml(CLOUD_SOLR_XML_WITH_10S_CREATE_COLL_WAIT)- .useOtherClusterStateUpdateStrategy()+ .useOtherCollectionConfigSetExecution().configure();}diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.javaindex de1d19aad23..31c1deb9abc 100644--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java@@ -77,7 +77,7 @@ public class DeleteReplicaTest extends SolrCloudTestCase {// these tests need to be isolated, so we dont share the miniclusterconfigureCluster(4).addConfig("conf", configset("cloud-minimal"))- .useOtherClusterStateUpdateStrategy() // Some tests (this one) use "the other" cluster state update strategy to increase coverage+ .useOtherCollectionConfigSetExecution() // Some tests (this one) use "the other" cluster Collection API execution strategy to increase coverage.configure();}diff --git a/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.java b/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.javanew file mode 100644index 00000000000..994e1984bfd--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/DistributedApiAsyncTrackerTest.java@@ -0,0 +1,192 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.client.solrj.response.RequestStatusState;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.util.NamedList;+import org.junit.Test;++/**+ * Test async id tracking scenarios as used when Collection API is distributed.+ */+public class DistributedApiAsyncTrackerTest extends SolrTestCaseJ4 {++ protected ZkTestServer zkServer;+ protected SolrZkClient zkClient;++ @Override+ public void setUp() throws Exception {+ super.setUp();++ zkServer = new ZkTestServer(createTempDir("zookeeperDir"));+ zkServer.run();+ zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);+ }++ @Override+ public void tearDown() throws Exception {+ super.tearDown();++ zkServer.shutdown();+ zkClient.close();+ }++ @Test+ @SuppressWarnings({"unchecked", "rawtypes"})+ public void testBasic() throws Exception {+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, "/basic");++ final String asyncId = "mario";++ assertTrue("Could not create async task " + asyncId ,daat.createNewAsyncJobTracker(asyncId));+ assertFalse("Should not have been able to create duplicate task " + asyncId ,daat.createNewAsyncJobTracker(asyncId));++ daat.cancelAsyncId(asyncId);+ assertEquals(RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());+ assertTrue("Could not create async task after cancel " + asyncId ,daat.createNewAsyncJobTracker(asyncId));++ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(asyncId).first());+ assertFalse("Can't delete a non completed/failed task " + asyncId, daat.deleteSingleAsyncId(asyncId));+ daat.deleteAllAsyncIds();+ assertEquals("Task should still be here because couldn't be deleted " + asyncId, RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(asyncId).first());++ daat.setTaskRunning(asyncId);+ assertEquals(RequestStatusState.RUNNING, daat.getAsyncTaskRequestStatus(asyncId).first());++ NamedList nl = new NamedList();+ nl.add("MyList", "myValue");+ daat.setTaskCompleted(asyncId, new OverseerSolrResponse(nl));+ assertEquals(RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(asyncId).first());+ assertEquals("Did not retrieve correct completed OverseerSolrResponse " + asyncId, "myValue", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("MyList"));++ assertTrue("Should be able to delete a completed task " + asyncId, daat.deleteSingleAsyncId(asyncId));+ assertEquals("A completed task should not be found", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());+ }++ @Test+ public void testDisconnect() throws Exception {+ final String TRACKER_ROOT = "/disconnect";+ DistributedApiAsyncTracker permanentDaat = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);+ DistributedApiAsyncTracker permanentDaat2 = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);++ final String asycPermanent = "permanentAsync";+ final String asyncTransient = "transientAsync";++ assertTrue("Could not create async task " + asycPermanent ,permanentDaat.createNewAsyncJobTracker(asycPermanent));++ try (SolrZkClient transientZkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) {+ DistributedApiAsyncTracker transientDaat = new DistributedApiAsyncTracker(transientZkClient, TRACKER_ROOT);+ assertTrue("Could not create async task " + asyncTransient ,transientDaat.createNewAsyncJobTracker(asyncTransient));++ assertEquals("permanentDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asycPermanent).first());+ assertEquals("permanentDaat2 can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asycPermanent).first());+ assertEquals("transientDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, transientDaat.getAsyncTaskRequestStatus(asycPermanent).first());+ assertEquals("permanentDaat can't see " + asyncTransient, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asyncTransient).first());+ assertEquals("permanentDaat2 can't see " + asyncTransient, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asyncTransient).first());+ assertEquals("transientDaat can't see " + asyncTransient, RequestStatusState.SUBMITTED, transientDaat.getAsyncTaskRequestStatus(asyncTransient).first());+ }++ // transientDaat connection closed, doesn't change a thing for asycPermanent...+ assertEquals("permanentDaat can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat.getAsyncTaskRequestStatus(asycPermanent).first());+ assertEquals("permanentDaat2 can't see " + asycPermanent, RequestStatusState.SUBMITTED, permanentDaat2.getAsyncTaskRequestStatus(asycPermanent).first());++ // ...but asyncTransient is now failed.+ assertEquals("permanentDaat can't see " + asyncTransient, RequestStatusState.FAILED, permanentDaat.getAsyncTaskRequestStatus(asyncTransient).first());+ assertEquals("permanentDaat2 can't see " + asyncTransient, RequestStatusState.FAILED, permanentDaat2.getAsyncTaskRequestStatus(asyncTransient).first());+ }++ @Test+ @SuppressWarnings({"unchecked", "rawtypes"})+ public void testDisconnectAfterCompletion() throws Exception {+ final String TRACKER_ROOT = "/multiConn";+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, TRACKER_ROOT);++ final String asyncId = "theId";++ try (SolrZkClient transientZkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT)) {+ DistributedApiAsyncTracker transientDaat = new DistributedApiAsyncTracker(transientZkClient, TRACKER_ROOT);+ assertTrue("Could not create async task " + asyncId ,transientDaat.createNewAsyncJobTracker(asyncId));++ // The task completes, then connection lost+ NamedList nl = new NamedList();+ nl.add("status", "I made it");+ transientDaat.setTaskCompleted(asyncId, new OverseerSolrResponse(nl));++ assertEquals("transientDaat can't see " + asyncId, RequestStatusState.COMPLETED, transientDaat.getAsyncTaskRequestStatus(asyncId).first());+ assertEquals("transientDaat can't retrieve correct completed OverseerSolrResponse " + asyncId, "I made it", transientDaat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));+ assertEquals("daat can't retrieve correct completed OverseerSolrResponse " + asyncId, "I made it", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));+ }++ // Even though connection was closed, the task should still be completed, and we should be able to retrieve its response+ assertEquals("daat can't see " + asyncId, RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(asyncId).first());+ assertEquals("Did not retrieve correct completed OverseerSolrResponse after other connection closes " + asyncId, "I made it", daat.getAsyncTaskRequestStatus(asyncId).second().getResponse().get("status"));++ // And given it completed, it should be deleted+ daat.deleteAllAsyncIds();+ assertEquals("task was not deleted " + asyncId, RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(asyncId).first());+ }++ @Test+ @SuppressWarnings({"unchecked", "rawtypes"})+ public void testIdCleanup() throws Exception {+ final int maxTasks = 30; // When cleaning up, 3 async id's will be removed+ DistributedApiAsyncTracker daat = new DistributedApiAsyncTracker(zkClient, "/manyIds", maxTasks);++ for (int asyncId = 1; asyncId <= maxTasks; asyncId++) {+ assertTrue(daat.createNewAsyncJobTracker(Integer.toString(asyncId)));+ }++ // All ids should be tracked+ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(1)).first());+ assertEquals(RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(maxTasks)).first());++ // Adding one more should trigger cleanup of earlier id's+ assertTrue(daat.createNewAsyncJobTracker("straw"));++ String cleanedUpId1 = Integer.toString(1);++ assertEquals("Cleanup should have been triggered and removed the first task", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());+ assertEquals("Last task should not have been cleaned up", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(Integer.toString(maxTasks)).first());+ assertEquals("Task having triggered cleanup should not have been cleaned up", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus("straw").first());++ // Identical to the test 3 lines above but repeated to be considered in context+ assertEquals("Cleaned up ID (1) should no longer be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());+ // Creating a new task with same id fails but "revives" the ID if the task is still in progress+ assertFalse("Should not be able to create a task with same id as task in progress", daat.createNewAsyncJobTracker(cleanedUpId1));+ assertEquals("Cleaned up ID now visible again", RequestStatusState.SUBMITTED, daat.getAsyncTaskRequestStatus(cleanedUpId1).first());++ // 2 is also in progress and was also cleaned up. It should be markable complete without issues (since that's what will happen when it completed).+ String cleanedUpId2 = Integer.toString(2);+ assertEquals("Another cleaned up ID (2) should not be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId2).first());+ NamedList nl = new NamedList();+ nl.add("code", "da vinci");+ daat.setTaskCompleted(cleanedUpId2, new OverseerSolrResponse(nl));++ assertEquals("task should now be completed " + cleanedUpId2, RequestStatusState.COMPLETED, daat.getAsyncTaskRequestStatus(cleanedUpId2).first());+ assertEquals("task should have correct OverseerSolrResponse " + cleanedUpId2, "da vinci", daat.getAsyncTaskRequestStatus(cleanedUpId2).second().getResponse().get("code"));++ // 3 is also in progress and was also cleaned up. It should be markable running without issues, but that doesn't revive it (still not found).+ // (if we want setTaskRunning to revive a removed task, we'd have to check the persistent node existence each time, which is likely not worth it)+ String cleanedUpId3 = Integer.toString(3);+ assertEquals("Another cleaned up ID (3) should not be visible", RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId3).first());+ daat.setTaskRunning(cleanedUpId3);+ assertEquals("task should now be running " + cleanedUpId3, RequestStatusState.NOT_FOUND, daat.getAsyncTaskRequestStatus(cleanedUpId3).first());+ }+}diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.javaindex 53c47dfd546..ff96d6cbf3c 100644--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java@@ -53,6 +53,7 @@ public class MoveReplicaTest extends SolrCloudTestCase {// used by MoveReplicaHDFSTestprotected boolean inPlaceMove = true;+ protected boolean isCollectionApiDistributed = false;protected String getConfigSet() {return "cloud-dynamic";@@ -68,6 +69,11 @@ public class MoveReplicaTest extends SolrCloudTestCase {.withSolrXml(TEST_PATH().resolve("solr.xml")).configure();+ // If Collection API is distributed let's not wait for Overseer.+ if (isCollectionApiDistributed = new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ return;+ }+NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());JettySolrRunner overseerJetty = null;String overseerLeader = (String) overSeerStatus.get("leader");@@ -267,7 +273,7 @@ public class MoveReplicaTest extends SolrCloudTestCase {Collections.shuffle(l, random());String targetNode = null;for (String node : liveNodes) {- if (!replica.getNodeName().equals(node) && !overseerLeader.equals(node)) {+ if (!replica.getNodeName().equals(node) && (isCollectionApiDistributed || !overseerLeader.equals(node))) {targetNode = node;break;}diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.javaindex c8357afad01..270010ee7ec 100644--- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java@@ -23,7 +23,6 @@ import java.util.Random;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.cloud.DistributedQueue;import org.apache.solr.client.solrj.impl.HttpSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create;@@ -32,15 +31,11 @@ import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.client.solrj.response.RequestStatusState;import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.ModifiableSolrParams;-import org.apache.solr.common.util.Utils;import org.junit.Test;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;import static org.apache.solr.cloud.OverseerTaskProcessor.MAX_PARALLEL_TASKS;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOCK_COLL_TASK;-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;/*** Tests the Multi threaded Collections API.@@ -69,49 +64,47 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {private void testFillWorkQueue() throws Exception {try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {- DistributedQueue distributedQueue = new ZkDistributedQueue(cloudClient.getZkStateReader().getZkClient(),- "/overseer/collection-queue-work", new Stats());- //fill the work queue with blocked tasks by adding more than the no:of parallel tasks- for (int i = 0; i < MAX_PARALLEL_TASKS + 15; i++) {- distributedQueue.offer(Utils.toJSON(Utils.makeMap(- "collection", "A_COLL",- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),- ASYNC, Integer.toString(i),-- // third task waits for a long time, and thus blocks the queue for all other tasks for A_COLL.- // Subsequent tasks as well as the first two only wait for 1ms- "sleep", (i == 2 ? "10000" : "1")- )));- log.info("MOCK task added {}", i);- }+ // This test does not make sense when the Collection API execution is distributed. There is no queue to fill+ if (!new CollectionAdminRequest.RequestApiDistributedProcessing().process(client).getIsCollectionApiDistributed()) {++ //fill the work queue with blocked tasks by adding more than the no:of parallel tasks+ for (int i = 0; i < MAX_PARALLEL_TASKS + 15; i++) {++ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("A_COLL");+ // third task waits for a long time, and thus blocks the queue for all other tasks for A_COLL.+ // Subsequent tasks as well as the first two only wait for 1ms+ mockTask.setSleep(i == 2 ? "5000" : "1");+ mockTask.processAsync(Integer.toString(i), client);++ log.info("MOCK task added {}", i);+ }- // Wait until we see the second A_COLL task getting processed (assuming the first got processed as well)- Long task1CollA = waitForTaskToCompleted(client, 1);+ // Wait until we see the second A_COLL task getting processed (assuming the first got processed as well)+ Long task1CollA = waitForTaskToCompleted(client, 1);- assertNotNull("Queue did not process first two tasks on A_COLL, can't run test", task1CollA);+ assertNotNull("Queue did not process first two tasks on A_COLL, can't run test", task1CollA);- // Make sure the long running task did not finish, otherwise no way the B_COLL task can be tested to run in parallel with it- assertNull("Long running task finished too early, can't test", checkTaskHasCompleted(client, 2));+ // Make sure the long running task did not finish, otherwise no way the B_COLL task can be tested to run in parallel with it+ assertNull("Long running task finished too early, can't test", checkTaskHasCompleted(client, 2));- // Enqueue a task on another collection not competing with the lock on A_COLL and see that it can be executed right away- distributedQueue.offer(Utils.toJSON(Utils.makeMap(- "collection", "B_COLL",- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),- ASYNC, "200",- "sleep", "1"- )));+ // Enqueue a task on another collection not competing with the lock on A_COLL and see that it can be executed right away- // We now check that either the B_COLL task has completed before the third (long running) task on A_COLL,- // Or if both have completed (if this check got significantly delayed for some reason), we verify B_COLL was first.- Long taskCollB = waitForTaskToCompleted(client, 200);+ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("B_COLL");+ mockTask.setSleep("1");+ mockTask.processAsync("200", client);- // We do not wait for the long running task to finish, that would be a waste of time.- Long task2CollA = checkTaskHasCompleted(client, 2);+ // We now check that either the B_COLL task has completed before the third (long running) task on A_COLL,+ // Or if both have completed (if this check got significantly delayed for some reason), we verify B_COLL was first.+ Long taskCollB = waitForTaskToCompleted(client, 200);- // Given the wait delay (500 iterations of 100ms), the task has plenty of time to complete, so this is not expected.- assertNotNull("Task on B_COLL did not complete, can't test", taskCollB);- // We didn't wait for the 3rd A_COLL task to complete (test can run quickly) but if it did, we expect the B_COLL to have finished first.- assertTrue("task2CollA: " + task2CollA + " taskCollB: " + taskCollB, task2CollA == null || task2CollA > taskCollB);+ // We do not wait for the long running task to finish, that would be a waste of time.+ Long task2CollA = checkTaskHasCompleted(client, 2);++ // Given the wait delay (500 iterations of 100ms), the task has plenty of time to complete, so this is not expected.+ assertNotNull("Task on B_COLL did not complete, can't test", taskCollB);+ // We didn't wait for the 3rd A_COLL task to complete (test can run quickly) but if it did, we expect the B_COLL to have finished first.+ assertTrue("task2CollA: " + task2CollA + " taskCollB: " + taskCollB, task2CollA == null || task2CollA > taskCollB);+ }}}@@ -141,8 +134,9 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {}private void testParallelCollectionAPICalls() throws IOException, SolrServerException {+ final int ASYNC_SHIFT = 10000;try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {- for(int i = 1 ; i <= NUM_COLLECTIONS ; i++) {+ for(int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++) {CollectionAdminRequest.createCollection("ocptest" + i,"conf1",3,1).processAsync(String.valueOf(i), client);}@@ -150,7 +144,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {int counter = 0;while(true) {int numRunningTasks = 0;- for (int i = 1; i <= NUM_COLLECTIONS; i++)+ for (int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++)if (getRequestState(i + "", client) == RequestStatusState.RUNNING) {numRunningTasks++;}@@ -167,7 +161,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {}}assertTrue("More than one tasks were supposed to be running in parallel but they weren't.", pass);- for (int i = 1; i <= NUM_COLLECTIONS; i++) {+ for (int i = 1 + ASYNC_SHIFT; i <= NUM_COLLECTIONS + ASYNC_SHIFT; i++) {final RequestStatusState state = getRequestStateAfterCompletion(i + "", REQUEST_STATUS_TIMEOUT, client);assertSame("Task " + i + " did not complete, final state: " + state, RequestStatusState.COMPLETED, state);}@@ -175,26 +169,18 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {}private void testTaskExclusivity() throws Exception, SolrServerException {-- DistributedQueue distributedQueue = new ZkDistributedQueue(cloudClient.getZkStateReader().getZkClient(),- "/overseer/collection-queue-work", new Stats());try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {Create createCollectionRequest = CollectionAdminRequest.createCollection("ocptest_shardsplit","conf1",4,1);createCollectionRequest.processAsync("1000",client);- distributedQueue.offer(Utils.toJSON(Utils.makeMap(- "collection", "ocptest_shardsplit",- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),- ASYNC, "1001",- "sleep", "100"- )));- distributedQueue.offer(Utils.toJSON(Utils.makeMap(- "collection", "ocptest_shardsplit",- QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),- ASYNC, "1002",- "sleep", "100"- )));+ CollectionAdminRequest.MockCollTask mockTask = CollectionAdminRequest.mockCollTask("ocptest_shardsplit");+ mockTask.setSleep("100");+ mockTask.processAsync("1001", client);++ mockTask = CollectionAdminRequest.mockCollTask("ocptest_shardsplit");+ mockTask.setSleep("100");+ mockTask.processAsync("1002", client);int iterations = 0;while(true) {@@ -256,7 +242,6 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {}private void testLongAndShortRunningParallelApiCalls() throws InterruptedException, IOException, SolrServerException {-Thread indexThread = new Thread() {@Overridepublic void run() {diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.javaindex 05974a3afa0..c437a30e3a6 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java@@ -102,6 +102,10 @@ public class OverseerRolesTest extends SolrCloudTestCase {@Testpublic void testOverseerRole() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ return;+ }logOverseerState();List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());@@ -169,6 +173,10 @@ public class OverseerRolesTest extends SolrCloudTestCase {@Testpublic void testDesignatedOverseerRestarts() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ return;+ }logOverseerState();// Remove the OVERSEER role, in case it was already assigned by another test in this suitefor (String node: OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient())) {diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.javaindex d2d96047658..bf636e78f40 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java@@ -44,6 +44,10 @@ public class OverseerStatusTest extends SolrCloudTestCase {CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1).process(cluster.getSolrClient());NamedList<Object> resp = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient()).getResponse();+ // When running with Distributed Collection API, no real data in Overseer status, but the Collection API call shouldn't fail+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ return;+ }NamedList<Object> collection_operations = (NamedList<Object>) resp.get("collection_operations");NamedList<Object> overseer_operations = (NamedList<Object>) resp.get("overseer_operations");SimpleOrderedMap<Object> createcollectiondiff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.javaindex c95b72f38e8..7c5c86476f9 100644--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java@@ -751,7 +751,8 @@ public class OverseerTest extends SolrTestCaseJ4 {httpShardHandlerFactory.init(new PluginInfo("shardHandlerFactory", Collections.emptyMap()));httpShardHandlerFactorys.add(httpShardHandlerFactory);Overseer overseer = new Overseer((HttpShardHandler) httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, zkController,- new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").setUseDistributedClusterStateUpdates(false).build());+ new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").setUseDistributedClusterStateUpdates(false).+ setUseDistributedCollectionConfigSetExecution(false).build());overseers.add(overseer);ElectionContext ec = new OverseerElectionContext(zkClient, overseer,server.getZkAddress().replaceAll("/", "_"));diff --git a/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java b/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.javaindex 8941e0dee7e..de939a7ed83 100644--- a/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java@@ -48,6 +48,11 @@ public class RollingRestartTest extends AbstractFullDistribZkTestBase {@Test//commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18public void test() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cloudClient).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ return;+ }+waitForRecoveriesToFinish(false);restartWithRolesTest();diff --git a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.javaindex 197f0352f2d..db96a7672b6 100644--- a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java@@ -162,6 +162,13 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {@Testpublic void testBackCompat() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ // TODO once we completely remove Overseer, do we need to move the back compat check to some other place, for example+ // to when the .system collection is opened?+ return;+ }+CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();CloudSolrClient solrClient = cluster.getSolrClient();CollectionAdminResponse adminResponse = status.process(solrClient);diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java b/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.javaindex ada8bd358a8..3d893b4cc2f 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestSkipOverseerOperations.java@@ -18,6 +18,7 @@package org.apache.solr.cloud;import java.io.IOException;+import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.List;import java.util.SortedSet;@@ -33,9 +34,13 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.junit.After;import org.junit.Before;import org.junit.Test;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;public class TestSkipOverseerOperations extends SolrCloudTestCase {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+@Beforepublic void setupCluster() throws Exception {System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");@@ -53,6 +58,10 @@ public class TestSkipOverseerOperations extends SolrCloudTestCase {}public void testSkipLeaderOperations() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ return;+ }String overseerLeader = getOverseerLeader();@@ -136,6 +145,11 @@ public class TestSkipOverseerOperations extends SolrCloudTestCase {@Test// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018public void testSkipDownOperations() throws Exception {+ if (new CollectionAdminRequest.RequestApiDistributedProcessing().process(cluster.getSolrClient()).getIsCollectionApiDistributed()) {+ log.info("Skipping test because Collection API is distributed");+ return;+ }+String overseerLeader = getOverseerLeader();List<JettySolrRunner> notOverseerNodes = cluster.getJettySolrRunners().stream()diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.javanew file mode 100644index 00000000000..1b88c15384e--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/ZkDistributedLockTest.java@@ -0,0 +1,230 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud;++import java.util.concurrent.CountDownLatch;++import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.params.CollectionParams;+import org.junit.Test;++public class ZkDistributedLockTest extends SolrTestCaseJ4 {++ private static final String COLLECTION_NAME = "lockColl";+ private final String SHARD_NAME = "lockShard";+ private final String REPLICA_NAME = "lockReplica";++ private static final String CONFIG_SET_NAME = "lockConfigSet";++ static final int TIMEOUT = 10000;++ /**+ * Tests the obtention of a single read or write lock at a specific hierarchical level, for distributed Collection API+ * locking as well as distributed Config Set API locking.<br>+ * Tests the logic with a single thread, then tests multithreaded wait for lock acquire works.+ * Tests grouped to pay setup only once.<p>+ *+ * Higher level locking tests can be found at {@link org.apache.solr.cloud.api.collections.CollectionApiLockingTest} and+ * {@link org.apache.solr.cloud.ConfigSetApiLockingTest}+ */+ @Test+ public void testSingleLocks() throws Exception {+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));+ try {+ server.run();+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {+ DistributedCollectionLockFactory collLockFactory = new ZkDistributedCollectionLockFactory(zkClient, "/lockTestCollectionRoot");++ monothreadedCollectionTests(collLockFactory);+ multithreadedCollectionTests(collLockFactory);++ DistributedConfigSetLockFactory configSetLockFactory = new ZkDistributedConfigSetLockFactory(zkClient, "/lockTestConfigSetRoot");++ monothreadedConfigSetTests(configSetLockFactory);+ multithreadedConfigSetTests(configSetLockFactory);+ }+ } finally {+ server.shutdown();+ }+ }++ private void monothreadedCollectionTests(DistributedCollectionLockFactory factory) {+ // Collection level locks+ DistributedLock collRL1 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertTrue("collRL1 should have been acquired", collRL1.isAcquired());++ DistributedLock collRL2 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertTrue("collRL1 should have been acquired", collRL2.isAcquired());++ DistributedLock collWL3 = factory.createLock(true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertFalse("collWL3 should not have been acquired, due to collRL1 and collRL2", collWL3.isAcquired());++ assertTrue("collRL2 should have been acquired, that should not have changed", collRL2.isAcquired());++ collRL1.release();+ collRL2.release();+ assertTrue("collWL3 should have been acquired, collRL1 and collRL2 were released", collWL3.isAcquired());++ DistributedLock collRL4 = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertFalse("collRL4 should not have been acquired, due to collWL3 locking the collection", collRL4.isAcquired());++ // Collection is write locked by collWL3 and collRL4 read lock waiting behind. Now moving to request shard level locks.+ // These are totally independent from the Collection level locks so should see no impact.+ DistributedLock shardWL5 = factory.createLock(true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null);+ assertTrue("shardWL5 should have been acquired, there is no lock on that shard", shardWL5.isAcquired());++ DistributedLock shardWL6 = factory.createLock(true, CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD_NAME, null);+ assertFalse("shardWL6 should not have been acquired, shardWL5 is locking that shard", shardWL6.isAcquired());++ // Get a lock on a Replica. Again this is independent of collection or shard level+ DistributedLock replicaRL7 = factory.createLock(false, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME);+ assertTrue("replicaRL7 should have been acquired", replicaRL7.isAcquired());++ DistributedLock replicaWL8 = factory.createLock(true, CollectionParams.LockLevel.REPLICA, COLLECTION_NAME, SHARD_NAME, REPLICA_NAME);+ assertFalse("replicaWL8 should not have been acquired, replicaRL7 is read locking that replica", replicaWL8.isAcquired());++ replicaRL7.release();+ assertTrue("replicaWL8 should have been acquired, as replicaRL7 got released", replicaWL8.isAcquired());+++ collWL3.release();+ assertTrue("collRL4 should have been acquired given collWL3 released", collRL4.isAcquired());+ shardWL5.release();+ assertTrue("shardWL6 should have been acquired, now that shardWL5 was released", shardWL6.isAcquired());++ replicaWL8.release();+ try {+ replicaWL8.isAcquired();+ fail("isAcquired() called after release() on a lock should have thrown exception");+ } catch (IllegalStateException ise) {+ // expected+ }++ // Releasing the collection lock used in the multithreaded phase+ collRL4.release();+ }++ private void multithreadedCollectionTests(DistributedCollectionLockFactory factory) throws Exception {+ // Acquiring right away a read lock+ DistributedLock readLock = factory.createLock(false, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertTrue("readLock should have been acquired", readLock.isAcquired());++ // And now creating a write lock, that can't be acquired just yet, because of the read lock+ DistributedLock writeLock = factory.createLock(true, CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertFalse("writeLock should not have been acquired", writeLock.isAcquired());++ // Wait for acquisition of the write lock on another thread (and be notified via a latch)+ final CountDownLatch latch = new CountDownLatch(1);+ new Thread(() -> {+ writeLock.waitUntilAcquired();+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind+ latch.countDown();+ }).start();++ // Wait for the thread to start and to get blocked in waitUntilAcquired()+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).+ Thread.sleep(30);++ assertEquals("we should not have been notified that writeLock was acquired", 1, latch.getCount());+ assertFalse("writeLock should not have been acquired", writeLock.isAcquired());++ readLock.release();+ assertTrue("writeLock should have been acquired now that readlock was released", writeLock.isAcquired());++ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause+ int i = 0;+ while (i < 1000 && latch.getCount() != 0) {+ Thread.sleep(10);+ i++;+ }+ assertEquals("we should have been notified that writeLock was acquired", 0, latch.getCount());+ }++ private void monothreadedConfigSetTests(DistributedConfigSetLockFactory factory) {+ DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME);+ assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired());++ DistributedLock configSetRL2 = factory.createLock(false, CONFIG_SET_NAME);+ assertTrue("configSetRL2 should have been acquired", configSetRL2.isAcquired());++ DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME);+ assertFalse("configSetWL1 should not have been acquired due to configSetRL1 and configSetRL2", configSetWL1.isAcquired());++ configSetRL1.release();+ configSetRL2.release();+ assertTrue("configSetWL1 should have been acquired, configSetRL1 and configSetRL2 were released", configSetWL1.isAcquired());++ DistributedLock configSetRL3 = factory.createLock(false, CONFIG_SET_NAME);+ assertFalse("configSetRL3 should not have been acquired due to configSetWL1", configSetRL3.isAcquired());++ configSetWL1.release();+ assertTrue("configSetRL3 should have been acquired, configSetWL1 was released", configSetRL3.isAcquired());++ configSetRL3.release();++ try {+ configSetRL3.isAcquired();+ fail("isAcquired() called after release() on a lock should have thrown exception");+ } catch (IllegalStateException ise) {+ // expected+ }+ }++ private void multithreadedConfigSetTests(DistributedConfigSetLockFactory factory) throws Exception {+ // Acquiring right away a read lock+ DistributedLock configSetRL1 = factory.createLock(false, CONFIG_SET_NAME);+ assertTrue("configSetRL1 should have been acquired", configSetRL1.isAcquired());++ // And now creating a write lock, that can't be acquired just yet, because of the read lock+ DistributedLock configSetWL1 = factory.createLock(true, CONFIG_SET_NAME);+ assertFalse("configSetWL1 should not have been acquired due to configSetRL1", configSetWL1.isAcquired());++ // Wait for acquisition of the write lock on another thread (and be notified via a latch)+ final CountDownLatch latch = new CountDownLatch(1);+ new Thread(() -> {+ configSetWL1.waitUntilAcquired();+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind+ latch.countDown();+ }).start();++ // Wait for the thread to start and to get blocked in waitUntilAcquired()+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).+ Thread.sleep(30);++ assertEquals("we should not have been notified that configSetWL1 was acquired", 1, latch.getCount());+ assertFalse("configSetWL1 should not have been acquired", configSetWL1.isAcquired());++ configSetRL1.release();+ assertTrue("configSetWL1 should have been acquired now that configSetRL1 was released", configSetWL1.isAcquired());++ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause+ int i = 0;+ while (i < 1000 && latch.getCount() != 0) {+ Thread.sleep(10);+ i++;+ }+ assertEquals("we should have been notified that configSetWL1 was acquired", 0, latch.getCount());+ }+}\ No newline at end of filediff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.javanew file mode 100644index 00000000000..e59ec4d0105--- /dev/null+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionApiLockingTest.java@@ -0,0 +1,149 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cloud.api.collections;++import java.util.concurrent.CountDownLatch;++import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.cloud.DistributedMultiLock;+import org.apache.solr.cloud.ZkDistributedCollectionLockFactory;+import org.apache.solr.cloud.ZkTestServer;+import org.apache.solr.common.cloud.SolrZkClient;+import org.apache.solr.common.params.CollectionParams;+import org.junit.Test;++public class CollectionApiLockingTest extends SolrTestCaseJ4 {+ private static final String COLLECTION_NAME = "lockColl";+ final String SHARD1_NAME = "lockShard1";+ final String SHARD2_NAME = "lockShard2";+ final String REPLICA_NAME = "lockReplica";++ static final int TIMEOUT = 10000;++ /**+ * Tests the Collection API locking. All tests run from single test method to save on setup time.+ */+ @Test+ public void monothreadedApiLockTests() throws Exception {+ ZkTestServer server = new ZkTestServer(createTempDir("zkData"));+ try {+ server.run();+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {+ CollectionApiLockFactory apiLockFactory = new CollectionApiLockFactory(new ZkDistributedCollectionLockFactory(zkClient, "/apiLockTestRoot"));++ monothreadedTests(apiLockFactory);+ multithreadedTests(apiLockFactory);+ }+ } finally {+ server.shutdown();+ }+ }++ private void monothreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception {+ // Lock at collection level (which prevents locking + acquiring on any other level of the hierarchy)+ DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertTrue("Collection should have been acquired", collLock.isAcquired());+ assertEquals("Lock at collection level expected to need one distributed lock", 1, collLock.getCountInternalLocks());++ // Request a shard lock. Will not be acquired as long as we don't release the collection lock above+ DistributedMultiLock shard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, null);+ assertFalse("Shard1 should not have been acquired", shard1Lock.isAcquired());+ assertEquals("Lock at shard level expected to need two distributed locks", 2, shard1Lock.getCountInternalLocks());++ // Request a lock on another shard. Will not be acquired as long as we don't release the collection lock above+ DistributedMultiLock shard2Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, null);+ assertFalse("Shard2 should not have been acquired", shard2Lock.isAcquired());++ assertTrue("Collection should still be acquired", collLock.isAcquired());++ collLock.release();++ assertTrue("Shard1 should have been acquired now that collection lock released", shard1Lock.isAcquired());+ assertTrue("Shard2 should have been acquired now that collection lock released", shard2Lock.isAcquired());++ // Request a lock on replica of shard1+ DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME);+ assertFalse("replicaShard1Lock should not have been acquired, shard1 is locked", replicaShard1Lock.isAcquired());++ // Now ask for a new lock on the collection+ collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);++ assertFalse("Collection should not have been acquired, shard1 and shard2 locks preventing it", collLock.isAcquired());++ shard1Lock.release();+ assertTrue("replicaShard1Lock should have been acquired, as shard1 got released", replicaShard1Lock.isAcquired());+ assertFalse("Collection should not have been acquired, shard2 lock is preventing it", collLock.isAcquired());++ replicaShard1Lock.release();++ // Request a lock on replica of shard2+ DistributedMultiLock replicaShard2Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD2_NAME, REPLICA_NAME);+ assertFalse("replicaShard2Lock should not have been acquired, shard2 is locked", replicaShard2Lock.isAcquired());++ shard2Lock.release();++ assertTrue("Collection should have been acquired as shard2 got released and replicaShard2Locks was requested after the collection lock", collLock.isAcquired());+ assertFalse("replicaShard2Lock should not have been acquired, collLock is locked", replicaShard2Lock.isAcquired());++ collLock.release();+ assertTrue("replicaShard2Lock should have been acquired, the collection lock got released", replicaShard2Lock.isAcquired());++ // Release remaining lock to allow the multithreaded locking to succeed+ replicaShard2Lock.release();+ }++ private void multithreadedTests(CollectionApiLockFactory apiLockingHelper) throws Exception {+ // Lock on collection...+ DistributedMultiLock collLock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.COLLECTION, COLLECTION_NAME, null, null);+ assertTrue("Collection should have been acquired", collLock.isAcquired());++ // ...blocks a lock on replica from being acquired+ final DistributedMultiLock replicaShard1Lock = apiLockingHelper.createCollectionApiLock(CollectionParams.LockLevel.SHARD, COLLECTION_NAME, SHARD1_NAME, REPLICA_NAME);+ assertFalse("replicaShard1Lock should not have been acquired, because collection is locked", replicaShard1Lock.isAcquired());++ // Wait for acquisition of the replica lock on another thread (and be notified via a latch)+ final CountDownLatch latch = new CountDownLatch(1);+ new Thread(() -> {+ replicaShard1Lock.waitUntilAcquired();+ // countDown() will not be called if waitUntilAcquired() threw exception of any kind+ latch.countDown();+ }).start();++ // Wait for the thread to start and to get blocked in waitUntilAcquired()+ // (thread start could have been checked more reliably using another latch, and verifying the thread is in waitUntilAcquired+ // done through that thread stacktrace, but that would be overkill compared to the very slight race condition of waiting 30ms,+ // but a race that would not cause the test to fail since we're testing... that nothing happened yet).+ Thread.sleep(30);++ assertEquals("we should not have been notified that replica was acquired", 1, latch.getCount());+ assertFalse("replica lock should not have been acquired", replicaShard1Lock.isAcquired());++ collLock.release();+ assertTrue("replica lock should have been acquired now that collection lock was released", replicaShard1Lock.isAcquired());++ // Wait for the Zookeeper watch to fire + the thread to be unblocked and countdown the latch+ // We'll wait up to 10 seconds here, so should be safe even if GC is extraordinarily high with a pause+ int i = 0;+ while (i < 1000 && latch.getCount() != 0) {+ Thread.sleep(10);+ i++;+ }+ assertEquals("we should have been notified that replica lock was acquired", 0, latch.getCount());+ }++}diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.javaindex a9fdcaee150..21dd9dbea67 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java@@ -55,7 +55,7 @@ import org.slf4j.LoggerFactory;public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {private static final int MAX_TIMEOUT_SECONDS = 90;-+private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@Before@@ -65,7 +65,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")).configure();}-+@Afterpublic void tearDown() throws Exception {super.tearDown();@@ -107,9 +107,9 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {.processAndWait(client, MAX_TIMEOUT_SECONDS);assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state);-+cluster.waitForActiveCollection(collection, 1, 1);-+//Add a few documents to shard1int numDocs = TestUtil.nextInt(random(), 10, 100);List<SolrInputDocument> docs = new ArrayList<>(numDocs);@@ -135,7 +135,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {assertSame("CreateShard did not complete", RequestStatusState.COMPLETED, state);client.getZkStateReader().forceUpdateCollection(collection);-+//Add a doc to shard2 to make sure shard2 was created properlySolrInputDocument doc = new SolrInputDocument();doc.addField("id", numDocs + 1);@@ -187,7 +187,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {} catch (SolrException e) {//expected}-+Slice shard1 = client.getZkStateReader().getClusterState().getCollection(collection).getSlice("shard1");Replica replica = shard1.getReplicas().iterator().next();for (String liveNode : client.getZkStateReader().getClusterState().getLiveNodes()) {@@ -199,7 +199,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {}}client.getZkStateReader().forceUpdateCollection(collection);-+shard1 = client.getZkStateReader().getClusterState().getCollection(collection).getSlice("shard1");String replicaName = shard1.getReplicas().iterator().next().getName();state = CollectionAdminRequest.deleteReplica(collection, "shard1", replicaName)@@ -223,17 +223,17 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {.setShards("shard1").processAndWait(cluster.getSolrClient(), MAX_TIMEOUT_SECONDS);assertSame("CreateCollection task did not complete!", RequestStatusState.COMPLETED, state);-+int numThreads = 10;final AtomicInteger numSuccess = new AtomicInteger(0);final AtomicInteger numFailure = new AtomicInteger(0);final CountDownLatch latch = new CountDownLatch(numThreads);-+ExecutorService es = ExecutorUtil.newMDCAwareFixedThreadPool(numThreads, new SolrNamedThreadFactory("testAsyncIdRaceCondition"));try {for (int i = 0; i < numThreads; i++) {es.submit(new Runnable() {-+@Overridepublic void run() {CollectionAdminRequest.Reload reloadCollectionRequest = CollectionAdminRequest.reloadCollection("testAsyncIdRaceCondition");@@ -243,7 +243,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {} catch (InterruptedException e) {throw new RuntimeException();}-+try {if (log.isInfoEnabled()) {log.info("{} - Reloading Collection.", Thread.currentThread().getName());@@ -254,7 +254,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {if (log.isInfoEnabled()) {log.info("Exception during collection reloading, we were waiting for one: ", e);}- assertEquals("Task with the same requestid already exists.", e.getMessage());+ assertEquals("Task with the same requestid already exists. (repeatedId)", e.getMessage());numFailure.incrementAndGet();} catch (IOException e) {throw new RuntimeException();diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.javaindex 884ea222fa0..90e36c5158c 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java@@ -62,7 +62,9 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {@Test@ShardsFixed(num = 2)public void test() throws Exception {+ final boolean isDistributedCollectionApi;try (CloudSolrClient client = createCloudClient(null)) {+ isDistributedCollectionApi = new CollectionAdminRequest.RequestApiDistributedProcessing().process(client).getIsCollectionApiDistributed();CollectionAdminRequest.Create req;if (useTlogReplicas()) {req = CollectionAdminRequest.createCollection(COLLECTION_NAME, "conf1",2, 0, 1, 1);@@ -85,7 +87,9 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {clusterStatusWithCollectionAndMultipleShards();clusterStatusWithRouteKey();clusterStatusAliasTest();- clusterStatusRolesTest();+ if (!isDistributedCollectionApi) {+ clusterStatusRolesTest();+ }clusterStatusBadCollectionTest();replicaPropTest();clusterStatusZNodeVersion();diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.javaindex dc49620c01e..97aecf07930 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestRequestStatusCollectionAPI.java@@ -164,7 +164,7 @@ public class TestRequestStatusCollectionAPI extends BasicDistributedZkTest {e.printStackTrace();}- assertEquals("Task with the same requestid already exists.", r.get("error"));+ assertEquals("Task with the same requestid already exists. (1002)", r.get("error"));}@SuppressWarnings("unchecked")diff --git a/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java b/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.javaindex f0ce7ef5a9a..eb97ea7cf11 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/TestConfigsApi.java@@ -42,7 +42,7 @@ public class TestConfigsApi extends SolrTestCaseJ4 {}@Override- protected void sendToZk(SolrQueryResponse rsp,+ protected void sendToOverseer(SolrQueryResponse rsp,ConfigSetOperation operation,Map<String, Object> result) {result.put(QUEUE_OPERATION, operation.action.toLower());diff --git a/solr/server/solr/solr.xml b/solr/server/solr/solr.xmlindex 0fc94e24473..8c73d838fc1 100644--- a/solr/server/solr/solr.xml+++ b/solr/server/solr/solr.xml@@ -46,6 +46,7 @@<str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str><str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str><bool name="distributedClusterStateUpdates">${distributedClusterStateUpdates:false}</bool>+ <bool name="distributedCollectionConfigSetExecution">${distributedCollectionConfigSetExecution:false}</bool></solrcloud>diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.javaindex 7d1f977f24d..607a5a8f915 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java@@ -848,14 +848,14 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>}/**- * Return a SolrRequest for low-level detailed status of the specified collection.+ * Return a SolrRequest for low-level detailed status of the specified collection.* @param collection the collection to get the status of.*/public static ColStatus collectionStatus(String collection) {checkNotNull(CoreAdminParams.COLLECTION, collection);return new ColStatus(collection);}-+/*** Return a SolrRequest for low-level detailed status of all collections on the cluster.*/@@ -878,7 +878,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>super(CollectionAction.COLSTATUS);this.collection = collection;}-+private ColStatus() {super(CollectionAction.COLSTATUS);}@@ -1340,6 +1340,35 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>}+ /**+ * Returns a SolrRequest to run a mock task. For tests only.+ */+ public static MockCollTask mockCollTask(String collection) {+ return new MockCollTask(collection);+ }++ // MOCK_COLL_TASK request+ public static class MockCollTask extends AsyncCollectionAdminRequest {+ protected final String collection;+ protected String sleep;++ private MockCollTask(String collection) {+ super(CollectionAction.MOCK_COLL_TASK);+ this.collection = checkNotNull(CoreAdminParams.COLLECTION, collection);++ }++ public MockCollTask setSleep(String sleep) { this.sleep = sleep; return this; }++ @Override+ public SolrParams getParams() {+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();+ params.add(CoreAdminParams.COLLECTION, collection);+ params.setNonNull("sleep", sleep);+ return params;+ }+ }+/*** Returns a SolrRequest to split a shard in a collection*/@@ -2605,6 +2634,30 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>}+ // DISTRIBUTEDAPIPROCESSING request for verifying if Collection API execution is distributed+ public static class RequestApiDistributedProcessing extends CollectionAdminRequest<RequestApiDistributedProcessingResponse> {++ public RequestApiDistributedProcessing() {+ super(CollectionAction.DISTRIBUTEDAPIPROCESSING);+ }++ @Override+ protected RequestApiDistributedProcessingResponse createResponse(SolrClient client) {+ return new RequestApiDistributedProcessingResponse();+ }+ }++ /**+ * A response object for {@link RequestApiDistributedProcessing} requests+ */+ public static class RequestApiDistributedProcessingResponse extends CollectionAdminResponse {+ public boolean getIsCollectionApiDistributed() {+ @SuppressWarnings({"rawtypes"})+ Boolean isDistributedApi = (Boolean) getResponse().get("isDistributedApi");+ return isDistributedApi;+ }+ }+/*** Return a SolrRequest to get the Cluster status*/diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.javaindex 1ffa05e7980..b63690fbd8b 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java@@ -104,11 +104,13 @@ public interface CollectionParams {ADDREPLICA(true, LockLevel.SHARD),MOVEREPLICA(true, LockLevel.SHARD),OVERSEERSTATUS(false, LockLevel.NONE),+ // For testing. Could eventually be exposed publicly if needed+ DISTRIBUTEDAPIPROCESSING(false, LockLevel.NONE),LIST(false, LockLevel.NONE),CLUSTERSTATUS(false, LockLevel.NONE),ADDREPLICAPROP(true, LockLevel.REPLICA),DELETEREPLICAPROP(true, LockLevel.REPLICA),- BALANCESHARDUNIQUE(true, LockLevel.SHARD),+ BALANCESHARDUNIQUE(true, LockLevel.COLLECTION),REBALANCELEADERS(true, LockLevel.COLLECTION),MODIFYCOLLECTION(true, LockLevel.COLLECTION),BACKUP(true, LockLevel.COLLECTION),@@ -143,7 +145,7 @@ public interface CollectionParams {CollectionAction(boolean isWrite, LockLevel level) {this.isWrite = isWrite;this.lockLevel = level;- lowerName = toString().toLowerCase(Locale.ROOT);+ lowerName = name().toLowerCase(Locale.ROOT);}public static CollectionAction get(String p) {diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex f14b6f2b1da..79876b6f0ff 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -121,6 +121,7 @@ public class MiniSolrCloudCluster {" <str name=\"pkiHandlerPrivateKeyPath\">${pkiHandlerPrivateKeyPath:cryptokeys/priv_key512_pkcs8.pem}</str> \n" +" <str name=\"pkiHandlerPublicKeyPath\">${pkiHandlerPublicKeyPath:cryptokeys/pub_key512.der}</str> \n" +" <str name=\"distributedClusterStateUpdates\">${solr.distributedClusterStateUpdates:false}</str> \n" ++ " <str name=\"distributedCollectionConfigSetExecution\">${solr.distributedCollectionConfigSetExecution:false}</str> \n" +" </solrcloud>\n" +// NOTE: this turns off the metrics collection unless overriden by a sysprop" <metrics enabled=\"${metricsEnabled:false}\">\n" +diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.javaindex afb6d4dc339..08cce5d0dda 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java@@ -109,6 +109,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {private Map<String, Object> clusterProperties = new HashMap<>();private boolean trackJettyMetrics;+ private boolean useDistributedCollectionConfigSetExecution;private boolean useDistributedClusterStateUpdate;/**@@ -120,8 +121,11 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {public Builder(int nodeCount, Path baseDir) {this.nodeCount = nodeCount;this.baseDir = baseDir;- // By default the MiniSolrCloudCluster being built will randomly (seed based) decide which cluster update strategy to use- this.useDistributedClusterStateUpdate = LuceneTestCase.random().nextInt(2) == 0;+ // By default the MiniSolrCloudCluster being built will randomly (seed based) decide which collection API strategy+ // to use (distributed or Overseer based) and which cluster update strategy to use (distributed if collection API+ // is distributed, but Overseer based or distributed randomly chosen if Collection API is Overseer based)+ this.useDistributedCollectionConfigSetExecution = LuceneTestCase.random().nextInt(2) == 0;+ this.useDistributedClusterStateUpdate = useDistributedCollectionConfigSetExecution || LuceneTestCase.random().nextInt(2) == 0;}/**@@ -190,21 +194,25 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {}/**- * This method makes the MiniSolrCloudCluster use the "other" cluster state update strategy than it normally would.+ * This method makes the MiniSolrCloudCluster use the "other" Collection API execution strategy than it normally would.* When some test classes call this method (and some don't) we make sure that a run of multiple tests with a single- * seed will exercise both code lines (distributed updates and Overseer based updates) so regressions can be spotted+ * seed will exercise both code lines (distributed and Overseer based Collection API) so regressions can be spotted* faster.<p>** The real need is for a few tests covering reasonable use cases to call this method. If you're adding a new test,* you don't have to call it (but it's ok if you do).*/- public Builder useOtherClusterStateUpdateStrategy() {- useDistributedClusterStateUpdate = !useDistributedClusterStateUpdate;+ public Builder useOtherCollectionConfigSetExecution() {+ // Switch from Overseer to distributed Collection execution and vice versa+ useDistributedCollectionConfigSetExecution = !useDistributedCollectionConfigSetExecution;+ // Reverse distributed cluster state updates as well if possible (state can't be Overseer based if Collections API is distributed)+ useDistributedClusterStateUpdate = !useDistributedClusterStateUpdate || useDistributedCollectionConfigSetExecution;return this;}/**- * Force the cluster state update strategy to be either Overseer based or distributed. <b>This method can be useful when+ * Force the cluster Collection and config state API execution as well as the cluster state update strategy to be+ * either Overseer based or distributed. <b>This method can be useful when* debugging tests</b> failing in only one of the two modes to have all local runs exhibit the issue, as well obviously for* tests that are not compatible with one of the two modes.* <p>@@ -215,19 +223,24 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {* <solrcloud>* ....* <str name="distributedClusterStateUpdates">${solr.distributedClusterStateUpdates:false}</str>+ * <str name="distributedCollectionConfigSetExecution">${solr.distributedCollectionConfigSetExecution:false}</str>* ....* </solrcloud>* }</pre>* For an example of a configuration supporting this setting, see {@link MiniSolrCloudCluster#DEFAULT_CLOUD_SOLR_XML}.* When a test sets a different {@code solr.xml} config (using {@link #withSolrXml}), if the config does not contain- * the placeholder, the strategy will be defined by the value assigned to {@code useDistributedClusterStateUpdates}- * in {@link org.apache.solr.core.CloudConfig.CloudConfigBuilder}.+ * the placeholder, the strategy will be defined by the values assigned to {@code useDistributedClusterStateUpdates}+ * and {@code useDistributedCollectionConfigSetExecution} in {@link org.apache.solr.core.CloudConfig.CloudConfigBuilder}.*- * @param distributed When {@code true}, cluster state updates are handled in a distributed way by nodes. When- * {@code false}, cluster state updates are handled by Overseer.+ * @param distributedCollectionConfigSetApi When {@code true}, Collection and Config Set API commands are executed in+ * a distributed way by nodes. When {@code false}, they are executed by Overseer.+ * @param distributedClusterStateUpdates When {@code true}, cluster state updates are handled in a distributed way by nodes. When+ * {@code false}, cluster state updates are handled by Overseer.<p>+ * If {@code distributedCollectionConfigSetApi} is {@code true} then this parameter must be {@code true}.*/- public Builder withDistributedClusterStateUpdates(boolean distributed) {- useDistributedClusterStateUpdate = distributed;+ public Builder withDistributedClusterStateUpdates(boolean distributedCollectionConfigSetApi, boolean distributedClusterStateUpdates) {+ useDistributedCollectionConfigSetExecution = distributedCollectionConfigSetApi;+ useDistributedClusterStateUpdate = distributedClusterStateUpdates;return this;}@@ -262,12 +275,13 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {* @throws Exception if an error occurs on startup*/public MiniSolrCloudCluster build() throws Exception {- // This will have an impact on how the MiniSolrCloudCluster and therefore the test run if the config being- // used does have the appropriate placeholder.- // It is a good place to hard code true or false instead of useDistributedClusterStateUpdate to run all qualifying- // tests with a given cluster state update strategy (non qualifying tests will use the default value assigned to- // useDistributedClusterStateUpdates in org.apache.solr.core.CloudConfig.CloudConfigBuilder, so if you really want- // ALL tests to run with a given strategy, patch it there too (and revert before commit!)+ // Two lines below will have an impact on how the MiniSolrCloudCluster and therefore the test run if the config being+ // used in the test does have the appropriate placeholders. See for example DEFAULT_CLOUD_SOLR_XML in MiniSolrCloudCluster.+ // Hard coding values here will impact such tests.+ // To hard code behavior for tests not having these placeholders - and for SolrCloud as well for that matter! -+ // change the values assigned to useDistributedClusterStateUpdates and useDistributedCollectionConfigSetExecution in+ // org.apache.solr.core.CloudConfig.CloudConfigBuilder. Do not forget then to revert before commit!+ System.setProperty("solr.distributedCollectionConfigSetExecution", Boolean.toString(useDistributedCollectionConfigSetExecution));System.setProperty("solr.distributedClusterStateUpdates", Boolean.toString(useDistributedClusterStateUpdate));JettyConfig jettyConfig = jettyConfigBuilder.build();