Apache Solr open-source search software
SOLR-14964: remove Autoscaling related withCollection and COLOCATED_WITH (#2034)
Ilan Ginzburgb649f3f40d08
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.javaindex ca5639f4989..9897f078953 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/AddReplicaCmd.java@@ -28,7 +28,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;-import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonAdminParams.TIMEOUT;@@ -61,7 +60,6 @@ import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.CollectionAdminParams;-import org.apache.solr.common.params.CommonAdminParams;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.ShardParams;@@ -202,29 +200,6 @@ public class AddReplicaCmd implements OverseerCollectionMessageHandler.Cmd {}private ModifiableSolrParams getReplicaParams(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results, String collectionName, DocCollection coll, boolean skipCreateReplicaInClusterState, String asyncId, ShardHandler shardHandler, CreateReplica createReplica) throws IOException, InterruptedException, KeeperException {- if (coll.getStr(WITH_COLLECTION) != null) {- String withCollectionName = coll.getStr(WITH_COLLECTION);- DocCollection withCollection = clusterState.getCollection(withCollectionName);- if (withCollection.getActiveSlices().size() > 1) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + withCollection.getActiveSlices().size());- }- String withCollectionShard = withCollection.getActiveSlices().iterator().next().getName();-- List<Replica> replicas = withCollection.getReplicas(createReplica.node);- if (replicas == null || replicas.isEmpty()) {- // create a replica of withCollection on the identified node before proceeding further- ZkNodeProps props = new ZkNodeProps(- Overseer.QUEUE_OPERATION, ADDREPLICA.toString(),- ZkStateReader.COLLECTION_PROP, withCollectionName,- ZkStateReader.SHARD_ID_PROP, withCollectionShard,- "node", createReplica.node,- // since we already computed node assignments (which include assigning a node for this withCollection replica) we want to skip the assignment step- CollectionAdminParams.SKIP_NODE_ASSIGNMENT, "true",- CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created- addReplica(clusterState, props, results, null);- }- }-ModifiableSolrParams params = new ModifiableSolrParams();ZkStateReader zkStateReader = ocmh.zkStateReader;diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.javaindex d3567880bad..c938d7573f5 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java@@ -30,7 +30,6 @@ import java.util.Map;import java.util.NoSuchElementException;import java.util.Properties;import java.util.concurrent.TimeUnit;-import java.util.concurrent.TimeoutException;import org.apache.solr.client.solrj.cloud.AlreadyExistsException;import org.apache.solr.client.solrj.cloud.BadVersionException;@@ -78,9 +77,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;-import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MODIFYCOLLECTION;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;import static org.apache.solr.common.params.CommonParams.NAME;@@ -118,21 +115,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmdthrow new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection alias already exists: " + collectionName);}- String withCollection = message.getStr(CollectionAdminParams.WITH_COLLECTION);- String withCollectionShard = null;- if (withCollection != null) {- String realWithCollection = aliases.resolveSimpleAlias(withCollection);- if (!clusterState.hasCollection(realWithCollection)) {- throw new SolrException(ErrorCode.BAD_REQUEST, "The 'withCollection' does not exist: " + realWithCollection);- } else {- DocCollection collection = clusterState.getCollection(realWithCollection);- if (collection.getActiveSlices().size() > 1) {- throw new SolrException(ErrorCode.BAD_REQUEST, "The `withCollection` must have only one shard, found: " + collection.getActiveSlices().size());- }- withCollectionShard = collection.getActiveSlices().iterator().next().getName();- }- }-String configName = getConfigName(collectionName, message);if (configName == null) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");@@ -209,22 +191,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmdfor (ReplicaPosition replicaPosition : replicaPositions) {String nodeName = replicaPosition.node;- if (withCollection != null) {- // check that we have a replica of `withCollection` on this node and if not, create one- DocCollection collection = clusterState.getCollection(withCollection);- List<Replica> replicas = collection.getReplicas(nodeName);- if (replicas == null || replicas.isEmpty()) {- ZkNodeProps props = new ZkNodeProps(- Overseer.QUEUE_OPERATION, ADDREPLICA.toString(),- ZkStateReader.COLLECTION_PROP, withCollection,- ZkStateReader.SHARD_ID_PROP, withCollectionShard,- "node", nodeName,- CommonAdminParams.WAIT_FOR_FINAL_STATE, Boolean.TRUE.toString()); // set to true because we want `withCollection` to be ready after this collection is created- new AddReplicaCmd(ocmh).call(clusterState, props, results);- clusterState = zkStateReader.getClusterState(); // refresh- }- }-String coreName = Assign.buildSolrCoreName(ocmh.cloudManager.getDistribStateManager(),ocmh.cloudManager.getClusterStateProvider().getClusterState().getCollection(collectionName),replicaPosition.shard, replicaPosition.type, true);@@ -309,22 +275,6 @@ public class CreateCollectionCmd implements OverseerCollectionMessageHandler.Cmd}}- // modify the `withCollection` and store this new collection's name with it- if (withCollection != null) {- ZkNodeProps props = new ZkNodeProps(- Overseer.QUEUE_OPERATION, MODIFYCOLLECTION.toString(),- ZkStateReader.COLLECTION_PROP, withCollection,- CollectionAdminParams.COLOCATED_WITH, collectionName);- ocmh.overseer.offerStateUpdate(Utils.toJSON(props));- try {- zkStateReader.waitForState(withCollection, 5, TimeUnit.SECONDS, (collectionState) -> collectionName.equals(collectionState.getStr(COLOCATED_WITH)));- } catch (TimeoutException e) {- log.warn("Timed out waiting to see the {} property set on collection: {}", COLOCATED_WITH, withCollection);- // maybe the overseer queue is backed up, we don't want to fail the create request- // because of this time out, continue- }- }-// create an alias pointing to the new collection, if different from the collectionNameif (!alias.equals(collectionName)) {ocmh.zkStateReader.aliasesManager.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(alias, collectionName));diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.javaindex 5c080578790..d9b66799aa7 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java@@ -52,9 +52,7 @@ import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;-import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonParams.NAME;@@ -94,8 +92,6 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmdcollection = extCollection;}- checkNotColocatedWith(zkStateReader, collection);-final boolean deleteHistory = message.getBool(CoreAdminParams.DELETE_METRICS_HISTORY, true);boolean removeCounterNode = true;@@ -262,21 +258,4 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd.map(Map.Entry::getKey) // alias name.collect(Collectors.toList());}-- private void checkNotColocatedWith(ZkStateReader zkStateReader, String collection) throws Exception {- DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collection);- if (docCollection != null) {- String colocatedWith = docCollection.getStr(COLOCATED_WITH);- if (colocatedWith != null) {- DocCollection colocatedCollection = zkStateReader.getClusterState().getCollectionOrNull(colocatedWith);- if (colocatedCollection != null && collection.equals(colocatedCollection.getStr(WITH_COLLECTION))) {- // todo how do we clean up if reverse-link is not present?- // can't delete this collection because it is still co-located with another collection- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,- "Collection: " + collection + " is co-located with collection: " + colocatedWith- + " remove the link using modify collection API or delete the co-located collection: " + colocatedWith);- }- }- }- }}diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.javaindex f567b2ef2ca..451bfad7cda 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java@@ -33,7 +33,6 @@ import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CollectionParams;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.util.NamedList;@@ -126,26 +125,6 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {replica = sliceReplicas.iterator().next();}- if (coll.getStr(CollectionAdminParams.COLOCATED_WITH) != null) {- // we must ensure that moving this replica does not cause the co-location to break- String sourceNode = replica.getNodeName();- String colocatedCollectionName = coll.getStr(CollectionAdminParams.COLOCATED_WITH);- DocCollection colocatedCollection = clusterState.getCollectionOrNull(colocatedCollectionName);- if (colocatedCollection != null) {- if (colocatedCollection.getReplica((s, r) -> sourceNode.equals(r.getNodeName())) != null) {- // check if we have at least two replicas of the collection on the source node- // only then it is okay to move one out to another node- List<Replica> replicasOnSourceNode = coll.getReplicas(replica.getNodeName());- if (replicasOnSourceNode == null || replicasOnSourceNode.size() < 2) {- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,- "Collection: " + collection + " is co-located with collection: " + colocatedCollectionName- + " and has a single replica: " + replica.getName() + " on node: " + replica.getNodeName()- + " so it is not possible to move it to another node");- }- }- }- }-log.info("Replica will be moved to node {}: {}", targetNode, replica);Slice slice = null;for (Slice s : coll.getSlices()) {diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.javaindex 32c7e132fc9..42ee53d6599 100644--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java@@ -102,8 +102,6 @@ import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;-import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;-import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonParams.NAME;@@ -143,9 +141,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,ZkStateReader.REPLICATION_FACTOR, "1",ZkStateReader.NRT_REPLICAS, "1",ZkStateReader.TLOG_REPLICAS, "0",- ZkStateReader.PULL_REPLICAS, "0",- WITH_COLLECTION, null,- COLOCATED_WITH, null));+ ZkStateReader.PULL_REPLICAS, "0"));private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());public static final String FAILURE_FIELD = "failure";diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex 3e9f443870b..9242d6c2816 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -136,7 +136,6 @@ import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASESimport static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_NAME;import static org.apache.solr.common.params.CollectionAdminParams.PROPERTY_VALUE;import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT;-import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;import static org.apache.solr.common.params.CommonAdminParams.ASYNC;import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;@@ -463,7 +462,6 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionTLOG_REPLICAS,NRT_REPLICAS,WAIT_FOR_FINAL_STATE,- WITH_COLLECTION,ALIAS);if (props.get(REPLICATION_FACTOR) != null && props.get(NRT_REPLICAS) != null) {diff --git a/solr/dev-docs/overseer/overseer.adoc b/solr/dev-docs/overseer/overseer.adocindex 51f8116ee61..6929270a6de 100644--- a/solr/dev-docs/overseer/overseer.adoc+++ b/solr/dev-docs/overseer/overseer.adoc@@ -531,7 +531,7 @@ Then finally the real work is done! The (deprecated) http shard handler is usedIt ends up executing `CoreAdminOperation.CREATE_OP` that calls `CoreContainer.create()` for the core of the replica.-The rest of the method deals with failures, with `withCollection` and with `alias`.+The rest of the method deals with failures with `alias`.[[bug9]][WARNING]diff --git a/solr/solr-ref-guide/src/collection-management.adoc b/solr/solr-ref-guide/src/collection-management.adocindex 6e900f9bf38..c1d3af0e1ac 100644--- a/solr/solr-ref-guide/src/collection-management.adoc+++ b/solr/solr-ref-guide/src/collection-management.adoc@@ -225,7 +225,6 @@ The attributes that can be modified are:* rule* snitch* policy-* withCollection* readOnly* other custom properties that use a `property.` prefixdiff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex b6204c8dca0..503a30bf13c 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -111,7 +111,7 @@ _(raw; not yet edited)_* SOLR-14656: Autoscaling framework removedThis includes:Autoscaling, policy, triggers etc.- withCollection handling+ withCollection handling (SOLR-14964)UTILIZENODE commandSim frameworkSuggestions tab in UIdiff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.javaindex b323ae627c1..cba6a079282 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java@@ -44,7 +44,6 @@ import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.rule.ImplicitSnitch;import org.apache.solr.common.cloud.rule.SnitchContext;-import org.apache.solr.common.params.CollectionAdminParams;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;@@ -74,7 +73,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterprivate Map<String, Object> snitchSession = new HashMap<>();@SuppressWarnings({"rawtypes"})private Map<String, Map> nodeVsTags = new HashMap<>();- private Map<String, String> withCollectionsMap = new HashMap<>();public SolrClientNodeStateProvider(CloudSolrClient solrClient) {this.solrClient = solrClient;@@ -100,9 +98,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterall.forEach((collName, ref) -> {DocCollection coll = ref.get();if (coll == null) return;- if (coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION) != null) {- withCollectionsMap.put(coll.getName(), (String) coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION));- }coll.forEachReplica((shard, replica) -> {Map<String, Map<String, List<Replica>>> nodeData = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());Map<String, List<Replica>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.javaindex dbd580184c0..44454e0e146 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java@@ -61,13 +61,11 @@ import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;import static org.apache.solr.common.cloud.ZkStateReader.TLOG_REPLICAS;import static org.apache.solr.common.params.CollectionAdminParams.ALIAS;import static org.apache.solr.common.params.CollectionAdminParams.COLL_CONF;-import static org.apache.solr.common.params.CollectionAdminParams.COLOCATED_WITH;import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_PARAM;import static org.apache.solr.common.params.CollectionAdminParams.CREATE_NODE_SET_SHUFFLE_PARAM;import static org.apache.solr.common.params.CollectionAdminParams.ROUTER_PREFIX;import static org.apache.solr.common.params.CollectionAdminParams.SKIP_NODE_ASSIGNMENT;-import static org.apache.solr.common.params.CollectionAdminParams.WITH_COLLECTION;/*** This class is experimental and subject to change.@@ -82,8 +80,6 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>public static final java.util.List<String> MODIFIABLE_COLLECTION_PROPERTIES = Arrays.asList(REPLICATION_FACTOR,COLL_CONF,- WITH_COLLECTION,- COLOCATED_WITH,READ_ONLY);protected final CollectionAction action;diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.javaindex 3fc11616f37..fee3b7ad8c6 100644--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java@@ -80,17 +80,6 @@ public interface CollectionAdminParams {*/String COLL_CONF = "collection.configName";- /**- * The name of the collection with which a collection is to be co-located- */- String WITH_COLLECTION = "withCollection";-- /**- * The reverse-link to WITH_COLLECTION flag. It is stored in the cluster state of the `withCollection`- * and points to the collection on which the `withCollection` was specified.- */- String COLOCATED_WITH = "COLOCATED_WITH";-/*** Used by cluster properties API as a wrapper key to provide defaults for collection, cluster etc.*