Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15694: Node roles

Ishan Chattopadhyayae32e58c20be0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c0147577cb9..62cf2f1caad 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -68,6 +68,8 @@ New Features
* SOLR-14608: Faster sorting for the /export handler (Joel Bernstein)
+* SOLR-15694: Node roles (Ishan Chattopadhyaya, noble)
+
Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
index c02403e8a3a..2a02562bce7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
@@ -17,15 +17,19 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.solr.client.solrj.impl.ZkDistribStateManager;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.NodeRoles;
+import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
@@ -59,11 +63,26 @@ public class OverseerNodePrioritizer {
public synchronized void prioritizeOverseerNodes(String overseerId) throws Exception {
SolrZkClient zk = zkStateReader.getZkClient();
- if(!zk.exists(ZkStateReader.ROLES,true))return;
- Map<?,?> m = (Map<?,?>) Utils.fromJSON(zk.getData(ZkStateReader.ROLES, null, new Stat(), true));
+ List<String> overseerDesignates = new ArrayList<>();
+ if (zk.exists(ZkStateReader.ROLES,true)) {
+ Map<?,?> m = (Map<?,?>) Utils.fromJSON(zk.getData(ZkStateReader.ROLES, null, new Stat(), true));
+ @SuppressWarnings("unchecked")
+ List<String> l = (List<String>) m.get("overseer");
+ if (l != null) {
+ overseerDesignates.addAll(l);
+ }
+ }
- List<?> overseerDesignates = (List<?>) m.get("overseer");
- if(overseerDesignates==null || overseerDesignates.isEmpty()) return;
+ List<String> preferredOverseers = ClusterAPI.getNodesByRole(NodeRoles.Role.OVERSEER, NodeRoles.MODE_PREFERRED,
+ new ZkDistribStateManager(zkStateReader.getZkClient()));
+ for (String preferred: preferredOverseers) {
+ if (overseerDesignates.contains(preferred)) {
+ log.warn("Node {} has been configured to be a preferred overseer using both ADDROLE API command " +
+ "as well as using Node Roles (i.e. -Dsolr.node.roles start up property). Only the latter is recommended.", preferred);
+ }
+ }
+ overseerDesignates.addAll(preferredOverseers);
+ if (overseerDesignates.isEmpty()) return;
String ldr = OverseerTaskProcessor.getLeaderNode(zk);
if(overseerDesignates.contains(ldr)) return;
log.info("prioritizing overseer nodes at {} overseer designates are {}", overseerId, overseerDesignates);
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 75a4e8382ac..723a49fedc1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -64,7 +64,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.*;
import org.apache.solr.common.cloud.Replica.Type;
-import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -79,6 +78,7 @@ import org.apache.solr.core.CloudConfig;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.core.SolrCoreInitializationException;
import org.apache.solr.handler.component.HttpShardHandler;
import org.apache.solr.logging.MDCLoggingContext;
@@ -105,6 +105,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE;
/**
@@ -357,7 +358,9 @@ public class ZkController implements Closeable {
overseerElector.setup(context);
- overseerElector.joinElection(context, true);
+ if (cc.nodeRoles.isOverseerAllowedOrPreferred()) {
+ overseerElector.joinElection(context, true);
+ }
}
cc.cancelCoreRecoveries();
@@ -854,6 +857,14 @@ public class ZkController implements Closeable {
throws KeeperException, InterruptedException, IOException {
ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient);
+ cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES, zkClient);
+ for (NodeRoles.Role role: NodeRoles.Role.values()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRole(role), zkClient);
+ for (String mode: role.supportedModes()) {
+ cmdExecutor.ensureExists(NodeRoles.getZNodeForRoleMode(role, mode), zkClient);
+ }
+ }
+
cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient);
cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient);
byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8);
@@ -910,10 +921,11 @@ public class ZkController implements Closeable {
overseerElector = new LeaderElector(zkClient);
this.overseer = new Overseer((HttpShardHandler) cc.getShardHandlerFactory().getShardHandler(), cc.getUpdateShardHandler(),
CommonParams.CORES_HANDLER_PATH, zkStateReader, this, cloudConfig);
- ElectionContext context = new OverseerElectionContext(zkClient,
- overseer, getNodeName());
+ ElectionContext context = new OverseerElectionContext(zkClient, overseer, getNodeName());
overseerElector.setup(context);
- overseerElector.joinElection(context, false);
+ if (cc.nodeRoles.isOverseerAllowedOrPreferred()) {
+ overseerElector.joinElection(context, false);
+ }
}
Stat stat = zkClient.exists(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
@@ -1086,6 +1098,11 @@ public class ZkController implements Closeable {
log.info("Register node as live in ZooKeeper:{}", nodePath);
List<Op> ops = new ArrayList<>(2);
ops.add(Op.create(nodePath, null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL));
+
+ // Create the roles node as well
+ cc.nodeRoles.getRoles().forEach((role, mode) -> ops.add(Op.create(NodeRoles.getZNodeForRoleMode(role, mode) +"/" + nodeName,
+ null, zkClient.getZkACLProvider().getACLsToAdd(nodePath), CreateMode.EPHEMERAL)));
+
zkClient.multi(ops, true);
}
@@ -2198,11 +2215,7 @@ public class ZkController implements Closeable {
List<?> nodeList = (List<?>) roles.get("overseer");
if (nodeList == null) return;
if (nodeList.contains(getNodeName())) {
- ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, CollectionParams.CollectionAction.ADDROLE.toString().toLowerCase(Locale.ROOT),
- "node", getNodeName(),
- "role", "overseer");
- log.info("Going to add role {} ", props);
- getOverseerCollectionQueue().offer(Utils.toJSON(props));
+ setPreferredOverseer();
}
} catch (NoNodeException nne) {
return;
@@ -2211,6 +2224,15 @@ public class ZkController implements Closeable {
}
}
+ public void setPreferredOverseer() throws KeeperException, InterruptedException {
+ ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDROLE.toString().toLowerCase(Locale.ROOT),
+ "node", getNodeName(),
+ "role", "overseer",
+ "persist", "false");
+ log.warn("Going to add role {}. It is deprecated to use ADDROLE and consider using Node Roles instead.", props);
+ getOverseerCollectionQueue().offer(Utils.toJSON(props));
+ }
+
public CoreContainer getCoreContainer() {
return cc;
}
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
index 4dd2dbba24e..664bc502f54 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/Assign.java
@@ -21,20 +21,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
@@ -54,6 +41,8 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.NodeRoles;
+import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.util.NumberUtils;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -229,7 +218,9 @@ public class Assign {
return false;
}
- public static List<String> getLiveOrLiveAndCreateNodeSetList(final Set<String> liveNodes, final ZkNodeProps message, final Random random) {
+ public static List<String> getLiveOrLiveAndCreateNodeSetList(final Set<String> liveNodes, final ZkNodeProps message, final Random random,
+ DistribStateManager zk) {
+
List<String> nodeList;
final String createNodeSetStr = message.getStr(CREATE_NODE_SET);
final List<String> createNodeList = (createNodeSetStr == null) ? null :
@@ -244,13 +235,28 @@ public class Assign {
Collections.shuffle(nodeList, random);
}
} else {
- nodeList = new ArrayList<>(liveNodes);
+ nodeList = new ArrayList<>(filterNonDataNodes(zk, liveNodes));
Collections.shuffle(nodeList, random);
}
return nodeList;
}
+ public static Collection<String> filterNonDataNodes(DistribStateManager zk, Collection<String> liveNodes) {
+ try {
+ List<String> noData = ClusterAPI.getNodesByRole(NodeRoles.Role.DATA, NodeRoles.MODE_OFF, zk);
+ if (noData.isEmpty()) {
+ return liveNodes;
+ } else {
+ liveNodes = new HashSet<>(liveNodes);
+ liveNodes.removeAll(noData);
+ return liveNodes;
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error fetching roles from Zookeeper", e);
+ }
+ }
+
static class ReplicaCount {
public final String nodeName;
public Map<String, Integer> collectionReplicas;
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
index b7a0c978625..60a5bb8fae9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/CreateCollectionCmd.java
@@ -404,7 +404,7 @@ public class CreateCollectionCmd implements CollApiCmds.CollectionApiCommand {
// but (for now) require that each core goes on a distinct node.
List<ReplicaPosition> replicaPositions;
- List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, CollectionHandlingUtils.RANDOM);
+ List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(clusterState.getLiveNodes(), message, CollectionHandlingUtils.RANDOM, cloudManager.getDistribStateManager());
if (nodeList.isEmpty()) {
log.warn("It is unusual to create a collection ({}) without cores.", collectionName);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
index e0766e6e386..e2e34f6693b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerRoleCmd.java
@@ -64,6 +64,10 @@ public class OverseerRoleCmd implements CollApiCmds.CollectionApiCommand {
SolrZkClient zkClient = zkStateReader.getZkClient();
Map<String, List<String>> roles = null;
String node = message.getStr("node");
+ if ("false".equals(message.getStr("persist"))) { // no need to persist to roles.json
+ runPrioritizer();
+ return;
+ }
String roleName = message.getStr("role");
boolean nodeExists = false;
@@ -89,6 +93,11 @@ public class OverseerRoleCmd implements CollApiCmds.CollectionApiCommand {
} else {
zkClient.create(ZkStateReader.ROLES, Utils.toJSON(roles), CreateMode.PERSISTENT, true);
}
+ runPrioritizer();
+
+ }
+
+ private void runPrioritizer() {
//if there are too many nodes this command may time out. And most likely dedicated
// overseers are created when there are too many nodes . So , do this operation in a separate thread
new Thread(() -> {
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index 3ff38ae3b74..5c305f8e5d4 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -182,7 +182,7 @@ public class RestoreCmd implements CollApiCmds.CollectionApiCommand {
this.shardHandler = ccc.newShardHandler();
this.nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(
- zkStateReader.getClusterState().getLiveNodes(), message, CollectionHandlingUtils.RANDOM);
+ zkStateReader.getClusterState().getLiveNodes(), message, CollectionHandlingUtils.RANDOM, container.getZkController().getSolrCloudManager().getDistribStateManager());
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/cluster/Cluster.java b/solr/core/src/java/org/apache/solr/cluster/Cluster.java
index 3b7bdd47c8f..b3b210f9784 100644
--- a/solr/core/src/java/org/apache/solr/cluster/Cluster.java
+++ b/solr/core/src/java/org/apache/solr/cluster/Cluster.java
@@ -31,6 +31,11 @@ public interface Cluster {
*/
Set<Node> getLiveNodes();
+ /**
+ * @return current set of live nodes that are supposed to host data.
+ */
+ Set<Node> getLiveDataNodes();
+
/**
* Returns info about the given collection if one exists.
*
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
index ff3f09062f2..285ffba2c15 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/PlacementRequestImpl.java
@@ -82,11 +82,17 @@ public class PlacementRequestImpl implements PlacementRequest {
// If no nodes specified, use all live nodes. If nodes are specified, use specified list.
if (assignRequest.nodes != null) {
nodes = SimpleClusterAbstractionsImpl.NodeImpl.getNodes(assignRequest.nodes);
+
+ for (Node n: nodes) {
+ if (!cluster.getLiveDataNodes().contains(n)) {
+ throw new Assign.AssignmentException("Bad assign request: specified node is a non-data hosting node (" + n.getName() + ") for collection " + solrCollection.getName());
+ }
+ }
if (nodes.isEmpty()) {
throw new Assign.AssignmentException("Bad assign request: empty list of nodes for collection " + solrCollection.getName());
}
} else {
- nodes = cluster.getLiveNodes();
+ nodes = cluster.getLiveDataNodes();
if (nodes.isEmpty()) {
throw new Assign.AssignmentException("Impossible assign request: no live nodes for collection " + solrCollection.getName());
}
diff --git a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
index 48f3f506104..e5d99b2f6ca 100644
--- a/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
+++ b/solr/core/src/java/org/apache/solr/cluster/placement/impl/SimpleClusterAbstractionsImpl.java
@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
import com.google.common.collect.Maps;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
+import org.apache.solr.cloud.api.collections.Assign;
import org.apache.solr.cluster.*;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -53,10 +54,16 @@ class SimpleClusterAbstractionsImpl {
static class ClusterImpl implements Cluster {
private final Set<Node> liveNodes;
+ private final Set<Node> liveNodesWithData;
private final ClusterState clusterState;
ClusterImpl(SolrCloudManager solrCloudManager) throws IOException {
- liveNodes = NodeImpl.getNodes(solrCloudManager.getClusterStateProvider().getLiveNodes());
+ Set<String> liveNodes = solrCloudManager.getClusterStateProvider().getLiveNodes();
+ Collection<String> liveNodesWithData = Assign.filterNonDataNodes( solrCloudManager.getDistribStateManager(), liveNodes);
+ this.liveNodes = NodeImpl.getNodes(liveNodes);
+ this.liveNodesWithData = liveNodesWithData.size() == liveNodes.size() ?
+ this.liveNodes :
+ NodeImpl.getNodes(liveNodesWithData);
clusterState = solrCloudManager.getClusterStateProvider().getClusterState();
}
@@ -65,6 +72,10 @@ class SimpleClusterAbstractionsImpl {
return liveNodes;
}
+ public Set<Node> getLiveDataNodes() {
+ return liveNodesWithData;
+ }
+
@Override
public SolrCollection getCollection(String collectionName) {
return SolrCollectionImpl.createCollectionFacade(clusterState, collectionName);
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index d4d20301b5f..b915330427a 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -246,6 +246,8 @@ public class CoreContainer {
private final ObjectCache objectCache = new ObjectCache();
+ public final NodeRoles nodeRoles = new NodeRoles(System.getProperty(NodeRoles.NODE_ROLES_PROP));
+
private final ClusterSingletons clusterSingletons = new ClusterSingletons(
() -> getZkController() != null &&
getZkController().getOverseer() != null &&
@@ -945,6 +947,14 @@ public class CoreContainer {
});
clusterSingletons.setReady();
+ if (NodeRoles.MODE_PREFERRED.equals(nodeRoles.getRoleMode(NodeRoles.Role.OVERSEER))) {
+ try {
+ log.info("This node has been started as a preferred overseer");
+ zkSys.getZkController().setPreferredOverseer();
+ } catch (KeeperException | InterruptedException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
if (!distributedCollectionCommandRunner.isPresent()) {
zkSys.getZkController().checkOverseerDesignate();
}
diff --git a/solr/core/src/java/org/apache/solr/core/NodeRoles.java b/solr/core/src/java/org/apache/solr/core/NodeRoles.java
new file mode 100644
index 00000000000..d13b651f666
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/NodeRoles.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.StrUtils;
+
+import java.util.*;
+
+public class NodeRoles {
+ public static final String NODE_ROLES_PROP = "solr.node.roles";
+
+ /**
+ * Roles to be assumed on nodes that don't have roles specified for them at startup
+ */
+ public static final String DEFAULT_ROLES_STRING = "data:on,overseer:allowed";
+
+ // Map of roles to mode that are applicable for this node.
+ private Map<Role, String> nodeRoles;
+
+ public NodeRoles(String rolesString) {
+ Map<Role, String> roles = new EnumMap<Role, String>(Role.class);
+ if (StringUtils.isEmpty(rolesString)) {
+ rolesString = DEFAULT_ROLES_STRING;
+ }
+ List<String> rolesList = StrUtils.splitSmart(rolesString, ',');
+ for (String s: rolesList) {
+ List<String> roleMode = StrUtils.splitSmart(s,':');
+ Role r = Role.getRole(roleMode.get(0));
+ String m = roleMode.get(1);
+ if (r.supportedModes().contains(m)) {
+ roles.put(r, m);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Unknown role mode '" + roleMode.get(1) + "' for role '" + r + "'");
+ }
+ }
+ for(Role r: Role.values()) {
+ if (!roles.containsKey(r)) {
+ roles.put(r, r.modeWhenRoleIsAbsent());
+ }
+ }
+ nodeRoles = Collections.unmodifiableMap(roles);
+ }
+
+ public Map<Role, String> getRoles() {
+ return nodeRoles;
+ }
+
+ public String getRoleMode(Role role) {
+ return nodeRoles.get(role);
+ }
+
+ public boolean isOverseerAllowedOrPreferred() {
+ String roleMode = nodeRoles.get(Role.OVERSEER);
+ return MODE_ALLOWED.equals(roleMode) || MODE_PREFERRED.equals(roleMode);
+ }
+
+ public final static String MODE_ON = "on";
+ public final static String MODE_OFF = "off";
+ public final static String MODE_ALLOWED = "allowed";
+ public final static String MODE_PREFERRED = "preferred";
+ public final static String MODE_DISALLOWED = "disallowed";
+
+ public enum Role {
+ DATA("data") {
+ @Override
+ public Set<String> supportedModes() {
+ return Set.of(MODE_ON, MODE_OFF);
+ }
+ @Override
+ public String modeWhenRoleIsAbsent() {
+ return MODE_OFF;
+ }
+ },
+ OVERSEER("overseer") {
+ @Override
+ public Set<String> supportedModes() {
+ return Set.of(MODE_ALLOWED, MODE_PREFERRED, MODE_DISALLOWED);
+ }
+ @Override
+ public String modeWhenRoleIsAbsent() {
+ return MODE_DISALLOWED;
+ }
+ };
+
+ public final String roleName;
+
+ Role(String name) {
+ this.roleName = name;
+ }
+
+ public static Role getRole(String value) {
+ for (Role role: Role.values()) {
+ if (value.equals(role.roleName)) return role;
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown role: " + value);
+ }
+
+ public abstract Set<String> supportedModes();
+
+ /**
+ * Default mode for a role in nodes where this role is not specified.
+ */
+ public abstract String modeWhenRoleIsAbsent();
+
+ @Override
+ public String toString() {
+ return roleName;
+ }
+ }
+
+ public static String getZNodeForRole(Role role) {
+ return ZkStateReader.NODE_ROLES + "/" + role.roleName;
+ }
+
+ public static String getZNodeForRoleMode(Role role, String mode) {
+ return ZkStateReader.NODE_ROLES + "/" + role.roleName + "/" + mode;
+ }
+
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
index be5c7de48b6..a40acc04521 100644
--- a/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
@@ -17,13 +17,14 @@
package org.apache.solr.handler;
-import java.util.HashMap;
-import java.util.Map;
+import java.io.IOException;
+import java.util.*;
import com.google.common.collect.Maps;
import org.apache.solr.api.Command;
import org.apache.solr.api.EndPoint;
import org.apache.solr.api.PayloadObj;
+import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.request.beans.ClusterPropPayload;
import org.apache.solr.client.solrj.request.beans.CreateConfigPayload;
import org.apache.solr.client.solrj.request.beans.RateLimiterPayload;
@@ -31,6 +32,7 @@ import org.apache.solr.cloud.ConfigSetCmds;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.annotation.JsonProperty;
import org.apache.solr.common.cloud.ClusterProperties;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ConfigSetParams;
@@ -39,10 +41,12 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.ReflectMapWriter;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.NodeRoles;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.ConfigSetsHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.zookeeper.KeeperException;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.DELETE;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
@@ -57,15 +61,15 @@ import static org.apache.solr.security.PermissionNameProvider.Name.COLL_READ_PER
import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM;
import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM;
-/** All V2 APIs that have a prefix of /api/cluster/
- *
+/**
+ * All V2 APIs that have a prefix of /api/cluster/
*/
public class ClusterAPI {
private final CollectionsHandler collectionsHandler;
private final ConfigSetsHandler configSetsHandler;
- public final Commands commands = new Commands();
- public final ConfigSetCommands configSetCommands = new ConfigSetCommands();
+ public final Commands commands = new Commands();
+ public final ConfigSetCommands configSetCommands = new ConfigSetCommands();
public ClusterAPI(CollectionsHandler ch, ConfigSetsHandler configSetsHandler) {
this.collectionsHandler = ch;
@@ -73,8 +77,107 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/aliases",
- permission = COLL_READ_PERM)
+ path = "/cluster/node-roles",
+ permission = COLL_READ_PERM)
+ public void roles(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ rsp.add("node-roles", readRecursive(ZkStateReader.NODE_ROLES,
+ collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(), 3));
+ }
+
+ Object readRecursive(String path, DistribStateManager zk, int depth) {
+ if (depth == 0) return null;
+ Map<String, Object> result;
+ try {
+ List<String> children = zk.listData(path);
+ if (children != null && !children.isEmpty()) {
+ result = new HashMap<>();
+ } else {
+ return Collections.emptySet();
+ }
+ for (String child: children) {
+ Object c = readRecursive(path + "/" + child, zk, depth - 1);
+ result.put(child, c);
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ if (depth == 1) {
+ return result.keySet();
+ } else {
+ return result;
+ }
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/role/{role}",
+ permission = COLL_READ_PERM)
+ public void nodesWithRole(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ String role = req.getPathTemplateValues().get("role");
+ rsp.add("node-roles", Map.of(role,
+ readRecursive(ZkStateReader.NODE_ROLES + "/" + role,
+ collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(), 2)));
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/node/{node}",
+ permission = COLL_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void rolesForNode(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ String node = req.getPathTemplateValues().get("node");
+ Map<String, String> ret = new HashMap<String, String>();
+ Map<String, Map<String, Set<String>>> roles = (Map<String, Map<String, Set<String>>>) readRecursive(ZkStateReader.NODE_ROLES,
+ collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager().getDistribStateManager(), 3);
+ for (String role: roles.keySet()) {
+ for (String mode: roles.get(role).keySet()) {
+ if (roles.get(role).get(mode).isEmpty()) continue;
+ Set<String> nodes = roles.get(role).get(mode);
+ if (nodes.contains(node)) ret.put(role, mode);
+ }
+ }
+ for (String role: ret.keySet()) {
+ rsp.add(role, ret.get(role));
+ }
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/supported",
+ permission = COLL_READ_PERM)
+ public void supportedRoles(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ Map<String, Object> roleModesSupportedMap = new HashMap<>();
+ for (NodeRoles.Role role: NodeRoles.Role.values()) {
+ roleModesSupportedMap.put(role.toString(),
+ Map.of("modes", role.supportedModes()));
+ }
+ rsp.add("supported-roles", roleModesSupportedMap);
+ }
+
+ @EndPoint(method = GET,
+ path = "/cluster/node-roles/role/{role}/{mode}",
+ permission = COLL_READ_PERM)
+ public void nodesWithRoleMode(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ // Here, deal with raw strings instead of Role & Mode types so as to handle roles and modes
+ // that are not understood by this node (possibly during a rolling upgrade)
+ String roleStr = req.getPathTemplateValues().get("role");
+ String modeStr = req.getPathTemplateValues().get("mode");
+
+ List<String> nodes = collectionsHandler.getCoreContainer().getZkController().getSolrCloudManager()
+ .getDistribStateManager().listData(ZkStateReader.NODE_ROLES + "/" + roleStr + "/" + modeStr);
+ rsp.add( "node-roles", Map.of(roleStr, Collections.singletonMap(modeStr, nodes)));
+ }
+
+ public static List<String> getNodesByRole(NodeRoles.Role role, String mode, DistribStateManager zk)
+ throws InterruptedException, IOException, KeeperException {
+ try {
+ return zk.listData(ZkStateReader.NODE_ROLES + "/" + role + "/" + mode);
+ } catch (NoSuchElementException e) {
+ return Collections.emptyList();
+ }
+ }
+
+
+ @EndPoint(method = GET,
+ path = "/cluster/aliases",
+ permission = COLL_READ_PERM)
public void aliases(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION, CollectionParams.CollectionAction.LISTALIASES.lowerName);
@@ -82,58 +185,58 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/overseer",
- permission = COLL_READ_PERM)
+ path = "/cluster/overseer",
+ permission = COLL_READ_PERM)
public void getOverseerStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
collectionsHandler.handleRequestBody(wrapParams(req, "action", OVERSEERSTATUS.lowerName), rsp);
}
@EndPoint(method = GET,
- path = "/cluster",
- permission = COLL_READ_PERM)
+ path = "/cluster",
+ permission = COLL_READ_PERM)
public void getCluster(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
collectionsHandler.handleRequestBody(wrapParams(req, "action", LIST.lowerName), rsp);
}
@EndPoint(method = DELETE,
- path = "/cluster/command-status/{id}",
- permission = COLL_EDIT_PERM)
+ path = "/cluster/command-status/{id}",
+ permission = COLL_EDIT_PERM)
public void deleteCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION, DELETESTATUS.lowerName);
v1Params.put(REQUESTID, req.getPathTemplateValues().get("id"));
collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp);
}
-
+
@EndPoint(method = DELETE,
- path = "/cluster/command-status",
- permission = COLL_EDIT_PERM)
+ path = "/cluster/command-status",
+ permission = COLL_EDIT_PERM)
public void flushCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
CollectionsHandler.CollectionOperation.DELETESTATUS_OP.execute(req, rsp, collectionsHandler);
}
@EndPoint(method = DELETE,
- path = "/cluster/configs/{name}",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}",
+ permission = CONFIG_EDIT_PERM
)
public void deleteConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
req = wrapParams(req,
- "action", ConfigSetParams.ConfigSetAction.DELETE.toString(),
- CommonParams.NAME, req.getPathTemplateValues().get("name"));
+ "action", ConfigSetParams.ConfigSetAction.DELETE.toString(),
+ CommonParams.NAME, req.getPathTemplateValues().get("name"));
configSetsHandler.handleRequestBody(req, rsp);
}
@EndPoint(method = GET,
- path = "/cluster/configs",
- permission = CONFIG_READ_PERM)
+ path = "/cluster/configs",
+ permission = CONFIG_READ_PERM)
public void listConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
req = wrapParams(req, "action", ConfigSetParams.ConfigSetAction.LIST.toString());
configSetsHandler.handleRequestBody(req, rsp);
}
@EndPoint(method = POST,
- path = "/cluster/configs",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs",
+ permission = CONFIG_EDIT_PERM
)
public class ConfigSetCommands {
@@ -141,8 +244,8 @@ public class ClusterAPI {
@SuppressWarnings("unchecked")
public void create(PayloadObj<CreateConfigPayload> obj) throws Exception {
Map<String, Object> mapVals = obj.get().toMap(new HashMap<>());
- Map<String,Object> customProps = obj.get().properties;
- if(customProps!= null) {
+ Map<String, Object> customProps = obj.get().properties;
+ if (customProps != null) {
customProps.forEach((k, o) -> mapVals.put(ConfigSetCmds.CONFIG_SET_PROPERTY_PREFIX + k, o));
}
mapVals.put("action", ConfigSetParams.ConfigSetAction.CREATE.toString());
@@ -152,8 +255,8 @@ public class ClusterAPI {
}
@EndPoint(method = PUT,
- path = "/cluster/configs/{name}",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}",
+ permission = CONFIG_EDIT_PERM
)
public void uploadConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
req = wrapParams(req,
@@ -165,8 +268,8 @@ public class ClusterAPI {
}
@EndPoint(method = PUT,
- path = "/cluster/configs/{name}/*",
- permission = CONFIG_EDIT_PERM
+ path = "/cluster/configs/{name}/*",
+ permission = CONFIG_EDIT_PERM
)
public void insertIntoConfigSet(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
String path = req.getPathTemplateValues().get("*");
@@ -189,17 +292,17 @@ public class ClusterAPI {
public static SolrQueryRequest wrapParams(SolrQueryRequest req, Map<String, Object> m) {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
m.forEach((k, v) -> {
- if(v == null) return;
+ if (v == null) return;
solrParams.add(k.toString(), String.valueOf(v));
});
- DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(),solrParams);
+ DefaultSolrParams dsp = new DefaultSolrParams(req.getParams(), solrParams);
req.setParams(dsp);
return req;
}
@EndPoint(method = GET,
- path = "/cluster/command-status/{id}",
- permission = COLL_READ_PERM)
+ path = "/cluster/command-status/{id}",
+ permission = COLL_READ_PERM)
public void getCommandStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
final Map<String, Object> v1Params = Maps.newHashMap();
v1Params.put(ACTION, REQUESTSTATUS.lowerName);
@@ -208,8 +311,8 @@ public class ClusterAPI {
}
@EndPoint(method = GET,
- path = "/cluster/nodes",
- permission = COLL_READ_PERM)
+ path = "/cluster/nodes",
+ permission = COLL_READ_PERM)
public void getNodes(SolrQueryRequest req, SolrQueryResponse rsp) {
rsp.add("nodes", getCoreContainer().getZkController().getClusterState().getLiveNodes());
}
@@ -219,13 +322,13 @@ public class ClusterAPI {
}
@EndPoint(method = POST,
- path = "/cluster",
- permission = COLL_EDIT_PERM)
+ path = "/cluster",
+ permission = COLL_EDIT_PERM)
public class Commands {
@Command(name = "add-role")
public void addRole(PayloadObj<RoleInfo> obj) throws Exception {
RoleInfo info = obj.get();
- Map<String,Object> m = info.toMap(new HashMap<>());
+ Map<String, Object> m = info.toMap(new HashMap<>());
m.put("action", ADDROLE.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse());
}
@@ -233,7 +336,7 @@ public class ClusterAPI {
@Command(name = "remove-role")
public void removeRole(PayloadObj<RoleInfo> obj) throws Exception {
RoleInfo info = obj.get();
- Map<String,Object> m = info.toMap(new HashMap<>());
+ Map<String, Object> m = info.toMap(new HashMap<>());
m.put("action", REMOVEROLE.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse());
}
@@ -241,7 +344,7 @@ public class ClusterAPI {
@Command(name = "set-obj-property")
public void setObjProperty(PayloadObj<ClusterPropPayload> obj) {
//Not using the object directly here because the API differentiate between {name:null} and {}
- Map<String,Object> m = obj.getDataMap();
+ Map<String, Object> m = obj.getDataMap();
ClusterProperties clusterProperties = new ClusterProperties(getCoreContainer().getZkController().getZkClient());
try {
clusterProperties.setClusterProperties(m);
@@ -251,8 +354,8 @@ public class ClusterAPI {
}
@Command(name = "set-property")
- public void setProperty(PayloadObj<Map<String,String>> obj) throws Exception {
- Map<String,Object> m = obj.getDataMap();
+ public void setProperty(PayloadObj<Map<String, String>> obj) throws Exception {
+ Map<String, Object> m = obj.getDataMap();
m.put("action", CLUSTERPROP.toString());
collectionsHandler.handleRequestBody(wrapParams(obj.getRequest(), m), obj.getResponse());
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java
new file mode 100644
index 00000000000..1a573d68935
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/NodeRolesTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Collection;
+import java.util.Collections;
+
+import java.util.Map;
+
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.response.V2Response;
+import org.apache.solr.core.NodeRoles;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NodeRolesTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Before
+ public void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @After
+ public void tearDownCluster() throws Exception {
+ shutdownCluster();
+ }
+
+ public void testRoleIntegration() throws Exception {
+ JettySolrRunner j0 = cluster.getJettySolrRunner(0);
+ testSupportedRolesAPI();
+
+ // Start a dedicated overseer node
+ JettySolrRunner j1 = startNodeWithRoles("overseer:preferred,data:off");
+ validateNodeRoles(j1.getNodeName(), "node-roles/overseer/preferred", j1.getNodeName(), "node-roles/data/off");
+
+ V2Response rsp;
+ OverseerRolesTest.waitForNewOverseer(20, j1.getNodeName(), true);
+
+ // Start another node that is allowed or preferred overseer but has data
+ String overseerModeOnDataNode = random().nextBoolean() ? "preferred" : "allowed";
+ JettySolrRunner j2 = startNodeWithRoles("overseer:" + overseerModeOnDataNode + ",data:on");
+ validateNodeRoles(j2.getNodeName(), "node-roles/overseer/" + overseerModeOnDataNode, j2.getNodeName(), "node-roles/data/on");
+
+ // validate the preferred overseers
+ validateNodeRoles(j2.getNodeName(), "node-roles/overseer/" + overseerModeOnDataNode, j1.getNodeName(), "node-roles/overseer/preferred");
+
+ String COLLECTION_NAME = "TEST_ROLES";
+ CollectionAdminRequest
+ .createCollection(COLLECTION_NAME, "conf", 3, 1)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(COLLECTION_NAME, 3, 3);
+
+ // Assert that no replica was placed on the dedicated overseer node
+ String dedicatedOverseer = j1.getNodeName();
+ cluster.getSolrClient().getClusterStateProvider().getCollection(COLLECTION_NAME)
+ .forEachReplica((s, replica) -> assertNotEquals(replica.node, dedicatedOverseer));
+
+ // Shutdown the dedicated overseer, make sure that node disappears from the roles output
+ j1.stop();
+
+ // Wait and make sure that another node picks up overseer responsibilities
+ OverseerRolesTest.waitForNewOverseer(20, it -> !dedicatedOverseer.equals(it), false);
+
+ // Make sure the stopped node no longer has the role assigned
+ rsp = new V2Request.Builder("/cluster/node-roles/role/overseer/" + overseerModeOnDataNode).GET().build().process(cluster.getSolrClient());
+ assertFalse(((Collection) rsp._get("node-roles/overseer/" + overseerModeOnDataNode, null)).contains(j1.getNodeName()));
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void validateNodeRoles(String... nodenamePaths) throws org.apache.solr.client.solrj.SolrServerException, java.io.IOException {
+ V2Response rsp = new V2Request.Builder("/cluster/node-roles").GET().build().process(cluster.getSolrClient());
+ for (int i = 0; i < nodenamePaths.length; i += 2) {
+ String nodename = nodenamePaths[i];
+ String path = nodenamePaths[i + 1];
+ assertTrue("Didn't find " + nodename + " at " + path + ". Full response: " + rsp.jsonStr(),
+ ((Collection) rsp._get(path, Collections.emptyList())).contains(nodename));
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void testSupportedRolesAPI() throws Exception {
+ V2Response rsp = new V2Request.Builder("/cluster/node-roles/supported").GET().build().process(cluster.getSolrClient());
+ Map<String, Object> l = (Map<String, Object>) rsp._get("supported-roles", Collections.emptyMap());
+ assertTrue(l.containsKey("data"));
+ assertTrue(l.containsKey("overseer"));
+ }
+
+ private JettySolrRunner startNodeWithRoles(String roles) throws Exception {
+ JettySolrRunner jetty;
+ System.setProperty(NodeRoles.NODE_ROLES_PROP, roles);
+ try {
+ jetty = cluster.startJettySolrRunner();
+ } finally {
+ System.clearProperty(NodeRoles.NODE_ROLES_PROP);
+ }
+ return jetty;
+ }
+
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index c437a30e3a6..6e59f092882 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -54,7 +54,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
shutdownCluster();
}
- private void waitForNewOverseer(int seconds, Predicate<String> state, boolean failOnIntermediateTransition) throws Exception {
+ public static void waitForNewOverseer(int seconds, Predicate<String> state, boolean failOnIntermediateTransition) throws Exception {
TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS, TimeSource.NANO_TIME);
String current = null;
while (timeout.hasTimedOut() == false) {
@@ -72,7 +72,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
fail("Timed out waiting for overseer state change. The current overseer is: "+current);
}
- private void waitForNewOverseer(int seconds, String expected, boolean failOnIntermediateTransition) throws Exception {
+ public static void waitForNewOverseer(int seconds, String expected, boolean failOnIntermediateTransition) throws Exception {
log.info("Expecting node: {}", expected);
waitForNewOverseer(seconds, s -> Objects.equals(s, expected), failOnIntermediateTransition);
}
diff --git a/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java b/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
index 1260a1063c2..adf4396efd2 100644
--- a/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
+++ b/solr/core/src/test/org/apache/solr/cluster/placement/ClusterAbstractionsForTest.java
@@ -42,6 +42,11 @@ class ClusterAbstractionsForTest {
return liveNodes;
}
+ @Override
+ public Set<Node> getLiveDataNodes() {
+ return liveNodes;
+ }
+
@Override
public SolrCollection getCollection(String collectionName) {
return collections.get(collectionName);
diff --git a/solr/solr-ref-guide/src/node-roles.adoc b/solr/solr-ref-guide/src/node-roles.adoc
new file mode 100644
index 00000000000..71d9c6ccb57
--- /dev/null
+++ b/solr/solr-ref-guide/src/node-roles.adoc
@@ -0,0 +1,196 @@
+= Node Roles
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+A node in Solr is usually capable of performing various types of operations, e.g. hosting replicas, performing indexing and querying, collection management tasks, etc. To set up a cluster where these functions are isolated to certain dedicated nodes, we can use the concept of node roles.
+
+== Definitions
+
+=== Node role
+
+A role is a designation for a node that indicates that the node may perform a certain function.
+
+=== Mode
+Every role has a list of modes under which a node can be. It can be simple (e.g. `["on", "off"]`) or more granular (e.g. `["allowed", "preferred", "disallowed"]`).
+
+== Roles
+
+In order to specify role(s) for a node, one needs to start a Solr node with the following parameter.
+
+.Startup Parameter
+[cols="1,2,1,1"]
+|===
+|Parameter |Value |Required? | Default
+
+|solr.node.roles
+|Comma separated list of roles (in the format: `<role>:<mode>`) for this node.
+Examples: `-Dsolr.node.roles=data:on,overseer:allowed` or `-Dsolr.node.roles=overseer:preferred`
+|No
+|`data:on,overseer:allowed`
+|===
+
+[TIP]
+====
+If a node has been started with no `solr.node.roles` parameter, it will be assumed to have the data role turned on and overseer role allowed on it. If you've never used roles before, you likely won't need to change anything in your startup parameters to accommodate the functionality associated with these roles.
+====
+
+.Supported roles
+[cols="1,1"]
+|===
+|Role |Modes
+
+|`data`
+|on, off
+
+|`overseer`
+|allowed, preferred, disallowed
+|===
+
+=== `data` role
+A node with this role (in mode "on") can host shards and replicas for collections.
+
+=== `overseer` role
+A node with this role can perform duties of an overseer node (unless mode is `disallowed`). When one or more nodes have the overseer role in `preferred` mode, the overseer leader will be elected from one of these nodes. In case no node is designated as a preferred overseer or no such node is live, the overseer leader will be elected from one of the nodes that have the overseer role in `allowed` mode. If all nodes that are designated with overseer role (allowed or preferred) are down, the cluster will be left without an overseer.
+
+== Example usage
+
+Sometimes, when the nodes in a cluster are under heavy querying or indexing load, the overseer leader node might be unable to perform collection management duties efficiently. It might be reasonable to have dedicated nodes to act as the overseer. Such an effect can be achieved as follows:
+
+* Most nodes (data nodes) in the cluster start with `-Dsolr.node.roles=data:on,overseer:allowed` (or with no parameter, since the default value for `solr.node.roles` is the same).
+* One or more nodes (dedicated overseer nodes) can start with `-Dsolr.node.roles=overseer:preferred` (or `-Dsolr.node.roles=overseer:preferred,data:off`)
+
+In this arrangement, such dedicated nodes can be provisioned on hardware with lesser resources like CPU, memory or disk space than other data nodes (since these are stateless nodes) and yet the cluster will behave optimally. In case the dedicated overseer nodes go down for some reason, the overseer leader will be elected from one of the data nodes (since they have overseer in "allowed" mode), and once one of the dedicated overseer nodes are back up again, it will be re-elected for the overseer leadership.
+
+== Roles API
+
+=== GET /api/cluster/node-roles/supported
+
+Fetches the list of supported roles and their supported modes for this cluster.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles/supported
+----
+
+*Output*
+[source,text]
+----
+{
+ "supported-roles":{
+ "data":{
+ "modes":["off",
+ "on"]
+ },
+ "overseer":{
+ "modes":["disallowed",
+ "allowed",
+ "preferred"]
+ }
+ }
+}
+----
+
+=== GET /api/cluster/node-roles
+
+Fetches the current node roles assignment for all the nodes in the cluster.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"],
+ "on":["solr1:8983_solr"]
+ },
+ "overseer":{
+ "allowed":["solr1:8983_solr"],
+ "disallowed":[],
+ "preferred":["solr2:8983_solr"]
+ }
+ }
+}
+----
+
+=== GET /api/cluster/node-roles/role/{role}
+
+Fetches the current node roles assignment for a specified role.
+
+*Input*
+[source,text]
+----
+http://localhost:8983/api/node-roles/role/data
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"],
+ "on":["solr1:8983_solr"]
+ }
+ }
+}
+----
+
+*Input*
+[source,text]
+----
+http://localhost:8983/api/node-roles/role/data/off
+----
+
+*Output*
+[source,text]
+----
+{
+ "node-roles":{
+ "data":{
+ "off":["solr2:8983_solr"]
+ }
+ }
+}
+----
+
+
+
+=== GET /api/cluster/node-roles/node/{node}
+
+Fetches the current node roles assignment for the specified node.
+
+*Input*
+[source,text]
+----
+curl http://localhost:8983/api/cluster/node-roles/node/solr1:8983_solr
+----
+
+*Output*
+[source,text]
+----
+{
+ "data":"on",
+ "overseer":"allowed"
+}
+----
diff --git a/solr/solr-ref-guide/src/solrcloud-clusters.adoc b/solr/solr-ref-guide/src/solrcloud-clusters.adoc
index 83b3fde57d5..03723f5fe93 100644
--- a/solr/solr-ref-guide/src/solrcloud-clusters.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-clusters.adoc
@@ -3,6 +3,7 @@
solrcloud-recoveries-and-write-tolerance, \
solrcloud-distributed-requests, \
aliases, \
+ node-roles, \
cluster-node-management, \
shard-management, \
replica-management, \
@@ -47,6 +48,9 @@ ZooKeeper is a critical component of SolrCloud, used to manage shard and replica
| <<solrcloud-recoveries-and-write-tolerance.adoc#,SolrCloud Recoveries and Write Tolerance>>: Recovery in a SolrCloud cluster.
| <<solrcloud-distributed-requests.adoc#,SolrCloud Distributed Requests>>: Query routing in a SolrCloud cluster.
| <<aliases.adoc#,Aliases>>: Alternative names for collections.
+2+^h| Cluster Management
+| <<node-roles.adoc#,Node Roles>>: Functional roles for nodes for multi-tiered clusters.
+|
2+^h| Collections API
|<<cluster-node-management.adoc#,Cluster and Node Management>>: Cluster management commands of the Collections API.
| <<shard-management.adoc#,Shard Management>>: Shard management commands of the Collections API.
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 461dbaf9f96..848c69e6447 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -103,6 +103,16 @@ public class ZkStateReader implements SolrCloseable {
public static final String STATE_TIMESTAMP_PROP = "stateTimestamp";
public static final String COLLECTIONS_ZKNODE = "/collections";
public static final String LIVE_NODES_ZKNODE = "/live_nodes";
+
+ // TODO: Deprecate and remove support for roles.json in an upcoming release.
+ /**
+ * The following, node_roles and roles.json are for assigning roles to
+ * nodes. The node_roles is the preferred way (using -Dsolr.node.roles param),
+ * and roles.json is used by legacy ADDROLE API command.
+ */
+ public static final String NODE_ROLES = "/node_roles";
+ public static final String ROLES = "/roles.json";
+
public static final String ALIASES = "/aliases.json";
/**
* This ZooKeeper file is no longer used starting with Solr 9 but keeping the name around to check if it
@@ -124,7 +134,6 @@ public class ZkStateReader implements SolrCloseable {
public static final String TLOG_REPLICAS = "tlogReplicas";
public static final String READ_ONLY = "readOnly";
- public static final String ROLES = "/roles.json";
public static final String CONFIGS_ZKNODE = "/configs";
public final static String CONFIGNAME_PROP = "configName";