Apache Solr open-source search software
SOLR-17381: Make CLUSTERSTATUS configurable (#2599)
aparnasuresh85c7af8b82db21
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 84e4461ab15..0fe7e1b8d02 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -118,6 +118,9 @@ Optimizations* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yetthat which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley)+* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,+ reducing unnecessary data fetching. (Aparna Suresh, David Smiley)+* SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)Bug Fixesdiff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.javaindex f897aea129c..6c5998d17a2 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java@@ -34,9 +34,9 @@ import org.apache.solr.common.cloud.DocRouter;import org.apache.solr.common.cloud.PerReplicaStates;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.Slice;-import org.apache.solr.common.cloud.ZkNodeProps;import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.params.ShardParams;+import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.Utils;@@ -44,9 +44,14 @@ import org.apache.zookeeper.KeeperException;public class ClusterStatus {private final ZkStateReader zkStateReader;- private final ZkNodeProps message;+ private final SolrParams solrParams;private final String collection; // maybe null+ public static final String INCLUDE_ALL = "includeAll";+ public static final String LIVENODES_PROP = "liveNodes";+ public static final String CLUSTER_PROP = "clusterProperties";+ public static final String ALIASES_PROP = "aliases";+/** Shard / collection health state. */public enum Health {/** All replicas up, leader exists. */@@ -89,16 +94,72 @@ public class ClusterStatus {}}- public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {+ public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) {this.zkStateReader = zkStateReader;- this.message = props;- collection = props.getStr(ZkStateReader.COLLECTION_PROP);+ this.solrParams = params;+ collection = params.get(ZkStateReader.COLLECTION_PROP);}public void getClusterStatus(NamedList<Object> results)throws KeeperException, InterruptedException {+ NamedList<Object> clusterStatus = new SimpleOrderedMap<>();++ boolean includeAll = solrParams.getBool(INCLUDE_ALL, true);+ boolean withLiveNodes = solrParams.getBool(LIVENODES_PROP, includeAll);+ boolean withClusterProperties = solrParams.getBool(CLUSTER_PROP, includeAll);+ boolean withRoles = solrParams.getBool(ZkStateReader.ROLES_PROP, includeAll);+ boolean withCollection = includeAll || (collection != null);+ boolean withAliases = solrParams.getBool(ALIASES_PROP, includeAll);++ List<String> liveNodes = null;+ if (withLiveNodes || collection != null) {+ liveNodes =+ zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);+ // add live_nodes+ if (withLiveNodes) clusterStatus.add("live_nodes", liveNodes);+ }++ Aliases aliases = null;+ if (withCollection || withAliases) {+ aliases = zkStateReader.getAliases();+ }++ if (withCollection) {+ assert liveNodes != null;+ fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);+ }++ if (withAliases) {+ addAliasMap(aliases, clusterStatus);+ }++ if (withClusterProperties) {+ Map<String, Object> clusterProps = zkStateReader.getClusterProperties();+ if (clusterProps == null) {+ clusterProps = Collections.emptyMap();+ }+ clusterStatus.add("properties", clusterProps);+ }++ // add the roles map+ if (withRoles) {+ Map<?, ?> roles = Collections.emptyMap();+ if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {+ roles =+ (Map<?, ?>)+ Utils.fromJSON(+ zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));+ }+ clusterStatus.add("roles", roles);+ }++ results.add("cluster", clusterStatus);+ }++ private void fetchClusterStatusForCollOrAlias(+ NamedList<Object> clusterStatus, List<String> liveNodes, Aliases aliases) {+// read aliases- Aliases aliases = zkStateReader.getAliases();Map<String, List<String>> collectionVsAliases = new HashMap<>();Map<String, List<String>> aliasVsCollections = aliases.getCollectionAliasListMap();for (Map.Entry<String, List<String>> entry : aliasVsCollections.entrySet()) {@@ -112,18 +173,10 @@ public class ClusterStatus {}}- Map<?, ?> roles = null;- if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {- roles =- (Map<?, ?>)- Utils.fromJSON(- zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));- }-ClusterState clusterState = zkStateReader.getClusterState();- String routeKey = message.getStr(ShardParams._ROUTE_);- String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);+ String routeKey = solrParams.get(ShardParams._ROUTE_);+ String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP);Map<String, DocCollection> collectionsMap = null;if (collection == null) {@@ -139,6 +192,7 @@ public class ClusterStatus {if (didNotFindCollection && isAlias) {// In this case this.collection is an alias name not a collection// get all collections and filter out collections not in the alias+ // clusterState.getCollectionsMap() should be replaced with an inexpensive callcollectionsMap =clusterState.getCollectionsMap().entrySet().stream().filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey()))@@ -191,43 +245,25 @@ public class ClusterStatus {}String configName = clusterStateCollection.getConfigName();collectionStatus.put("configName", configName);- if (message.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {+ if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {PerReplicaStates prs = clusterStateCollection.getPerReplicaStates();collectionStatus.put("PRS", prs);}collectionProps.add(name, collectionStatus);}- List<String> liveNodes =- zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);-// now we need to walk the collectionProps tree to cross-check replica state with live nodescrossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);- NamedList<Object> clusterStatus = new SimpleOrderedMap<>();clusterStatus.add("collections", collectionProps);+ }- // read cluster properties- Map<String, Object> clusterProps = zkStateReader.getClusterProperties();- if (clusterProps != null && !clusterProps.isEmpty()) {- clusterStatus.add("properties", clusterProps);- }-+ private void addAliasMap(Aliases aliases, NamedList<Object> clusterStatus) {// add the alias map tooMap<String, String> collectionAliasMap = aliases.getCollectionAliasMap(); // comma delimif (!collectionAliasMap.isEmpty()) {clusterStatus.add("aliases", collectionAliasMap);}-- // add the roles map- if (roles != null) {- clusterStatus.add("roles", roles);- }-- // add live_nodes- clusterStatus.add("live_nodes", liveNodes);-- results.add("cluster", clusterStatus);}/**diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex 485670a4421..7ee6da6a0c1 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -100,7 +100,6 @@ import static org.apache.solr.common.params.CommonParams.TIMING;import static org.apache.solr.common.params.CommonParams.VALUE_LONG;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;-import static org.apache.solr.common.params.ShardParams._ROUTE_;import static org.apache.solr.common.util.StrUtils.formatString;import java.lang.invoke.MethodHandles;@@ -975,10 +974,7 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionCLUSTERSTATUS_OP(CLUSTERSTATUS,(req, rsp, h) -> {- Map<String, Object> all =- copy(req.getParams(), null, COLLECTION_PROP, SHARD_ID_PROP, _ROUTE_, "prs");- new ClusterStatus(- h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all))+ new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams()).getClusterStatus(rsp.getValues());return null;}),diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adocindex 2b60644c42b..6765c1ca93b 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc@@ -99,6 +99,51 @@ Multiple shard names can be specified as a comma-separated list.+This can be used if you need the details of the shard where a particular document belongs to and you don't know which shard it falls under.+`aliases`::+++[%autowidth,frame=none]+|===+|Optional |Default: will default to the default value of `includeAll` parameter specified below+|===++++`liveNodes`::+++[%autowidth,frame=none]+|===+|Optional |Default: will default to the default value of `includeAll` parameter specified below+|===+++If set to true, returns the status of live nodes in the cluster.++`clusterProperties`::+++[%autowidth,frame=none]+|===+|Optional |Default: will default to the default value of `includeAll` parameter specified below+|===+++If set to true, returns the properties of the cluster.++`roles`::+++[%autowidth,frame=none]+|===+|Optional |Default: will default to the default value of `includeAll` parameter specified below+|===+++If set to true, returns the roles within the cluster.++`includeAll`::+++[%autowidth,frame=none]+|===+|Optional |Default: true+|===+++If set to `true`, returns all information pertaining to live nodes, collections, aliases, cluster properties, roles, etc.+If set to `false`, the information returned is based on the other specified parameters.+=== CLUSTERSTATUS ResponseThe response will include the status of the request and the status of the cluster.diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.javaindex d5e2d188a75..2555a0cba4f 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java@@ -84,6 +84,13 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid/** Create a SolrClient implementation that uses the specified Solr node URL */protected abstract SolrClient getSolrClient(String baseUrl);+ @Override+ public DocCollection getCollection(String collection) {+ // This change is to prevent BaseHttpCSP make a call to fetch the entire cluster state, as the+ // default implementation calls getClusterState().getCollectionOrNull(name)+ return getState(collection).get();+ }+@Overridepublic ClusterState.CollectionRef getState(String collection) {for (String nodeName : liveNodes) {@@ -122,15 +129,9 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidprivate ClusterState fetchClusterState(SolrClient client, String collection, Map<String, Object> clusterProperties)throws SolrServerException, IOException, NotACollectionException {- ModifiableSolrParams params = new ModifiableSolrParams();- if (collection != null) {- params.set("collection", collection);- }- params.set("action", "CLUSTERSTATUS");- params.set("prs", "true");- QueryRequest request = new QueryRequest(params);- request.setPath("/admin/collections");- SimpleOrderedMap<?> cluster = (SimpleOrderedMap<?>) client.request(request).get("cluster");+ SimpleOrderedMap<?> cluster =+ submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION);+Map<String, Object> collectionsMap;if (collection != null) {collectionsMap =@@ -149,10 +150,16 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid} else {znodeVersion = -1;}- Set<String> liveNodes = new HashSet<>((List<String>) (cluster.get("live_nodes")));- this.liveNodes = liveNodes;- liveNodesTimestamp = System.nanoTime();- ClusterState cs = new ClusterState(liveNodes, new HashMap<>());++ ClusterState cs = new ClusterState(this.liveNodes, new HashMap<>());+ List<String> liveNodesList = (List<String>) cluster.get("live_nodes");+ if (liveNodesList != null) {+ Set<String> liveNodes = new HashSet<>(liveNodesList);+ this.liveNodes = liveNodes;+ liveNodesTimestamp = System.nanoTime();+ cs = new ClusterState(liveNodes, new HashMap<>());+ }+for (Map.Entry<String, Object> e : collectionsMap.entrySet()) {@SuppressWarnings("rawtypes")Map m = (Map) e.getValue();@@ -173,6 +180,30 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidreturn cs;}+ private SimpleOrderedMap<?> submitClusterStateRequest(+ SolrClient client, String collection, ClusterStateRequestType requestType)+ throws SolrServerException, IOException {++ ModifiableSolrParams params = new ModifiableSolrParams();+ params.set("action", "CLUSTERSTATUS");++ if (requestType == ClusterStateRequestType.FETCH_COLLECTION && collection != null) {+ params.set("collection", collection);+ } else if (requestType == ClusterStateRequestType.FETCH_LIVE_NODES) {+ params.set("liveNodes", "true");+ } else if (requestType == ClusterStateRequestType.FETCH_CLUSTER_PROP) {+ params.set("clusterProperties", "true");+ } else if (requestType == ClusterStateRequestType.FETCH_NODE_ROLES) {+ params.set("roles", "true");+ }++ params.set("includeAll", "false");+ params.set("prs", "true");+ QueryRequest request = new QueryRequest(params);+ request.setPath("/admin/collections");+ return (SimpleOrderedMap<?>) client.request(request).get("cluster");+ }+@SuppressWarnings({"rawtypes", "unchecked"})private DocCollection fillPrs(int znodeVersion, Map.Entry<String, Object> e, Instant creationTime, Map m) {@@ -228,12 +259,10 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid}@SuppressWarnings({"rawtypes", "unchecked"})- private static Set<String> fetchLiveNodes(SolrClient client) throws Exception {- ModifiableSolrParams params = new ModifiableSolrParams();- params.set("action", "CLUSTERSTATUS");- QueryRequest request = new QueryRequest(params);- request.setPath("/admin/collections");- NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster");+ private Set<String> fetchLiveNodes(SolrClient client) throws Exception {++ SimpleOrderedMap<?> cluster =+ submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_LIVE_NODES);return (Set<String>) new HashSet((List<String>) (cluster.get("live_nodes")));}@@ -335,21 +364,18 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid+ " solrUrl(s) or zkHost(s).");}+ @SuppressWarnings("unchecked")@Overridepublic Map<String, Object> getClusterProperties() {+ // Map<String, Object> clusterPropertiesMap = new HashMap<>();for (String nodeName : liveNodes) {String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme);try (SolrClient client = getSolrClient(baseUrl)) {- Map<String, Object> clusterProperties = new HashMap<>();- fetchClusterState(client, null, clusterProperties);- return clusterProperties;+ SimpleOrderedMap<?> cluster =+ submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_CLUSTER_PROP);+ return (Map<String, Object>) cluster.get("properties");} catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException | IOException e) {log.warn("Attempt to fetch cluster state from {} failed.", baseUrl, e);- } catch (NotACollectionException e) {- // not possible! (we passed in null for collection so it can't be an alias)- throw new RuntimeException(- "null should never cause NotACollectionException in "- + "fetchClusterState() Please report this as a bug!");}}throw new RuntimeException(@@ -399,4 +425,11 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid}return String.join(",", this.liveNodes);}++ private enum ClusterStateRequestType {+ FETCH_LIVE_NODES,+ FETCH_CLUSTER_PROP,+ FETCH_NODE_ROLES,+ FETCH_COLLECTION+ }}diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.javaindex 271afca7461..b6afac3114f 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java@@ -91,7 +91,11 @@ public interface ClusterStateProvider extends SolrCloseable {.anyMatch(e -> e.getKey().startsWith(CollectionAdminParams.ROUTER_PREFIX));}- /** Obtain the current cluster state. */+ /**+ * Obtain the current cluster state. WARNING: This method is quite expensive as it involves+ * fetching remote information. Use with caution and be aware of the potential performance+ * implications.+ */ClusterState getClusterState();default DocCollection getCollection(String name) throws IOException {diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.javaindex 6f9cb774beb..2a0693c2360 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java@@ -35,7 +35,6 @@ import java.util.Optional;import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;-import java.util.regex.Pattern;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.client.solrj.SolrClient;@@ -93,15 +92,6 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {private static CloudSolrClient httpBasedCloudSolrClient = null;private static CloudSolrClient zkBasedCloudSolrClient = null;- private static final Pattern PATTERN_WITH_COLLECTION =- Pattern.compile(- "path=/admin/collections.*params=\\{[^}]*action=CLUSTERSTATUS"- + "[^}]*collection=[^&}]+[^}]*\\}");- private static final Pattern PATTERN_WITHOUT_COLLECTION =- Pattern.compile(- "path=/admin/collections.*params=\\{[^}]*action=CLUSTERSTATUS"- + "(?![^}]*collection=)[^}]*\\}");-@BeforeClasspublic static void setupCluster() throws Exception {System.setProperty("metricsEnabled", "true");@@ -269,29 +259,40 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {.process(cluster.getSolrClient());cluster.waitForActiveCollection(collectionName, 2, 2);- try (LogListener entireClusterStateLogs =- LogListener.info(HttpSolrCall.class).regex(PATTERN_WITHOUT_COLLECTION);- LogListener collectionClusterStateLogs =- LogListener.info(HttpSolrCall.class).regex(PATTERN_WITH_COLLECTION);- LogListener adminRequestLogs = LogListener.info(HttpSolrCall.class).substring("[admin]");+ try (LogListener adminLogs = LogListener.info(HttpSolrCall.class).substring("[admin]");CloudSolrClient solrClient = createHttpCSPBasedCloudSolrClient(); ) {++ assertEquals(1, adminLogs.getCount());+ assertTrue(+ adminLogs+ .pollMessage()+ .contains(+ "path=/admin/collections params={prs=true&liveNodes=true&action"+ + "=CLUSTERSTATUS&includeAll=false"));+SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc");solrClient.add(collectionName, doc);++ // getCount seems to return a cumulative count, but add() results in only 1 additional admin+ // request to fetch CLUSTERSTATUS for the collection+ assertEquals(2, adminLogs.getCount());+ assertTrue(+ adminLogs+ .pollMessage()+ .contains(+ "path=/admin/collections "+ + "params={prs=true&action=CLUSTERSTATUS&includeAll=false"));+solrClient.commit(collectionName);+ // No additional admin requests sent+ assertEquals(2, adminLogs.getCount());+for (int i = 0; i < 3; i++) {assertEquals(1, solrClient.query(collectionName, params("q", "*:*")).getResults().getNumFound());+ // No additional admin requests sent+ assertEquals(2, adminLogs.getCount());}-- // 1 call to fetch entire cluster state via BaseHttpCSP.fetchLiveNodes()- // 1 call to fetch CLUSTERSTATUS for collection via getDocCollection() (first collection- // lookup)- assertLogCount(adminRequestLogs, 2);- // 1 call to fetch CLUSTERSTATUS for collection via getDocCollection() (first collection- // lookup)- assertLogCount(collectionClusterStateLogs, 1);- // 1 call to fetch entire cluster state from HttpCSP.fetchLiveNodes()- assertLogCount(entireClusterStateLogs, 1);}}@@ -301,16 +302,6 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {return new CloudHttp2SolrClient.Builder(solrUrls).build();}- private void assertLogCount(LogListener logListener, int expectedCount) {- int logCount = logListener.getCount();- assertEquals(expectedCount, logCount);- if (logCount > 0) {- for (int i = 0; i < logCount; i++) {- logListener.pollMessage();- }- }- }-@Testpublic void testRouting() throws Exception {CollectionAdminRequest.createCollection("routing_collection", "conf", 2, 1)diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.javaindex db385e26b83..1242cde9945 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java@@ -23,7 +23,6 @@ import java.util.List;import java.util.Map;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.util.Utils;@@ -88,7 +87,7 @@ public class HttpClusterStateSSLTest extends SolrCloudTestCase {new CloudSolrClient.Builder(Collections.singletonList(url0.toExternalForm())).build()) {ClusterStateProvider csp = httpBasedCloudSolrClient.getClusterStateProvider();assertTrue(csp instanceof Http2ClusterStateProvider);- verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas);+ verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas);}// http2@@ -97,20 +96,19 @@ public class HttpClusterStateSSLTest extends SolrCloudTestCase {.build()) {ClusterStateProvider csp = http2BasedClient.getClusterStateProvider();assertTrue(csp instanceof Http2ClusterStateProvider);- verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas);+ verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas);}// Zk cluster state nowClusterStateProvider csp = cluster.getSolrClient().getClusterStateProvider();assertTrue(csp instanceof ZkClientClusterStateProvider);- verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas);+ verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas);}private void verifyUrlSchemeInClusterState(- final ClusterState cs, final String collectionId, final int expectedReplicas) {- DocCollection dc = cs.getCollection(collectionId);- assertNotNull(dc);- List<Replica> replicas = dc.getReplicas();+ final DocCollection collection, final int expectedReplicas) {+ assertNotNull(collection);+ List<Replica> replicas = collection.getReplicas();assertNotNull(replicas);assertEquals(expectedReplicas, replicas.size());for (Replica r : replicas) {