Apache Solr open-source search software
SOLR-17582: Stream CLUSTERSTATUS API for SolrJ version >= 9.9 (#3156)
Matthew Biscocho5e328ee77e5e
diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/cli/StatusTool.javaindex fa6c2cbf2bd..a935a7b364f 100644--- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java+++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java@@ -353,7 +353,7 @@ public class StatusTool extends ToolBase {cloudStatus.put("liveNodes", String.valueOf(liveNodes.size()));// TODO get this as a metric from the metrics API instead, or something else.- var collections = (NamedList<Object>) json.findRecursive("cluster", "collections");+ var collections = (Map<String, Object>) json.findRecursive("cluster", "collections");cloudStatus.put("collections", String.valueOf(collections.size()));return cloudStatus;diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.javaindex 7a8ecf9c850..6ef549698d3 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java@@ -26,6 +26,7 @@ import java.util.Map;import java.util.Objects;import java.util.Set;import java.util.stream.Stream;+import org.apache.solr.client.api.util.SolrVersion;import org.apache.solr.common.MapWriter;import org.apache.solr.common.SolrException;import org.apache.solr.common.cloud.Aliases;@@ -34,7 +35,6 @@ import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.PerReplicaStates;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ShardParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;@@ -101,7 +101,7 @@ public class ClusterStatus {collection = params.get(ZkStateReader.COLLECTION_PROP);}- public void getClusterStatus(NamedList<Object> results)+ public void getClusterStatus(NamedList<Object> results, SolrVersion solrVersion)throws KeeperException, InterruptedException {NamedList<Object> clusterStatus = new SimpleOrderedMap<>();@@ -127,7 +127,7 @@ public class ClusterStatus {if (withCollection) {assert liveNodes != null;- fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);+ fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases, solrVersion);}if (withAliases) {@@ -158,7 +158,10 @@ public class ClusterStatus {}private void fetchClusterStatusForCollOrAlias(- NamedList<Object> clusterStatus, List<String> liveNodes, Aliases aliases) {+ NamedList<Object> clusterStatus,+ List<String> liveNodes,+ Aliases aliases,+ SolrVersion solrVersion) {// read aliasesMap<String, List<String>> collectionVsAliases = new HashMap<>();@@ -206,19 +209,7 @@ public class ClusterStatus {}}- // Because of back-compat for SolrJ, create the whole response into a NamedList- // Otherwise stream with MapWriter to save memory- if (CommonParams.JAVABIN.equals(solrParams.get(CommonParams.WT))) {- NamedList<Object> collectionProps = new SimpleOrderedMap<>();- collectionStream.forEach(- collectionState -> {- collectionProps.add(- collectionState.getName(),- buildResponseForCollection(- collectionState, collectionVsAliases, routeKey, liveNodes, requestedShards));- });- clusterStatus.add("collections", collectionProps);- } else {+ if (solrVersion == null || solrVersion.greaterThanOrEqualTo(SolrVersion.valueOf("9.9.0"))) {MapWriter collectionPropsWriter =ew -> {collectionStream.forEach(@@ -234,6 +225,16 @@ public class ClusterStatus {});};clusterStatus.add("collections", collectionPropsWriter);+ } else {+ NamedList<Object> collectionProps = new SimpleOrderedMap<>();+ collectionStream.forEach(+ collectionState -> {+ collectionProps.add(+ collectionState.getName(),+ buildResponseForCollection(+ collectionState, collectionVsAliases, routeKey, liveNodes, requestedShards));+ });+ clusterStatus.add("collections", collectionProps);}}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.javaindex aefc1033d5e..3b22b3072e5 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java@@ -975,7 +975,7 @@ public class CollectionsHandler extends RequestHandlerBase implements PermissionCLUSTERSTATUS,(req, rsp, h) -> {new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams())- .getClusterStatus(rsp.getValues());+ .getClusterStatus(rsp.getValues(), req.getHttpSolrCall().getUserAgentSolrVersion());return null;}),ADDREPLICAPROP_OP(diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.javaindex 0aa5d4ae2d0..ce605c70271 100644--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java@@ -131,10 +131,11 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {.getResponse();NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertEquals(1, collections.size());- assertEquals("25", collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));+ Map<?, ?> collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);+ assertEquals("25", collectionProperties.get("replicationFactor"));params = new ModifiableSolrParams();params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());@@ -153,10 +154,11 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {System.out.println(rsp);cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- collections = (NamedList<?>) cluster.get("collections");+ collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertEquals(1, collections.size());- assertNull(collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));+ collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);+ assertNull(collectionProperties.get("replicationFactor"));params = new ModifiableSolrParams();params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());@@ -255,7 +257,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<?> rsp = client.request(req);NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull("Testing to insure collections are returned", collections.get(COLLECTION_NAME1));@@ -282,7 +284,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = client.request(request);NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertEquals(1, collections.size());@SuppressWarnings({"unchecked"})@@ -304,7 +306,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = client.request(request);NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(COLLECTION_NAME));assertEquals(1, collections.size());@@ -330,7 +332,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = request.process(client).getResponse();NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(COLLECTION_NAME));assertEquals(1, collections.size());@@ -465,7 +467,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = client.request(request);NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(COLLECTION_NAME1));assertEquals(4, collections.size());@@ -487,7 +489,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = client.request(request);NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<?> collections = (NamedList<?>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertEquals(1, collections.size());@SuppressWarnings({"unchecked"})@@ -517,7 +519,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> rsp = client.request(request);NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertEquals(1, collections.size());Map<String, Object> collection = (Map<String, Object>) collections.get(cname);@@ -533,7 +535,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {rsp = client.request(request);cluster = (NamedList<Object>) rsp.get("cluster");- collections = (NamedList<Object>) cluster.get("collections");+ collections = (Map<?, ?>) cluster.get("collections");collection = (Map<String, Object>) collections.get(cname);Integer newVersion = (Integer) collection.get("znodeVersion");assertNotNull(newVersion);@@ -560,7 +562,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);@SuppressWarnings({"unchecked"})- NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(DEFAULT_COLLECTION));assertEquals(1, collections.size());@@ -607,7 +609,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {DEFAULT_COLLECTION + "," + COLLECTION_NAME,aliases.get("myalias"));- NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");+ Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(DEFAULT_COLLECTION));Map<String, Object> collection = (Map<String, Object>) collections.get(DEFAULT_COLLECTION);@@ -627,7 +629,7 @@ public class TestCollectionAPI extends ReplicaPropertiesBase {cluster = (NamedList<Object>) rsp.get("cluster");assertNotNull("Cluster state should not be null", cluster);- collections = (NamedList<Object>) cluster.get("collections");+ collections = (Map<?, ?>) cluster.get("collections");assertNotNull("Collections should not be null in cluster state", collections);assertNotNull(collections.get(DEFAULT_COLLECTION));assertNotNull(collections.get(COLLECTION_NAME));diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.javaindex a09d936582e..150b4fe0de2 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java@@ -29,7 +29,6 @@ import java.time.Instant;import java.util.Collections;import java.util.List;import java.util.Map;-import java.util.Map.Entry;import java.util.Set;import java.util.concurrent.TimeUnit;import java.util.stream.Collectors;@@ -46,7 +45,6 @@ import org.apache.solr.common.cloud.PerReplicaStates;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.CollectionUtil;import org.apache.solr.common.util.EnvUtils;-import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.SimpleOrderedMap;import org.apache.solr.common.util.URLUtil;import org.apache.solr.common.util.Utils;@@ -160,15 +158,12 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidliveNodesTimestamp = System.nanoTime();}- var collectionsNl = (NamedList<Map<String, Object>>) cluster.get("collections");+ var collectionsMap = (Map<String, Map<String, Object>>) cluster.get("collections");Map<String, DocCollection> collStateByName =- CollectionUtil.newLinkedHashMap(collectionsNl.size());- for (Entry<String, Map<String, Object>> entry : collectionsNl) {- collStateByName.put(- entry.getKey(), getDocCollectionFromObjects(entry.getKey(), entry.getValue()));- }-+ CollectionUtil.newLinkedHashMap(collectionsMap.size());+ collectionsMap.forEach(+ (key, value) -> collStateByName.put(key, getDocCollectionFromObjects(key, value)));return new ClusterState(this.liveNodes, collStateByName);}@@ -205,7 +200,7 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidSimpleOrderedMap<?> cluster =submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION);- var collStateMap = (Map<String, Object>) cluster.findRecursive("collections", collection);+ var collStateMap = (Map<String, Object>) cluster._get(List.of("collections", collection), null);if (collStateMap == null) {throw new NotACollectionException(); // probably an alias}diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.javaindex f3ee722ee58..5c6ce5fbdb9 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java@@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo;import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;import java.io.IOException;+import java.lang.invoke.MethodHandles;import java.lang.reflect.InvocationTargetException;import java.time.Instant;import java.util.List;@@ -36,6 +37,8 @@ import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.util.NamedList;+import org.eclipse.jetty.http.HttpField;+import org.eclipse.jetty.http.HttpHeader;import org.hamcrest.Matchers;import org.junit.After;import org.junit.BeforeClass;@@ -135,7 +138,6 @@ public class ClusterStateProviderTest extends SolrCloudTestCase {try (ClusterStateProvider provider = createClusterStateProvider()) {ClusterState.CollectionRef collectionRef = provider.getState("testGetState");-DocCollection docCollection = collectionRef.get();assertNotNull(docCollection);assertEquals(@@ -159,7 +161,7 @@ public class ClusterStateProviderTest extends SolrCloudTestCase {NamedList<Object> response = clusterStatusResponse.getResponse();NamedList<Object> cluster = (NamedList<Object>) response.get("cluster");- NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");+ Map<String, Object> collections = (Map<String, Object>) cluster.get("collections");Map<String, Object> collection = (Map<String, Object>) collections.get(collectionName);return Instant.ofEpochMilli((long) collection.get("creationTimeMillis"));}@@ -198,6 +200,70 @@ public class ClusterStateProviderTest extends SolrCloudTestCase {}}+ @Test+ public void testClusterStateProviderOldVersion() throws SolrServerException, IOException {+ CollectionAdminRequest.setClusterProperty("ext.foo", "bar").process(cluster.getSolrClient());+ createCollection("col1");+ createCollection("col2");++ try (var cspZk = zkClientClusterStateProvider();+ var cspHttp = http2ClusterStateProvider()) {+ // SolrJ < version 9.9.0 for non streamed response+ cspHttp+ .getHttpClient()+ .getHttpClient()+ .setUserAgentField(+ new HttpField(+ HttpHeader.USER_AGENT,+ "Solr[" + MethodHandles.lookup().lookupClass().getName() + "] " + "9.8.0"));++ assertThat(cspHttp.getCollection("col1"), equalTo(cspZk.getCollection("col1")));++ final var clusterStateZk = cspZk.getClusterState();+ final var clusterStateHttp = cspHttp.getClusterState();+ assertThat(+ clusterStateHttp.getLiveNodes(),+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));+ assertEquals(2, clusterStateZk.size());+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());+ assertThat(+ clusterStateHttp.collectionStream().toList(),+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));++ assertThat(+ clusterStateZk.getCollection("col2"), equalTo(clusterStateHttp.getCollection("col2")));+ }+ }++ @Test+ public void testClusterStateProviderEmptySolrVersion() throws SolrServerException, IOException {+ CollectionAdminRequest.setClusterProperty("ext.foo", "bar").process(cluster.getSolrClient());+ createCollection("col1");+ createCollection("col2");++ try (var cspZk = zkClientClusterStateProvider();+ var cspHttp = http2ClusterStateProvider()) {++ cspHttp.getHttpClient().getHttpClient().setUserAgentField(null);++ assertThat(cspHttp.getCollection("col1"), equalTo(cspZk.getCollection("col1")));++ final var clusterStateZk = cspZk.getClusterState();+ final var clusterStateHttp = cspHttp.getClusterState();+ assertThat(+ clusterStateHttp.getLiveNodes(),+ containsInAnyOrder(clusterStateHttp.getLiveNodes().toArray()));+ assertEquals(2, clusterStateZk.size());+ assertEquals(clusterStateZk.size(), clusterStateHttp.size());+ assertThat(+ clusterStateHttp.collectionStream().toList(),+ containsInAnyOrder(clusterStateHttp.collectionStream().toArray()));++ assertThat(+ clusterStateZk.getCollection("col2"), equalTo(clusterStateHttp.getCollection("col2")));+ }+ }+@Testpublic void testClusterStateProviderDownedInitialLiveNodes() throws Exception {try (var cspHttp = http2ClusterStateProvider()) {