Apache Solr open-source search software
SOLR-18130: New universal connection string to CloudSolrClient.Builder (#4260)
vvova15bf610531f1c0
diff --git a/changelog/unreleased/SOLR-18130-universalConnectionString.yml b/changelog/unreleased/SOLR-18130-universalConnectionString.ymlnew file mode 100644index 00000000000..fc235bccd0c--- /dev/null+++ b/changelog/unreleased/SOLR-18130-universalConnectionString.yml@@ -0,0 +1,7 @@+title: Added a universal connection string (ZK/HTTP) to CloudSolrClient.Builder+type: added+authors:+ - name: Vladimir Vyatkin+links:+ - name: SOLR-18130+ url: https://issues.apache.org/jira/browse/SOLR-18130diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.javaindex 3f5013fe664..1e8fdc9293b 100644--- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java+++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java@@ -18,7 +18,6 @@ package org.apache.solr.client.solrj.io;import java.io.Closeable;import java.util.HashMap;-import java.util.List;import java.util.Map;import java.util.Objects;import java.util.Optional;@@ -77,29 +76,36 @@ public class SolrClientCache implements Closeable {}}- public synchronized CloudSolrClient getCloudSolrClient(String zkHost) {+ public synchronized CloudSolrClient getCloudSolrClient(String connectionString) {ensureOpen();- Objects.requireNonNull(zkHost, "ZooKeeper host cannot be null!");- if (solrClients.containsKey(zkHost)) {- return (CloudSolrClient) solrClients.get(zkHost);+ Objects.requireNonNull(connectionString, "Connection string cannot be null!");+ if (solrClients.containsKey(connectionString)) {+ return (CloudSolrClient) solrClients.get(connectionString);}// Can only use ZK ACLs if there is a default ZK Host, and the given ZK host contains that// default.// Basically the ZK ACLs are assumed to be only used for the default ZK host,// thus we should only provide the ACLs to that Zookeeper instance.- String zkHostNoChroot = zkHost.split("/")[0];- boolean canUseACLs =- Optional.ofNullable(defaultZkHost.get()).map(zkHostNoChroot::equals).orElse(false);+ boolean canUseACLs = false;+ var cloudClientConnection = CloudSolrClient.CloudSolrClientConnection.parse(connectionString);+ if (cloudClientConnection.isZk()) {+ String chroot = cloudClientConnection.zkChroot();+ String zkHostNoChroot =+ chroot != null && connectionString.endsWith(chroot)+ ? connectionString.substring(0, connectionString.length() - chroot.length())+ : connectionString;+ canUseACLs =+ Optional.ofNullable(defaultZkHost.get()).map(zkHostNoChroot::equals).orElse(false);+ }- final var client = newCloudSolrClient(zkHost, httpSolrClient, canUseACLs);- solrClients.put(zkHost, client);+ final var client = newCloudSolrClient(connectionString, httpSolrClient, canUseACLs);+ solrClients.put(connectionString, client);return client;}protected CloudSolrClient newCloudSolrClient(- String zkHost, HttpSolrClientBase httpSolrClient, boolean canUseACLs) {- final List<String> hosts = List.of(zkHost);- var builder = new CloudSolrClient.Builder(hosts, Optional.empty());+ String connectionString, HttpSolrClientBase httpSolrClient, boolean canUseACLs) {+ var builder = new CloudSolrClient.Builder(connectionString);builder.canUseZkACLs(canUseACLs);// using internal builder to ensure the internal client gets closedbuilder = builder.withHttpClientBuilder(newHttpSolrClientBuilder(null, httpSolrClient));diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.javaindex 91b5364ed7f..f38bc577651 100644--- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java+++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java@@ -17,13 +17,16 @@package org.apache.solr.client.solrj.io;import java.util.Map;+import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.SolrException;+import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DigestZkACLProvider;import org.apache.solr.common.cloud.DigestZkCredentialsProvider;import org.apache.solr.common.cloud.SolrZkClient;import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector;import org.junit.AfterClass;+import org.junit.Assert;import org.junit.BeforeClass;import org.junit.Test;@@ -66,6 +69,27 @@ public class SolrClientCacheTest extends SolrCloudTestCase {}}+ @Test+ public void testGetClientWithHttp() {+ String solrUrl = cluster.getJettySolrRunner(0).getBaseUrl().toString();+ try (SolrClientCache cache = new SolrClientCache()) {+ CloudSolrClient cloudSolrClient = cache.getCloudSolrClient(solrUrl);+ ClusterState clusterState = cloudSolrClient.getClusterStateProvider().getClusterState();+ Assert.assertEquals(1, clusterState.getLiveNodes().size());+ }+ }++ @Test+ public void testGetClientWithZookeeper() {+ String zkConnectionString = zkClient().getZkServerAddress();+ try (SolrClientCache cache = new SolrClientCache()) {+ cache.setDefaultZKHost(zkClient().getZkServerAddress());+ CloudSolrClient cloudSolrClient = cache.getCloudSolrClient(zkConnectionString);+ ClusterState clusterState = cloudSolrClient.getClusterStateProvider().getClusterState();+ Assert.assertEquals(1, clusterState.getLiveNodes().size());+ }+ }+@Testpublic void testZkACLsUsedWithDifferentChroot() {try (SolrClientCache cache = new SolrClientCache()) {diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.javaindex 4f71b0cba79..3f79b11801c 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java@@ -241,6 +241,30 @@ public abstract class CloudSolrClient extends SolrClient {this.stateProvider = stateProvider;}+ /**+ * Creates a client builder based on a connection string of 2 possible formats:+ *+ * <ul>+ * <li>ZooKeeper connection string (optionally with chroot), e.g. {@code+ * zk1:2181,zk2:2181,zk3:2181/solr}+ * <li>Comma-separated list of Solr node base URLs (HTTP or HTTPS), e.g. {@code+ * http://solr1:8983/solr,http://solr2:8983/solr}+ * </ul>+ *+ * @param connectionString a string specifying either ZooKeeper connection string or HTTP(S)+ * Solr URLs+ * @throws IllegalArgumentException if string is null, empty, or malformed+ */+ public Builder(String connectionString) {+ CloudSolrClientConnection connection = CloudSolrClientConnection.parse(connectionString);+ if (connection.isZk()) {+ this.zkHosts = connection.quorumItems();+ this.zkChroot = connection.zkChroot();+ } else {+ this.solrUrls = connection.quorumItems();+ }+ }+/** Whether to use the default ZK ACLs when building a ZK Client. */public Builder canUseZkACLs(boolean canUseZkACLs) {this.canUseZkACLs = canUseZkACLs;@@ -1842,4 +1866,52 @@ public abstract class CloudSolrClient extends SolrClient {return true;}++ /** Universal connection string parser logic. */+ public record CloudSolrClientConnection(boolean isZk, List<String> quorumItems, String zkChroot) {++ public CloudSolrClientConnection {+ if (quorumItems == null || quorumItems.isEmpty()) {+ throw new IllegalArgumentException("No valid hosts/urls found");+ }+ }++ public static CloudSolrClientConnection parse(String connectionString) {+ if (connectionString == null || connectionString.trim().isEmpty()) {+ throw new IllegalArgumentException("Connection string must not be null or empty");+ }+ connectionString = connectionString.trim();+ if (connectionString.contains("://")) {+ return parseHttpQuorum(connectionString);+ }+ return parseZkQuorum(connectionString);+ }++ private static CloudSolrClientConnection parseZkQuorum(String connectionString) {+ String zkChroot = null;+ String zkHosts = connectionString;+ int slashIndex = connectionString.indexOf('/');+ if (slashIndex != -1) {+ zkHosts = connectionString.substring(0, slashIndex);+ zkChroot = connectionString.substring(slashIndex);+ }+ List<String> quorumItems = StrUtils.split(zkHosts, ',');+ for (String host : quorumItems) {+ if (host == null || host.isBlank()) {+ throw new IllegalArgumentException("Empty host in Zookeeper connection string");+ }+ }+ return new CloudSolrClientConnection(true, quorumItems, zkChroot);+ }++ private static CloudSolrClientConnection parseHttpQuorum(String connectionString) {+ List<String> quorumItems = StrUtils.split(connectionString, ',');+ for (String url : quorumItems) {+ if (url == null || url.isBlank()) {+ throw new IllegalArgumentException("Empty URL in HTTP(S) connection string");+ }+ }+ return new CloudSolrClientConnection(false, quorumItems, null);+ }+ }}diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.javaindex e80c60242a7..ea596e7a11a 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java@@ -105,6 +105,8 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {private static CloudHttp2SolrClient httpJettyBasedCloudSolrClient = null;private static CloudHttp2SolrClient httpJdkBasedCloudSolrClient = null;private static CloudHttp2SolrClient zkBasedCloudSolrClient = null;+ private static CloudHttp2SolrClient connectionStringZkBasedCloudSolrClient = null;+ private static CloudHttp2SolrClient connectionStringHttpBasedCloudSolrClient = null;@BeforeClasspublic static void setupCluster() throws Exception {@@ -161,6 +163,22 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {assertTrue(zkBasedCloudSolrClient.getHttpClient() instanceof HttpJettySolrClient);assertTrue(zkBasedCloudSolrClient.getClusterStateProvider() instanceof ZkClientClusterStateProvider);++ String zkConnString = cluster.getZkServer().getZkAddress();+ connectionStringZkBasedCloudSolrClient = new CloudSolrClient.Builder(zkConnString).build();+ assertTrue(+ connectionStringZkBasedCloudSolrClient.getHttpClient() instanceof HttpJettySolrClient);+ assertTrue(+ connectionStringZkBasedCloudSolrClient.getClusterStateProvider()+ instanceof ZkClientClusterStateProvider);++ String httpConnString = String.join(",", solrUrls);+ connectionStringHttpBasedCloudSolrClient = new CloudSolrClient.Builder(httpConnString).build();+ assertTrue(+ connectionStringHttpBasedCloudSolrClient.getHttpClient() instanceof HttpJettySolrClient);+ assertTrue(+ connectionStringHttpBasedCloudSolrClient.getClusterStateProvider()+ instanceof HttpClusterStateProvider<?>);}@AfterClass@@ -168,23 +186,28 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {IOUtils.closeQuietly(httpJettyBasedCloudSolrClient);IOUtils.closeQuietly(httpJdkBasedCloudSolrClient);IOUtils.closeQuietly(zkBasedCloudSolrClient);+ IOUtils.closeQuietly(connectionStringZkBasedCloudSolrClient);+ IOUtils.closeQuietly(connectionStringHttpBasedCloudSolrClient);shutdownCluster();httpJettyBasedCloudSolrClient = null;httpJdkBasedCloudSolrClient = null;zkBasedCloudSolrClient = null;+ connectionStringZkBasedCloudSolrClient = null;+ connectionStringHttpBasedCloudSolrClient = null;}/** Randomly return the cluster's ZK based CSC, or HttpClusterProvider based CSC. */private CloudSolrClient getRandomClient() {- int randInt = random().nextInt(3);- if (randInt == 0) {- return zkBasedCloudSolrClient;- }- if (randInt == 1) {- return httpJettyBasedCloudSolrClient;- }- return httpJdkBasedCloudSolrClient;+ CloudSolrClient[] clients = {+ zkBasedCloudSolrClient,+ httpJettyBasedCloudSolrClient,+ httpJdkBasedCloudSolrClient,+ connectionStringZkBasedCloudSolrClient,+ connectionStringHttpBasedCloudSolrClient+ };++ return clients[random().nextInt(clients.length)];}@Testdiff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientConnectionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientConnectionTest.javanew file mode 100644index 00000000000..f65a83cfe0e--- /dev/null+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientConnectionTest.java@@ -0,0 +1,148 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.solrj.impl;++import java.util.Collection;+import java.util.List;+import java.util.Set;+import org.apache.solr.SolrTestCase;+import org.junit.Assert;+import org.junit.Test;++public class CloudSolrClientConnectionTest extends SolrTestCase {++ private static <T> boolean collectionEqual(Collection<T> coll, Collection<T> coll2) {+ return coll.size() == coll2.size()+ && Set.copyOf(coll).containsAll(coll2)+ && Set.copyOf(coll2).containsAll(coll);+ }++ @Test+ public void testBuildQuorumForZk() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181/solr");+ Assert.assertTrue(parsed.isZk());+ List<String> expectedQuorum = List.of("zookeeper1:2181", "zookeeper2:2181", "zookeeper3:2181");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertEquals("/solr", parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForZkIpV4() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "192.0.2.10:2181,192.0.2.11:2181,192.0.2.12:2181/solr");+ Assert.assertTrue(parsed.isZk());+ List<String> expectedQuorum = List.of("192.0.2.10:2181", "192.0.2.11:2181", "192.0.2.12:2181");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertEquals("/solr", parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForZkIpV6() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "[2001:db8:85a3::8a2e:370:7334]:2181,"+ + "[2001:db8:85a3::8a2e:370:7335]:2181,"+ + "[2001:db8:85a3::8a2e:370:7336]:2181/solr");+ Assert.assertTrue(parsed.isZk());+ List<String> expectedQuorum =+ List.of(+ "[2001:db8:85a3::8a2e:370:7334]:2181",+ "[2001:db8:85a3::8a2e:370:7335]:2181",+ "[2001:db8:85a3::8a2e:370:7336]:2181");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertEquals("/solr", parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForZkNoChroot() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "zookeeper1:2181,zookeeper2:2181,zookeeper3:2181");+ Assert.assertTrue(parsed.isZk());+ List<String> expectedQuorum = List.of("zookeeper1:2181", "zookeeper2:2181", "zookeeper3:2181");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertNull(parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForHttp() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "http://solr1:8983/solr,http://solr2:8983/solr,http://solr3:8983/solr");+ Assert.assertFalse(parsed.isZk());+ List<String> expectedQuorum =+ List.of("http://solr1:8983/solr", "http://solr2:8983/solr", "http://solr3:8983/solr");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertNull(parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForHttps() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "https://solr1:8983/solr,https://solr2:8983/solr,https://solr3:8983/solr");+ Assert.assertFalse(parsed.isZk());+ List<String> expectedQuorum =+ List.of("https://solr1:8983/solr", "https://solr2:8983/solr", "https://solr3:8983/solr");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertNull(parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForIpV4() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "http://192.0.2.10:8983/solr,"+ + "http://192.0.2.11:8983/solr,"+ + "http://192.0.2.12:8983/solr");+ Assert.assertFalse(parsed.isZk());+ List<String> expectedQuorum =+ List.of(+ "http://192.0.2.10:8983/solr",+ "http://192.0.2.11:8983/solr",+ "http://192.0.2.12:8983/solr");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertNull(parsed.zkChroot());+ }++ @Test+ public void testBuildQuorumForIpV6() {+ CloudSolrClient.CloudSolrClientConnection parsed =+ CloudSolrClient.CloudSolrClientConnection.parse(+ "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr,"+ + "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr,"+ + "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr");+ Assert.assertFalse(parsed.isZk());+ List<String> expectedQuorum =+ List.of(+ "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr",+ "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr",+ "http://[2001:db8:85a3::8a2e:370:7334]:8983/solr");+ Assert.assertTrue(collectionEqual(expectedQuorum, parsed.quorumItems()));+ Assert.assertNull(parsed.zkChroot());+ }++ @Test+ public void testEmptyOrNullConnStringException() {+ Assert.assertThrows(+ IllegalArgumentException.class,+ () -> CloudSolrClient.CloudSolrClientConnection.parse(" "));+ }+}