Apache Solr open-source search software
SOLR-17607: Http ClusterStateProvider, lazy connect (#3249)
David Smiley9918ee640dce
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f2bd8735fb3..c575e6cc5b3 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -183,6 +183,8 @@ Improvements`org.apache.solr.client.solrj.request.FileStoreApi`. The older form of this endpoint (`GET /api/node/files`) is deprecated and will beremoved. (Jason Gerlowski)+* SOLR-17607: SolrJ CloudSolrClient configured with HTTP URLs will no longer eagerly connect to+ anything. (David Smiley)Optimizations---------------------diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.javaindex 150b4fe0de2..d3f002836ec 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java@@ -56,7 +56,7 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidprivate String urlScheme;private List<URL> configuredNodes;- volatile Set<String> liveNodes;+ volatile Set<String> liveNodes; // initially null then never nulllong liveNodesTimestamp = 0;volatile Map<String, List<String>> aliases;volatile Map<String, Map<String, String>> aliasProperties;@@ -65,41 +65,24 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid// the liveNodes and aliases cache will be invalidated after 5 secsprivate int cacheTimeout = EnvUtils.getPropertyAsInteger("solr.solrj.cache.timeout.sec", 5);- public void init(List<String> solrUrls) throws Exception {+ protected void initConfiguredNodes(List<String> solrUrls) throws Exception {this.configuredNodes =solrUrls.stream()- .map(- (solrUrl) -> {- try {- return new URI(solrUrl).toURL();- } catch (MalformedURLException | URISyntaxException e) {- throw new IllegalArgumentException(- "Failed to parse base Solr URL " + solrUrl, e);- }- })+ .map(BaseHttpClusterStateProvider::stringToUrl).collect(Collectors.toList());+ }- for (String solrUrl : solrUrls) {- urlScheme = solrUrl.startsWith("https") ? "https" : "http";- try (SolrClient initialClient = getSolrClient(solrUrl)) {- this.liveNodes = fetchLiveNodes(initialClient);- liveNodesTimestamp = System.nanoTime();- break;- } catch (SolrServerException | IOException e) {- log.warn("Attempt to fetch cluster state from {} failed.", solrUrl, e);- }+ private static URL stringToUrl(String solrUrl) {+ try {+ return new URI(solrUrl).toURL();+ } catch (MalformedURLException | URISyntaxException e) {+ throw new IllegalArgumentException("Failed to parse base Solr URL " + solrUrl, e);}+ }- if (this.liveNodes == null || this.liveNodes.isEmpty()) {- throw new RuntimeException(- "Tried fetching live_nodes using Solr URLs provided, i.e. "- + solrUrls- + ". However, "- + "succeeded in obtaining the cluster state from none of them."- + "If you think your Solr cluster is up and is accessible,"- + " you could try re-creating a new CloudSolrClient using working"- + " solrUrl(s).");- }+ @Override+ public void connect() {+ getLiveNodes();}/** Create a SolrClient implementation that uses the specified Solr node URL */@@ -114,7 +97,7 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid@Overridepublic ClusterState.CollectionRef getState(String collection) {- for (String nodeName : liveNodes) {+ for (String nodeName : getLiveNodes()) {String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme);try (SolrClient client = getSolrClient(baseUrl)) {DocCollection docCollection = fetchCollectionState(client, collection);@@ -230,43 +213,44 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid}@Override- public Set<String> getLiveNodes() {- if (TimeUnit.SECONDS.convert((System.nanoTime() - liveNodesTimestamp), TimeUnit.NANOSECONDS)- > getCacheTimeout()) {+ public synchronized Set<String> getLiveNodes() {+ // synchronized because there's no value in multiple doing this at the same time++ // only in the initial state, liveNodes is null+ if (liveNodes != null) {+ if (TimeUnit.SECONDS.convert((System.nanoTime() - liveNodesTimestamp), TimeUnit.NANOSECONDS)+ <= getCacheTimeout()) {+ return this.liveNodes; // cached copy is fresh enough+ }if (liveNodes.stream()- .anyMatch((node) -> updateLiveNodes(URLUtil.getBaseUrlForNodeName(node, urlScheme))))- return this.liveNodes;+ .map(node -> URLUtil.getBaseUrlForNodeName(node, urlScheme))+ .map(BaseHttpClusterStateProvider::stringToUrl)+ .anyMatch(this::updateLiveNodes)) return this.liveNodes;- log.warn(- "Attempt to fetch cluster state from all known live nodes {} failed. Trying backup nodes",- liveNodes);+ log.warn("Failed fetching live_nodes from {}. Trying configured nodes...", liveNodes);+ }- if (configuredNodes.stream().anyMatch((node) -> updateLiveNodes(node.toString())))- return this.liveNodes;+ if (configuredNodes.stream().anyMatch(this::updateLiveNodes)) return this.liveNodes;- throw new RuntimeException(- "Tried fetching live_nodes using all the node names we knew of, i.e. "- + liveNodes- + ". However, "- + "succeeded in obtaining the cluster state from none of them."- + "If you think your Solr cluster is up and is accessible,"- + " you could try re-creating a new CloudSolrClient using working"- + " solrUrl(s).");- } else {- return this.liveNodes; // cached copy is fresh enough- }+ throw new RuntimeException(+ "Failed fetching live_nodes from "+ + configuredNodes+ + ". If you think your Solr cluster is up and is accessible,"+ + " you could try re-creating a new CloudSolrClient using working"+ + " solr URLs.");}- private boolean updateLiveNodes(String liveNode) {- try (SolrClient client = getSolrClient(liveNode)) {+ private boolean updateLiveNodes(URL url) {+ try (SolrClient client = getSolrClient(url.toString())) {this.liveNodes = fetchLiveNodes(client);liveNodesTimestamp = System.nanoTime();+ urlScheme = url.getProtocol();return true;} catch (Exception e) {- log.warn("Attempt to fetch cluster state from {} failed.", liveNode, e);+ log.warn("Attempt to fetch live_nodes from {} failed.", url, e);+ return false;}- return false;}@SuppressWarnings({"unchecked"})@@ -291,20 +275,11 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid}private Map<String, List<String>> getAliases(boolean forceFetch) {- if (this.liveNodes == null) {- throw new RuntimeException(- "We don't know of any live_nodes to fetch the"- + " latest aliases information from. "- + "If you think your Solr cluster is up and is accessible,"- + " you could try re-creating a new CloudSolrClient using working"- + " solrUrl(s).");- }-if (forceFetch|| this.aliases == null|| TimeUnit.SECONDS.convert((System.nanoTime() - aliasesTimestamp), TimeUnit.NANOSECONDS)> getCacheTimeout()) {- for (String nodeName : liveNodes) {+ for (String nodeName : getLiveNodes()) {String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme);try (SolrClient client = getSolrClient(baseUrl)) {@@ -351,7 +326,7 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid@Overridepublic ClusterState getClusterState() {- for (String nodeName : liveNodes) {+ for (String nodeName : getLiveNodes()) {String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme);try (SolrClient client = getSolrClient(baseUrl)) {return fetchClusterState(client);@@ -377,7 +352,7 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid@SuppressWarnings("unchecked")@Overridepublic Map<String, Object> getClusterProperties() {- for (String nodeName : liveNodes) {+ for (String nodeName : getLiveNodes()) {String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme);try (SolrClient client = getSolrClient(baseUrl)) {SimpleOrderedMap<?> cluster =@@ -413,9 +388,6 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvidreturn getClusterProperties().get(propertyName);}- @Override- public void connect() {}-public int getCacheTimeout() {return cacheTimeout;}diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2ClusterStateProvider.javaindex 3bb222ae430..a7abb80b71d 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2ClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2ClusterStateProvider.java@@ -29,7 +29,7 @@ public class Http2ClusterStateProvider extends BaseHttpClusterStateProvider {throws Exception {this.httpClient = httpClient == null ? new Http2SolrClient.Builder().build() : httpClient;this.closeClient = httpClient == null;- init(solrUrls);+ initConfiguredNodes(solrUrls);}@Overridediff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.javaindex 56c6b08c7d9..353c15fd1b7 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpClusterStateProvider.java@@ -35,7 +35,7 @@ public class HttpClusterStateProvider extends BaseHttpClusterStateProvider {public HttpClusterStateProvider(List<String> solrUrls, HttpClient httpClient) throws Exception {this.httpClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient;this.clientIsInternal = httpClient == null;- init(solrUrls);+ initConfiguredNodes(solrUrls);}@Overridediff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.javaindex c98d75268f6..e8233413be1 100644--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java@@ -260,6 +260,7 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase {try (LogListener adminLogs = LogListener.info(HttpSolrCall.class).substring("[admin]");CloudSolrClient solrClient = createHttpCSPBasedCloudSolrClient(); ) {+ solrClient.getClusterStateProvider().getLiveNodes(); // talks to SolrassertEquals(1, adminLogs.getCount());assertTrue(