Apache Solr open-source search software
SOLR-16404: Use Http2 clients in the Prometheus Exporter (#1004)
Houston Putman9d6d88c8a37a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex f76e667420d..2a4541227f1 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -89,6 +89,8 @@ Improvements* SOLR-16361: mod() is now accurate for all integers, floats, doubles and longs upto 2^52 (Dan Rosher via Eric Pugh)+* SOLR-16404: Prometheus Exporter: Use HTTP2 Solr clients. (Houston Putman)+Optimizations---------------------* SOLR-16120: Optimise hl.fl expansion. (Christine Poerschke, David Smiley, Mike Drob)diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradleindex 1d6e69c3529..613318fa579 100644--- a/solr/prometheus-exporter/build.gradle+++ b/solr/prometheus-exporter/build.gradle@@ -22,6 +22,7 @@ description = 'Prometheus exporter for exposing metrics from Solr using Metricsdependencies {implementation project(':solr:solrj')+ runtimeOnly project(':solr:solrj-zookeeper')// ideally remove ZK depimplementation('org.apache.zookeeper:zookeeper', {exclude group: "org.apache.yetus", module: "audience-annotations"diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.javaindex 877058b2be6..f879dc3b0e7 100644--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrClientFactory.java@@ -19,9 +19,9 @@ package org.apache.solr.prometheus.exporter;import java.util.Optional;import java.util.stream.Collectors;-import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.zookeeper.client.ConnectStringParser;@@ -33,43 +33,34 @@ public class SolrClientFactory {this.settings = settings;}- public HttpSolrClient createStandaloneSolrClient(String solrHost) {- NoOpResponseParser responseParser = new NoOpResponseParser();- responseParser.setWriterType("json");+ public Http2SolrClient createStandaloneSolrClient(String solrHost) {+ Http2SolrClient http2SolrClient =+ new Http2SolrClient.Builder(solrHost)+ .idleTimeout(settings.getHttpReadTimeout())+ .connectionTimeout(settings.getHttpConnectionTimeout())+ .build();- HttpSolrClient.Builder standaloneBuilder = new HttpSolrClient.Builder();+ http2SolrClient.setParser(new NoOpResponseParser("json"));- standaloneBuilder.withBaseSolrUrl(solrHost);-- standaloneBuilder- .withConnectionTimeout(settings.getHttpConnectionTimeout())- .withSocketTimeout(settings.getHttpReadTimeout());-- HttpSolrClient httpSolrClient = standaloneBuilder.build();- httpSolrClient.setParser(responseParser);-- return httpSolrClient;+ return http2SolrClient;}public CloudSolrClient createCloudSolrClient(String zookeeperConnectionString) {- NoOpResponseParser responseParser = new NoOpResponseParser();- responseParser.setWriterType("json");-ConnectStringParser parser = new ConnectStringParser(zookeeperConnectionString);- var cloudBuilder =- new CloudLegacySolrClient.Builder(- parser.getServerAddresses().stream()- .map(address -> address.getHostString() + ":" + address.getPort())- .collect(Collectors.toList()),- Optional.ofNullable(parser.getChrootPath()));-- cloudBuilder- .withConnectionTimeout(settings.getHttpConnectionTimeout())- .withSocketTimeout(settings.getHttpReadTimeout());-- CloudSolrClient client = cloudBuilder.build();- client.setParser(responseParser);+ CloudSolrClient client =+ new CloudHttp2SolrClient.Builder(+ parser.getServerAddresses().stream()+ .map(address -> address.getHostString() + ":" + address.getPort())+ .collect(Collectors.toList()),+ Optional.ofNullable(parser.getChrootPath()))+ .withInternalClientBuilder(+ new Http2SolrClient.Builder()+ .idleTimeout(settings.getHttpReadTimeout())+ .connectionTimeout(settings.getHttpConnectionTimeout()))+ .build();++ client.setParser(new NoOpResponseParser("json"));client.connect();diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.javaindex 3944dc042a9..55ded335c1d 100644--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrCloudScraper.java@@ -26,7 +26,7 @@ import java.util.concurrent.ExecutorService;import java.util.function.Function;import java.util.stream.Collectors;import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;import org.apache.solr.common.cloud.DocCollection;import org.apache.solr.common.cloud.Replica;import org.apache.solr.common.util.IOUtils;@@ -40,7 +40,7 @@ public class SolrCloudScraper extends SolrScraper {private final CloudSolrClient solrClient;private final SolrClientFactory solrClientFactory;- private Cache<String, HttpSolrClient> hostClientCache = Caffeine.newBuilder().build();+ private Cache<String, Http2SolrClient> hostClientCache = Caffeine.newBuilder().build();public SolrCloudScraper(CloudSolrClient solrClient,@@ -54,7 +54,7 @@ public class SolrCloudScraper extends SolrScraper {@Overridepublic Map<String, MetricSamples> pingAllCores(MetricsQuery query) throws IOException {- Map<String, HttpSolrClient> httpSolrClients = createHttpSolrClients();+ Map<String, Http2SolrClient> httpSolrClients = createHttpSolrClients();Map<String, DocCollection> collectionState = solrClient.getClusterState().getCollectionsMap();@@ -67,7 +67,7 @@ public class SolrCloudScraper extends SolrScraper {List<String> coreNames =replicas.stream().map(Replica::getCoreName).collect(Collectors.toList());- Map<String, HttpSolrClient> coreToClient =+ Map<String, Http2SolrClient> coreToClient =replicas.stream().map(replica ->@@ -85,10 +85,10 @@ public class SolrCloudScraper extends SolrScraper {});}- private Map<String, HttpSolrClient> createHttpSolrClients() throws IOException {+ private Map<String, Http2SolrClient> createHttpSolrClients() throws IOException {return getBaseUrls().stream().map(url -> hostClientCache.get(url, solrClientFactory::createStandaloneSolrClient))- .collect(Collectors.toMap(HttpSolrClient::getBaseURL, Function.identity()));+ .collect(Collectors.toMap(Http2SolrClient::getBaseURL, Function.identity()));}@Override@@ -106,7 +106,7 @@ public class SolrCloudScraper extends SolrScraper {@Overridepublic Map<String, MetricSamples> metricsForAllHosts(MetricsQuery query) throws IOException {- Map<String, HttpSolrClient> httpSolrClients = createHttpSolrClients();+ Map<String, Http2SolrClient> httpSolrClients = createHttpSolrClients();return sendRequestsInParallel(httpSolrClients.keySet(),diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.javaindex 270abf76222..a65de0de431 100644--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java@@ -37,7 +37,7 @@ import net.thisptr.jackson.jq.exception.JsonQueryException;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudSolrClient;-import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;import org.apache.solr.client.solrj.request.QueryRequest;import org.apache.solr.common.util.NamedList;import org.apache.solr.prometheus.collector.MetricSamples;@@ -124,8 +124,8 @@ public abstract class SolrScraper implements Closeable {String baseUrlLabelValue = "";String zkHostLabelValue = "";- if (client instanceof HttpSolrClient) {- baseUrlLabelValue = ((HttpSolrClient) client).getBaseURL();+ if (client instanceof Http2SolrClient) {+ baseUrlLabelValue = ((Http2SolrClient) client).getBaseURL();} else if (client instanceof CloudSolrClient) {zkHostLabelValue = ((CloudSolrClient) client).getClusterStateProvider().getQuorumHosts();}@@ -170,12 +170,10 @@ public abstract class SolrScraper implements Closeable {}/* Labels due to client */- if (client instanceof HttpSolrClient) {+ if (!baseUrlLabelValue.isEmpty()) {labelNames.add(BASE_URL_LABEL);labelValues.add(baseUrlLabelValue);- }-- if (client instanceof CloudSolrClient) {+ } else if (!zkHostLabelValue.isEmpty()) {labelNames.add(ZK_HOST_LABEL);labelValues.add(zkHostLabelValue);}diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.javaindex 90139e968f4..734279653f8 100644--- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java+++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrStandaloneScraper.java@@ -25,7 +25,7 @@ import java.util.Map;import java.util.Set;import java.util.concurrent.ExecutorService;import org.apache.solr.client.solrj.SolrServerException;-import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;import org.apache.solr.client.solrj.request.CoreAdminRequest;import org.apache.solr.common.params.CoreAdminParams;import org.apache.solr.common.util.IOUtils;@@ -35,10 +35,10 @@ import org.apache.solr.prometheus.exporter.MetricsQuery;public class SolrStandaloneScraper extends SolrScraper {- private final HttpSolrClient solrClient;+ private final Http2SolrClient solrClient;public SolrStandaloneScraper(- HttpSolrClient solrClient, ExecutorService executor, String clusterId) {+ Http2SolrClient solrClient, ExecutorService executor, String clusterId) {super(executor, clusterId);this.solrClient = solrClient;}diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.javaindex dffb6bef17a..a5454296caf 100644--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrCloudScraperTest.java@@ -28,7 +28,7 @@ import java.util.Optional;import java.util.Set;import java.util.concurrent.ExecutorService;import java.util.stream.Collectors;-import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;+import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.DocCollection;@@ -56,7 +56,7 @@ public class SolrCloudScraperTest extends PrometheusExporterTestBase {private SolrCloudScraper createSolrCloudScraper() {var solrClient =- new CloudLegacySolrClient.Builder(+ new CloudSolrClient.Builder(Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty()).build();diff --git a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.javaindex 276e32fea77..c2d052c2a29 100644--- a/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java+++ b/solr/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/SolrStandaloneScraperTest.java@@ -24,7 +24,7 @@ import java.util.List;import java.util.Map;import java.util.concurrent.ExecutorService;import org.apache.commons.io.FileUtils;-import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.IOUtils;@@ -43,7 +43,7 @@ public class SolrStandaloneScraperTest extends RestTestBase {private static MetricsConfiguration configuration;private static SolrStandaloneScraper solrScraper;private static ExecutorService executor;- private static HttpSolrClient solrClient;+ private static Http2SolrClient solrClient;@BeforeClasspublic static void setupBeforeClass() throws Exception {@@ -63,7 +63,7 @@ public class SolrStandaloneScraperTest extends RestTestBase {configuration =Helpers.loadConfiguration("conf/prometheus-solr-exporter-scraper-test-config.xml");- solrClient = getHttpSolrClient(restTestHarness.getAdminURL());+ solrClient = new Http2SolrClient.Builder(restTestHarness.getAdminURL()).build();solrScraper = new SolrStandaloneScraper(solrClient, executor, "test");NoOpResponseParser responseParser = new NoOpResponseParser();