Apache Solr open-source search software
SOLR-17436: Create a v2 equivalent for /admin/metrics (#4057)
igiguere63d244ed89fd
diff --git a/changelog/unreleased/SOLR-17436-v2-metrics-api.yml b/changelog/unreleased/SOLR-17436-v2-metrics-api.ymlnew file mode 100644index 00000000000..e6378c05648--- /dev/null+++ b/changelog/unreleased/SOLR-17436-v2-metrics-api.yml@@ -0,0 +1,8 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: Create a v2 equivalent for /admin/metrics+type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other+authors:+ - name: Isabelle Giguère+links:+ - name: SOLR-17436+ url: https://issues.apache.org/jira/browse/SOLR-17436diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.javanew file mode 100644index 00000000000..64efee83591--- /dev/null+++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/MetricsApi.java@@ -0,0 +1,84 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.api.endpoint;++import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY;++import io.swagger.v3.oas.annotations.Operation;+import io.swagger.v3.oas.annotations.Parameter;+import io.swagger.v3.oas.annotations.extensions.Extension;+import io.swagger.v3.oas.annotations.extensions.ExtensionProperty;+import io.swagger.v3.oas.annotations.media.Schema;+import jakarta.ws.rs.GET;+import jakarta.ws.rs.HeaderParam;+import jakarta.ws.rs.Path;+import jakarta.ws.rs.QueryParam;+import jakarta.ws.rs.core.StreamingOutput;++/** V2 API definitions to fetch metrics. */+@Path("/metrics")+public interface MetricsApi {++ @GET+ @Operation(+ summary = "Retrieve metrics gathered by Solr.",+ tags = {"metrics"},+ extensions = {+ @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})+ })+ StreamingOutput getMetrics(+ @HeaderParam("Accept") String acceptHeader,+ @Parameter(+ schema =+ @Schema(+ name = "node",+ description = "Name of the node to which proxy the request.",+ defaultValue = "all"))+ @QueryParam(value = "node")+ String node,+ @Parameter(schema = @Schema(name = "name", description = "The metric name to filter on."))+ @QueryParam(value = "name")+ String name,+ @Parameter(+ schema = @Schema(name = "category", description = "The category label to filter on."))+ @QueryParam(value = "category")+ String category,+ @Parameter(+ schema =+ @Schema(+ name = "core",+ description =+ "TThe core name to filter on. More than one core can be specified in a comma-separated list."))+ @QueryParam(value = "core")+ String core,+ @Parameter(+ schema =+ @Schema(name = "collection", description = "The collection name to filter on. "))+ @QueryParam(value = "collection")+ String collection,+ @Parameter(schema = @Schema(name = "shard", description = "The shard name to filter on."))+ @QueryParam(value = "shard")+ String shard,+ @Parameter(+ schema =+ @Schema(+ name = "replica_type",+ description = "The replica type to filter on.",+ allowableValues = {"NRT", "TLOG", "PULL"}))+ @QueryParam(value = "replica_type")+ String replicaType);+}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.javaindex 5f253c4ec4e..a91db17d9bb 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlersProxy.java@@ -33,15 +33,18 @@ import java.util.concurrent.TimeoutException;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.GenericV2SolrRequest;import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.cloud.ZkController;import org.apache.solr.common.SolrException;+import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CoreContainer;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.util.stats.MetricUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -55,17 +58,34 @@ public class AdminHandlersProxy {private static final String PARAM_NODE = "node";private static final long PROMETHEUS_FETCH_TIMEOUT_SECONDS = 10;- /** Proxy this request to a different remote node if 'node' or 'nodes' parameter is provided */+ /**+ * Proxy this request to a different remote node's V1 API if 'node' or 'nodes' parameter is+ * provided. For V2, use {@link AdminHandlersProxy#maybeProxyToNodes(String, SolrQueryRequest,+ * SolrQueryResponse, CoreContainer)}+ */public static boolean maybeProxyToNodes(SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer container)throws IOException, SolrServerException, InterruptedException {+ return maybeProxyToNodes("V1", req, rsp, container);+ }++ /**+ * Proxy this request to a different remote node's selected API version if 'node' or 'nodes'+ * parameter is provided+ */+ public static boolean maybeProxyToNodes(+ String apiVersion, SolrQueryRequest req, SolrQueryResponse rsp, CoreContainer container)+ throws IOException, SolrServerException, InterruptedException {String pathStr = req.getPath();ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());// Check if response format is Prometheus/OpenMetrics- String wt = params.get("wt");- boolean isPrometheusFormat = "prometheus".equals(wt) || "openmetrics".equals(wt);+ String wt = params.get(CommonParams.WT);+ boolean isPrometheusFormat =+ MetricUtils.PROMETHEUS_METRICS_WT.equals(wt)+ || MetricUtils.OPEN_METRICS_WT.equals(wt)+ || (wt == null && pathStr.endsWith("/metrics"));if (isPrometheusFormat) {// Prometheus format: use singular 'node' parameter for single-node proxy@@ -75,7 +95,7 @@ public class AdminHandlersProxy {}params.remove(PARAM_NODE);- handlePrometheusSingleNode(nodeName, pathStr, params, container, rsp);+ handlePrometheusSingleNode(apiVersion, nodeName, pathStr, params, container, rsp);} else {// Other formats (JSON/XML): use plural 'nodes' parameter for multi-node aggregationString nodeNames = req.getParams().get(PARAM_NODES);@@ -85,7 +105,7 @@ public class AdminHandlersProxy {params.remove(PARAM_NODES);Set<String> nodes = resolveNodes(nodeNames, container);- handleNamedListFormat(nodes, pathStr, params, container.getZkController(), rsp);+ handleNamedListFormat(apiVersion, nodes, pathStr, params, container.getZkController(), rsp);}return true;@@ -93,6 +113,7 @@ public class AdminHandlersProxy {/** Handle non-Prometheus formats using the existing NamedList approach. */private static void handleNamedListFormat(+ String apiVersion,Set<String> nodes,String pathStr,SolrParams params,@@ -101,7 +122,7 @@ public class AdminHandlersProxy {Map<String, Future<NamedList<Object>>> responses = new LinkedHashMap<>();for (String node : nodes) {- responses.put(node, callRemoteNode(node, pathStr, params, zkController));+ responses.put(node, callRemoteNode(apiVersion, node, pathStr, params, zkController));}for (Map.Entry<String, Future<NamedList<Object>>> entry : responses.entrySet()) {@@ -125,8 +146,12 @@ public class AdminHandlersProxy {}/** Makes a remote request asynchronously. */- public static CompletableFuture<NamedList<Object>> callRemoteNode(- String nodeName, String uriPath, SolrParams params, ZkController zkController) {+ private static CompletableFuture<NamedList<Object>> callRemoteNode(+ String apiVersion,+ String nodeName,+ String uriPath,+ SolrParams params,+ ZkController zkController) {// Validate that the node exists in the clusterif (!zkController.zkStateReader.getClusterState().getLiveNodes().contains(nodeName)) {@@ -137,13 +162,17 @@ public class AdminHandlersProxy {log.debug("Proxying {} request to node {}", uriPath, nodeName);URI baseUri = URI.create(zkController.zkStateReader.getBaseUrlForNodeName(nodeName));- SolrRequest<?> proxyReq = new GenericSolrRequest(SolrRequest.METHOD.GET, uriPath, params);++ SolrRequest<?> proxyReq = createRequest(apiVersion, uriPath, params);// Set response parser based on wt parameter to ensure correct format is used- String wt = params.get("wt");- if ("prometheus".equals(wt) || "openmetrics".equals(wt)) {+ String wt = params.get(CommonParams.WT);+ if (MetricUtils.PROMETHEUS_METRICS_WT.equals(wt) || MetricUtils.OPEN_METRICS_WT.equals(wt)) {proxyReq.setResponseParser(new InputStreamResponseParser(wt));}+ if (wt == null && uriPath.endsWith("/metrics")) {+ proxyReq.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT));+ }try {return zkController@@ -195,6 +224,7 @@ public class AdminHandlersProxy {* @param rsp the response to populate*/private static void handlePrometheusSingleNode(+ String apiVersion,String nodeName,String pathStr,ModifiableSolrParams params,@@ -205,7 +235,7 @@ public class AdminHandlersProxy {// Keep wt=prometheus for the remote request so MetricsHandler accepts it// The InputStreamResponseParser will return the Prometheus text in a "stream" keyFuture<NamedList<Object>> response =- callRemoteNode(nodeName, pathStr, params, container.getZkController());+ callRemoteNode(apiVersion, nodeName, pathStr, params, container.getZkController());try {try {@@ -220,4 +250,12 @@ public class AdminHandlersProxy {throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, t);}}++ private static SolrRequest<?> createRequest(+ String apiVersion, String uriPath, SolrParams params) {+ if (apiVersion.equalsIgnoreCase("V1")) {+ return new GenericSolrRequest(SolrRequest.METHOD.GET, uriPath, params);+ }+ return new GenericV2SolrRequest(SolrRequest.METHOD.GET, uriPath, params);+ }}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex a0ab70e07fd..eaa5510bb02 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -17,28 +17,21 @@package org.apache.solr.handler.admin;-import io.prometheus.metrics.model.snapshots.CounterSnapshot;-import io.prometheus.metrics.model.snapshots.GaugeSnapshot;-import io.prometheus.metrics.model.snapshots.HistogramSnapshot;-import io.prometheus.metrics.model.snapshots.InfoSnapshot;import io.prometheus.metrics.model.snapshots.MetricSnapshot;import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.util.ArrayList;-import java.util.HashMap;+import java.util.Collection;import java.util.List;-import java.util.Map;import java.util.Set;import java.util.SortedMap;-import java.util.TreeMap;import java.util.function.BiConsumer;-import java.util.regex.Pattern;+import org.apache.solr.api.JerseyResource;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.SolrParams;-import org.apache.solr.common.util.CommonTestInjection;-import org.apache.solr.common.util.StrUtils;import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.RequestHandlerBase;+import org.apache.solr.handler.admin.api.GetMetrics;import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;import org.apache.solr.request.SolrQueryRequest;@@ -46,6 +39,7 @@ import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.AuthorizationContext;import org.apache.solr.security.PermissionNameProvider;+import org.apache.solr.util.stats.MetricUtils;/** Request handler to return metrics */public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {@@ -61,25 +55,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamepublic static final String EXPR_PARAM = "expr";public static final String TYPE_PARAM = "type";- // Prometheus filtering parameters- public static final String CATEGORY_PARAM = "category";- public static final String CORE_PARAM = "core";- public static final String COLLECTION_PARAM = "collection";- public static final String SHARD_PARAM = "shard";- public static final String REPLICA_TYPE_PARAM = "replica_type";- public static final String METRIC_NAME_PARAM = "name";- private static final Set<String> labelFilterKeys =- Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_TYPE_PARAM);-- public static final String PROMETHEUS_METRICS_WT = "prometheus";- public static final String OPEN_METRICS_WT = "openmetrics";-public static final String ALL = "all";- private static final Pattern KEY_SPLIT_REGEX =- Pattern.compile("(?<!" + Pattern.quote("\\") + ")" + Pattern.quote(":"));private final CoreContainer cc;- private final Map<String, String> injectedSysProps = CommonTestInjection.injectAdditionalProps();private final boolean enabled;public MetricsHandler(CoreContainer coreContainer) {@@ -115,7 +93,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNameif (format == null) {req.setParams(SolrParams.wrapDefaults(params, SolrParams.of("wt", "prometheus")));- } else if (!PROMETHEUS_METRICS_WT.equals(format) && !OPEN_METRICS_WT.equals(format)) {+ } else if (!MetricUtils.PROMETHEUS_METRICS_WT.equals(format)+ && !MetricUtils.OPEN_METRICS_WT.equals(format)) {throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Only Prometheus and OpenMetrics metric formats supported. Unsupported format requested: "@@ -139,13 +118,13 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn;}- Set<String> metricNames = readParamsAsSet(params, METRIC_NAME_PARAM);- SortedMap<String, Set<String>> labelFilters = labelFilters(params);+ Set<String> metricNames = MetricUtils.readParamsAsSet(params, MetricUtils.METRIC_NAME_PARAM);+ SortedMap<String, Set<String>> labelFilters = MetricUtils.labelFilters(params);if (metricNames.isEmpty() && labelFilters.isEmpty()) {consumer.accept("metrics",- mergeSnapshots(+ MetricUtils.mergeSnapshots(metricManager.getPrometheusMetricReaders().values().stream().flatMap(r -> r.collect().stream()).toList()));@@ -160,119 +139,10 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName}// Merge all filtered snapshots and return the merged result- MetricSnapshots mergedSnapshots = mergeSnapshots(allSnapshots);+ MetricSnapshots mergedSnapshots = MetricUtils.mergeSnapshots(allSnapshots);consumer.accept("metrics", mergedSnapshots);}- private SortedMap<String, Set<String>> labelFilters(SolrParams params) {- SortedMap<String, Set<String>> labelFilters = new TreeMap<>();- labelFilterKeys.forEach(- (paramName) -> {- Set<String> filterValues = readParamsAsSet(params, paramName);- if (!filterValues.isEmpty()) {- labelFilters.put(paramName, filterValues);- }- });-- return labelFilters;- }-- private Set<String> readParamsAsSet(SolrParams params, String paramName) {- String[] paramValues = params.getParams(paramName);- if (paramValues == null || paramValues.length == 0) {- return Set.of();- }-- List<String> paramSet = new ArrayList<>();- for (String param : paramValues) {- paramSet.addAll(StrUtils.splitSmart(param, ','));- }- return Set.copyOf(paramSet);- }-- /**- * Merge a collection of individual {@link MetricSnapshot} instances into one {@link- * MetricSnapshots}. This is necessary because we create a {@link- * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric- * names across cores which is an illegal format if under the same prometheus grouping.- */- private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {- Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();- Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();- Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();- InfoSnapshot otelInfoSnapshots = null;-- for (MetricSnapshot snapshot : snapshots) {- String metricName = snapshot.getMetadata().getPrometheusName();-- switch (snapshot) {- case CounterSnapshot counterSnapshot -> {- CounterSnapshot.Builder builder =- counterSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- CounterSnapshot.builder()- .name(counterSnapshot.getMetadata().getName())- .help(counterSnapshot.getMetadata().getHelp());- return counterSnapshot.getMetadata().hasUnit()- ? base.unit(counterSnapshot.getMetadata().getUnit())- : base;- });- counterSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case GaugeSnapshot gaugeSnapshot -> {- GaugeSnapshot.Builder builder =- gaugeSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- GaugeSnapshot.builder()- .name(gaugeSnapshot.getMetadata().getName())- .help(gaugeSnapshot.getMetadata().getHelp());- return gaugeSnapshot.getMetadata().hasUnit()- ? base.unit(gaugeSnapshot.getMetadata().getUnit())- : base;- });- gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case HistogramSnapshot histogramSnapshot -> {- HistogramSnapshot.Builder builder =- histogramSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- HistogramSnapshot.builder()- .name(histogramSnapshot.getMetadata().getName())- .help(histogramSnapshot.getMetadata().getHelp());- return histogramSnapshot.getMetadata().hasUnit()- ? base.unit(histogramSnapshot.getMetadata().getUnit())- : base;- });- histogramSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case InfoSnapshot infoSnapshot -> {- // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate- // metric called target_info containing OTEL SDK metadata. Only one of these need to be- // kept- if (otelInfoSnapshots == null)- otelInfoSnapshots =- new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());- }- default -> {- // Handle unexpected snapshot types gracefully- }- }- }-- MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();- counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);- return snapshotsBuilder.build();- }-@Overridepublic String getDescription() {return "A handler to return all the metrics gathered by Solr";@@ -282,4 +152,14 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamepublic Category getCategory() {return Category.ADMIN;}++ @Override+ public Collection<Class<? extends JerseyResource>> getJerseyResources() {+ return List.of(GetMetrics.class);+ }++ @Override+ public Boolean registerV2() {+ return Boolean.TRUE;+ }}diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java b/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.javanew file mode 100644index 00000000000..ad45a8ad14b--- /dev/null+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/GetMetrics.java@@ -0,0 +1,181 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.admin.api;++import io.prometheus.metrics.model.snapshots.MetricSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshots;+import jakarta.inject.Inject;+import jakarta.ws.rs.WebApplicationException;+import jakarta.ws.rs.core.StreamingOutput;+import java.io.IOException;+import java.io.OutputStream;+import java.lang.invoke.MethodHandles;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.SortedMap;+import org.apache.solr.client.api.endpoint.MetricsApi;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.core.CoreContainer;+import org.apache.solr.handler.admin.AdminHandlersProxy;+import org.apache.solr.jersey.PermissionName;+import org.apache.solr.metrics.SolrMetricManager;+import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.PrometheusResponseWriter;+import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.security.PermissionNameProvider;+import org.apache.solr.util.stats.MetricUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * V2 API implementation to fetch metrics gathered by Solr.+ *+ * <p>This API is analogous to the v1 /admin/metrics endpoint.+ */+public class GetMetrics extends AdminAPIBase implements MetricsApi {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private final SolrMetricManager metricManager;+ private final boolean enabled;++ @Inject+ public GetMetrics(+ CoreContainer coreContainer,+ SolrQueryRequest solrQueryRequest,+ SolrQueryResponse solrQueryResponse) {+ super(coreContainer, solrQueryRequest, solrQueryResponse);+ this.metricManager = coreContainer.getMetricManager();+ this.enabled = coreContainer.getConfig().getMetricsConfig().isEnabled();+ }++ @Override+ @PermissionName(PermissionNameProvider.Name.METRICS_READ_PERM)+ public StreamingOutput getMetrics(+ String acceptHeader,+ String node,+ String name,+ String category,+ String core,+ String collection,+ String shard,+ String replicaType) {++ // Convert request params into SolrParams, to reuse existing code.+ ModifiableSolrParams params =+ new ModifiableSolrParams(+ Map.of(+ MetricUtils.NODE_PARAM, new String[] {node},+ MetricUtils.METRIC_NAME_PARAM, new String[] {name},+ MetricUtils.CATEGORY_PARAM, new String[] {category},+ MetricUtils.CORE_PARAM, new String[] {core},+ MetricUtils.COLLECTION_PARAM, new String[] {collection},+ MetricUtils.SHARD_PARAM, new String[] {shard},+ MetricUtils.REPLICA_TYPE_PARAM, new String[] {replicaType}));++ solrQueryRequest.setParams(params);++ validateRequest(acceptHeader);++ if (proxyToNodes()) {+ return null;+ }++ // Using the same logic, same methods, as in MetricsHandler.handleRequest+ Set<String> metricNames = MetricUtils.readParamsAsSet(params, MetricUtils.METRIC_NAME_PARAM);+ SortedMap<String, Set<String>> labelFilters = MetricUtils.labelFilters(params);++ return doGetMetrics(metricNames, labelFilters);+ }++ private void validateRequest(String acceptHeader) {+ if (!enabled) {+ throw new SolrException(+ SolrException.ErrorCode.INVALID_STATE, "Metrics collection is disabled");+ }++ if (metricManager == null) {+ throw new SolrException(+ SolrException.ErrorCode.INVALID_STATE, "SolrMetricManager instance not initialized");+ }++ // Should handle 'Accept' header only, but a lot of code still expects 'wt'.+ if (acceptHeader == null) {+ solrQueryRequest.setParams(+ SolrParams.wrapDefaults(+ solrQueryRequest.getParams(),+ SolrParams.of(CommonParams.WT, MetricUtils.PROMETHEUS_METRICS_WT)));+ } else if (!PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS.equals(acceptHeader)+ && !PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS.equals(acceptHeader)) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST,+ "Only Prometheus and OpenMetrics metric formats supported. Unsupported format requested: "+ + acceptHeader);+ }+ }++ private boolean proxyToNodes() {+ try {+ if (coreContainer != null+ && AdminHandlersProxy.maybeProxyToNodes(+ "V2", solrQueryRequest, solrQueryResponse, coreContainer)) {+ return true; // Request was proxied to other node+ }+ } catch (Exception e) {+ log.warn("Exception proxying to other node", e);+ }+ return false;+ }++ private StreamingOutput doGetMetrics(+ Set<String> metricNames, SortedMap<String, Set<String>> labelFilters) {++ List<MetricSnapshot> snapshots = new ArrayList<>();++ if ((metricNames == null || metricNames.isEmpty()) && labelFilters.isEmpty()) {+ snapshots.addAll(+ metricManager.getPrometheusMetricReaders().values().stream()+ .flatMap(r -> r.collect().stream())+ .toList());+ } else {+ for (FilterablePrometheusMetricReader reader :+ metricManager.getPrometheusMetricReaders().values()) {+ MetricSnapshots filteredSnapshots = reader.collect(metricNames, labelFilters);+ filteredSnapshots.forEach(snapshots::add);+ }+ }++ return writeMetricSnapshots(MetricUtils.mergeSnapshots(snapshots));+ }++ private StreamingOutput writeMetricSnapshots(MetricSnapshots snapshots) {+ return new StreamingOutput() {+ @Override+ public void write(OutputStream output) throws IOException, WebApplicationException {+ PrometheusResponseWriter writer = new PrometheusResponseWriter();+ writer.writeMetricSnapshots(output, solrQueryRequest, snapshots);+ output.flush();+ }+ };+ }+}diff --git a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.javaindex 9a69d4e2364..737a63cef1a 100644--- a/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java+++ b/solr/core/src/java/org/apache/solr/handler/api/V2ApiUtils.java@@ -32,6 +32,7 @@ import org.apache.solr.common.util.EnvUtils;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.StrUtils;import org.apache.solr.common.util.Utils;+import org.apache.solr.response.PrometheusResponseWriter;import org.apache.solr.response.RawResponseWriter;import org.apache.solr.response.SolrQueryResponse;@@ -105,6 +106,10 @@ public class V2ApiUtils {return JAVABIN_CONTENT_TYPE_V2;case FILE_STREAM:return RawResponseWriter.CONTENT_TYPE;+ case "prometheus":+ return PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS;+ case "openmetrics":+ return PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS;default:return defaultMediaType;}diff --git a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.javaindex 3ac1ef79600..6eed07d2af8 100644--- a/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java+++ b/solr/core/src/java/org/apache/solr/jersey/JerseyApplications.java@@ -48,6 +48,8 @@ public class JerseyApplications {register(MessageBodyWriters.XmlMessageBodyWriter.class, 5);register(MessageBodyWriters.CsvMessageBodyWriter.class, 5);register(MessageBodyWriters.RawMessageBodyWriter.class, 5);+ register(MessageBodyWriters.PrometheusMessageBodyWriter.class, 5);+ register(MessageBodyWriters.OpenmetricsMessageBodyWriter.class, 5);register(MessageBodyReaders.CachingJsonMessageBodyReader.class, 2);register(SolrJacksonMapper.class);diff --git a/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java b/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.javaindex 4af087ead58..cf4d5a434ab 100644--- a/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java+++ b/solr/core/src/java/org/apache/solr/jersey/MessageBodyWriters.java@@ -38,6 +38,7 @@ import org.apache.solr.handler.api.V2ApiUtils;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.CSVResponseWriter;import org.apache.solr.response.JavaBinResponseWriter;+import org.apache.solr.response.PrometheusResponseWriter;import org.apache.solr.response.QueryResponseWriter;import org.apache.solr.response.RawResponseWriter;import org.apache.solr.response.SolrQueryResponse;@@ -107,6 +108,35 @@ public class MessageBodyWriters {}}+ @Produces(PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS)+ public static class PrometheusMessageBodyWriter extends BaseMessageBodyWriter+ implements MessageBodyWriter<Object> {+ @Override+ public QueryResponseWriter createResponseWriter() {+ return new PrometheusResponseWriter();+ }++ @Override+ public String getSupportedMediaType() {+ return PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS;+ }+ }++ @Produces(PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS)+ public static class OpenmetricsMessageBodyWriter extends BaseMessageBodyWriter+ implements MessageBodyWriter<Object> {+ @Override+ public QueryResponseWriter createResponseWriter() {+ // same writer handles both Prometheus and OpenMetrics+ return new PrometheusResponseWriter();+ }++ @Override+ public String getSupportedMediaType() {+ return PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS;+ }+ }+public abstract static class BaseMessageBodyWriter implements MessageBodyWriter<Object> {@Context protected ResourceContext resourceContext;diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.javaindex 610fbaa8df2..d56a85b9bbe 100644--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java@@ -16,8 +16,6 @@*/package org.apache.solr.response;-import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;-import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;import io.prometheus.metrics.model.snapshots.MetricSnapshots;@@ -27,10 +25,14 @@ import java.io.OutputStream;import java.nio.charset.StandardCharsets;import org.apache.solr.common.params.CommonParams;import org.apache.solr.handler.admin.MetricsHandler;+import org.apache.solr.handler.admin.api.GetMetrics;import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.util.stats.MetricUtils;-/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */-@SuppressWarnings(value = "unchecked")+/**+ * Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} and V2+ * API implementation {@link GetMetrics}+ */public class PrometheusResponseWriter implements QueryResponseWriter {// not TextQueryResponseWriter because Prometheus libs work with an OutputStream@@ -64,6 +66,12 @@ public class PrometheusResponseWriter implements QueryResponseWriter {throw new IOException("No metrics found in response");}MetricSnapshots snapshots = (MetricSnapshots) metrics;+ writeMetricSnapshots(out, request, snapshots);+ }++ /** Write MetricSnapshots in Prometheus or OpenMetrics format */+ public void writeMetricSnapshots(+ OutputStream out, SolrQueryRequest request, MetricSnapshots snapshots) throws IOException {if (writeOpenMetricsFormat(request)) {new OpenMetricsTextFormatWriter(false, true).write(out, snapshots);} else {@@ -78,7 +86,7 @@ public class PrometheusResponseWriter implements QueryResponseWriter {private boolean writeOpenMetricsFormat(SolrQueryRequest request) {String wt = request.getParams().get(CommonParams.WT);- if (OPEN_METRICS_WT.equals(wt)) {+ if (MetricUtils.OPEN_METRICS_WT.equals(wt)) {return true;}diff --git a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.javaindex f15feac9f67..83469dfece1 100644--- a/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/QueryResponseWriter.java@@ -45,6 +45,9 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin;public interface QueryResponseWriter extends NamedListInitializedPlugin {public static String CONTENT_TYPE_XML_UTF8 = "application/xml; charset=UTF-8";public static String CONTENT_TYPE_TEXT_UTF8 = "text/plain; charset=UTF-8";+ public static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4";+ public static final String CONTENT_TYPE_OPEN_METRICS =+ "application/openmetrics-text; version=1.0.0; charset=utf-8";/*** Writes the response to the {@link OutputStream}. {@code contentType} is from {@linkdiff --git a/solr/core/src/java/org/apache/solr/response/ResponseWritersRegistry.java b/solr/core/src/java/org/apache/solr/response/ResponseWritersRegistry.javaindex cfd04e28714..f7d342bc286 100644--- a/solr/core/src/java/org/apache/solr/response/ResponseWritersRegistry.java+++ b/solr/core/src/java/org/apache/solr/response/ResponseWritersRegistry.java@@ -16,8 +16,8 @@*/package org.apache.solr.response;-import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;-import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;+import static org.apache.solr.util.stats.MetricUtils.OPEN_METRICS_WT;+import static org.apache.solr.util.stats.MetricUtils.PROMETHEUS_METRICS_WT;import java.util.Map;import org.apache.solr.common.params.CommonParams;diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.javaindex 5c878ab6476..506799af323 100644--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java@@ -18,9 +18,25 @@ package org.apache.solr.util.stats;import com.codahale.metrics.Snapshot;import com.codahale.metrics.Timer;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;+import io.prometheus.metrics.model.snapshots.InfoSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.lang.management.OperatingSystemMXBean;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.SortedMap;+import java.util.TreeMap;import java.util.concurrent.TimeUnit;+import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.StrUtils;/** Metrics specific utility functions. */public class MetricUtils {@@ -49,6 +65,39 @@ public class MetricUtils {private static final String P999 = "p999";private static final String P999_MS = P999 + MS;+ // 'wt' values for V1 Metrics API+ public static final String PROMETHEUS_METRICS_WT = "prometheus";+ public static final String OPEN_METRICS_WT = "openmetrics";++ // Metrics API query params+ public static final String NODE_PARAM = "node";+ public static final String CATEGORY_PARAM = "category";+ public static final String CORE_PARAM = "core";+ public static final String COLLECTION_PARAM = "collection";+ public static final String SHARD_PARAM = "shard";+ public static final String REPLICA_TYPE_PARAM = "replica_type";+ public static final String METRIC_NAME_PARAM = "name";++ private static final Set<String> labelFilterKeys =+ Set.of(+ MetricUtils.CATEGORY_PARAM,+ MetricUtils.CORE_PARAM,+ MetricUtils.COLLECTION_PARAM,+ MetricUtils.SHARD_PARAM,+ MetricUtils.REPLICA_TYPE_PARAM);++ /**+ * These are well-known implementations of {@link java.lang.management.OperatingSystemMXBean}.+ * Some of them provide additional useful properties beyond those declared by the interface.+ */+ public static String[] OS_MXBEAN_CLASSES =+ new String[] {+ OperatingSystemMXBean.class.getName(),+ "com.sun.management.OperatingSystemMXBean",+ "com.sun.management.UnixOperatingSystemMXBean",+ "com.ibm.lang.management.OperatingSystemMXBean"+ };+/*** Adds metrics from a Timer to a NamedList, using well-known back-compat names.*@@ -89,14 +138,139 @@ public class MetricUtils {}/**- * These are well-known implementations of {@link java.lang.management.OperatingSystemMXBean}.- * Some of them provide additional useful properties beyond those declared by the interface.+ * Merge a collection of individual {@link MetricSnapshot} instances into one {@link+ * MetricSnapshots}. This is necessary because we create a {@link+ * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric+ * names across cores which is an illegal format if under the same prometheus grouping.*/- public static String[] OS_MXBEAN_CLASSES =- new String[] {- OperatingSystemMXBean.class.getName(),- "com.sun.management.OperatingSystemMXBean",- "com.sun.management.UnixOperatingSystemMXBean",- "com.ibm.lang.management.OperatingSystemMXBean"- };+ public static MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {+ Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();+ Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();+ Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();+ InfoSnapshot otelInfoSnapshots = null;++ for (MetricSnapshot snapshot : snapshots) {+ String metricName = snapshot.getMetadata().getPrometheusName();++ switch (snapshot) {+ case CounterSnapshot counterSnapshot -> {+ CounterSnapshot.Builder builder =+ counterSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ CounterSnapshot.builder()+ .name(counterSnapshot.getMetadata().getName())+ .help(counterSnapshot.getMetadata().getHelp());+ return counterSnapshot.getMetadata().hasUnit()+ ? base.unit(counterSnapshot.getMetadata().getUnit())+ : base;+ });+ counterSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case GaugeSnapshot gaugeSnapshot -> {+ GaugeSnapshot.Builder builder =+ gaugeSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ GaugeSnapshot.builder()+ .name(gaugeSnapshot.getMetadata().getName())+ .help(gaugeSnapshot.getMetadata().getHelp());+ return gaugeSnapshot.getMetadata().hasUnit()+ ? base.unit(gaugeSnapshot.getMetadata().getUnit())+ : base;+ });+ gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case HistogramSnapshot histogramSnapshot -> {+ HistogramSnapshot.Builder builder =+ histogramSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ HistogramSnapshot.builder()+ .name(histogramSnapshot.getMetadata().getName())+ .help(histogramSnapshot.getMetadata().getHelp());+ return histogramSnapshot.getMetadata().hasUnit()+ ? base.unit(histogramSnapshot.getMetadata().getUnit())+ : base;+ });+ histogramSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case InfoSnapshot infoSnapshot -> {+ // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate+ // metric called target_info containing OTEL SDK metadata. Only one of these need to be+ // kept+ if (otelInfoSnapshots == null)+ otelInfoSnapshots =+ new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());+ }+ default -> {+ // Handle unexpected snapshot types gracefully+ }+ }+ }++ MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();+ counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);+ return snapshotsBuilder.build();+ }++ /** Gather label filters */+ public static SortedMap<String, Set<String>> labelFilters(SolrParams params) {+ SortedMap<String, Set<String>> labelFilters = new TreeMap<>();+ labelFilterKeys.forEach(+ (paramName) -> {+ Set<String> filterValues = readParamsAsSet(params, paramName);+ if (!filterValues.isEmpty()) {+ labelFilters.put(paramName, filterValues);+ }+ });++ return labelFilters;+ }++ /** Add label filters to the filters map */+ public static void addLabelFilters(String value, Map<String, Set<String>> filters) {+ labelFilterKeys.forEach(+ (paramName) -> {+ Set<String> filterValues = paramValueAsSet(value);+ if (!filterValues.isEmpty()) {+ filters.put(paramName, filterValues);+ }+ });+ }++ /** Split the coma-separated param values into a set */+ public static Set<String> paramValueAsSet(String paramValue) {+ String[] values = paramValue.split(",");+ List<String> valuesSet = new ArrayList<>();+ for (String value : values) {+ valuesSet.add(value);+ }+ return Set.copyOf(valuesSet);+ }++ /**+ * Read Solr parameters as a Set.+ *+ * <p>Could probably be moved to a more generic utility class, but only used in MetricsHandler and+ * GetMetrics resource.+ */+ public static Set<String> readParamsAsSet(SolrParams params, String paramName) {+ String[] paramValues = params.getParams(paramName);+ if (paramValues == null || paramValues.length == 0) {+ return Set.of();+ }++ Set<String> paramSet = new HashSet<>();+ for (String param : paramValues) {+ if (param != null && param.length() > 0) paramSet.addAll(StrUtils.splitSmart(param, ','));+ }+ return paramSet;+ }}diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.javaindex 920b1aa2f3b..758588d7465 100644--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java@@ -59,7 +59,6 @@ import org.apache.solr.client.solrj.response.FacetField;import org.apache.solr.client.solrj.response.Group;import org.apache.solr.client.solrj.response.GroupCommand;import org.apache.solr.client.solrj.response.GroupResponse;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.client.solrj.response.UpdateResponse;import org.apache.solr.cloud.api.collections.CollectionHandlingUtils;@@ -1283,7 +1282,6 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {.withSocketTimeout(60000, TimeUnit.MILLISECONDS).build()) {var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = client.request(req);try (InputStream in = (InputStream) resp.get("stream")) {diff --git a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.javaindex 23b6721de33..a85bc5ab9cf 100644--- a/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java+++ b/solr/core/src/test/org/apache/solr/cloud/TestBaseStatsCacheCloud.java@@ -27,7 +27,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.MetricsRequest;import org.apache.solr.client.solrj.request.UpdateRequest;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.client.solrj.response.QueryResponse;import org.apache.solr.common.SolrDocument;import org.apache.solr.common.SolrInputDocument;@@ -132,7 +131,6 @@ public abstract class TestBaseStatsCacheCloud extends SolrCloudTestCase {for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {try (SolrClient client = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) {var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = client.request(req);try (InputStream in = (InputStream) resp.get("stream")) {diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.javaindex 7cf04d54375..7ade78cb6c6 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java@@ -29,6 +29,7 @@ import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.AuthorizationContext;+import org.apache.solr.util.stats.MetricUtils;import org.junit.BeforeClass;import org.junit.Test;@@ -53,8 +54,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.METRIC_NAME_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.METRIC_NAME_PARAM,expectedRequestsMetricName),resp);var metrics = resp.getValues().get("metrics");@@ -80,8 +81,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.METRIC_NAME_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.METRIC_NAME_PARAM,expectedRequestsMetricName + "," + expectedSearcherMetricName),resp);@@ -105,8 +106,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.METRIC_NAME_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.METRIC_NAME_PARAM,nonexistentMetricName),resp);var metrics = (MetricSnapshots) resp.getValues().get("metrics");@@ -126,8 +127,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.CATEGORY_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.CATEGORY_PARAM,"QUERY"),resp);var metrics = (MetricSnapshots) resp.getValues().get("metrics");@@ -137,7 +138,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {ms.getDataPoints().forEach((dp) -> {- assertEquals("QUERY", dp.getLabels().get(MetricsHandler.CATEGORY_PARAM));+ assertEquals("QUERY", dp.getLabels().get(MetricUtils.CATEGORY_PARAM));});});@@ -156,8 +157,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.CATEGORY_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.CATEGORY_PARAM,"QUERY" + "," + "SEARCHER"),resp);@@ -168,10 +169,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {.forEach((dp) -> {assertTrue(- dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("QUERY")- || dp.getLabels()- .get(MetricsHandler.CATEGORY_PARAM)- .equals("SEARCHER"));+ dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("QUERY")+ || dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("SEARCHER"));});});@@ -188,8 +187,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.CORE_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.CORE_PARAM,"nonexistent_core_name"),resp);@@ -210,10 +209,10 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.CORE_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.CORE_PARAM,"collection1",- MetricsHandler.CATEGORY_PARAM,+ MetricUtils.CATEGORY_PARAM,"SEARCHER"),resp);@@ -224,8 +223,8 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {.forEach((dp) -> {assertTrue(- dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("SEARCHER")- && dp.getLabels().get(MetricsHandler.CORE_PARAM).equals("collection1"));+ dp.getLabels().get(MetricUtils.CATEGORY_PARAM).equals("SEARCHER")+ && dp.getLabels().get(MetricUtils.CORE_PARAM).equals("collection1"));});});@@ -245,10 +244,10 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {CommonParams.QT,CommonParams.METRICS_PATH,CommonParams.WT,- MetricsHandler.PROMETHEUS_METRICS_WT,- MetricsHandler.CATEGORY_PARAM,+ MetricUtils.PROMETHEUS_METRICS_WT,+ MetricUtils.CATEGORY_PARAM,"CORE",- MetricsHandler.METRIC_NAME_PARAM,+ MetricUtils.METRIC_NAME_PARAM,expectedMetricName),resp);@@ -256,7 +255,7 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {assertEquals(1, metrics.size());var actualDatapoint = metrics.get(0).getDataPoints().getFirst();assertEquals(expectedMetricName, metrics.get(0).getMetadata().getPrometheusName());- assertEquals("CORE", actualDatapoint.getLabels().get(MetricsHandler.CATEGORY_PARAM));+ assertEquals("CORE", actualDatapoint.getLabels().get(MetricUtils.CATEGORY_PARAM));handler.close();}diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.javanew file mode 100644index 00000000000..34e9551c919--- /dev/null+++ b/solr/core/src/test/org/apache/solr/handler/admin/api/GetMetricsTest.java@@ -0,0 +1,276 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.admin.api;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.net.URL;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.StandardCopyOption;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.function.Consumer;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.response.PrometheusResponseWriter;+import org.apache.solr.util.stats.MetricUtils;+import org.eclipse.jetty.client.ContentResponse;+import org.eclipse.jetty.client.HttpClient;+import org.eclipse.jetty.http.HttpFields.Mutable;+import org.eclipse.jetty.http.HttpHeader;+import org.eclipse.jetty.http.HttpMethod;+import org.junit.After;+import org.junit.AfterClass;+import org.junit.Assert;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++/**+ * Unit tests for {@link GetMetrics}.+ *+ * <p>See also: TestMetricsRequest, in SolrJ+ */+@SuppressSSL+public class GetMetricsTest extends SolrTestCaseJ4 {++ // No need for the full output+ private static final int MAX_OUTPUT = 1024;++ private static final int TIMEOUT = 15000;++ private static HttpClient jettyHttpClient;+ private static String metricsV2Url;+ private static String baseV2Url;+ private static MiniSolrCloudCluster cluster;++ @BeforeClass+ public static void beforeClass() throws Exception {+ Path tempDir = createTempDir();+ copyMinConf(tempDir);+ Files.copy(+ SolrTestCaseJ4.TEST_PATH().resolve("solr.xml"),+ tempDir.resolve("solr.xml"),+ StandardCopyOption.REPLACE_EXISTING);++ MiniSolrCloudCluster.Builder clusterBuilder = new MiniSolrCloudCluster.Builder(2, tempDir);+ cluster = clusterBuilder.withSolrXml(tempDir.resolve("solr.xml")).build();++ baseV2Url = cluster.getJettySolrRunner(0).getBaseURLV2().toString();++ metricsV2Url = baseV2Url.concat("/metrics");++ jettyHttpClient = new HttpClient();+ jettyHttpClient.setConnectTimeout(TIMEOUT);+ jettyHttpClient.setMaxConnectionsPerDestination(1);+ jettyHttpClient.setMaxRequestsQueuedPerDestination(1);+ }++ @AfterClass+ public static void afterClass() throws Exception {+ jettyHttpClient.destroy();+ cluster.shutdown();+ }++ @Before+ public void beforeTest() throws Exception {+ // stop and start Jetty client for each test, otherwise, it seems responses get mixed!+ jettyHttpClient.start();+ }++ @After+ public void afterTest() throws Exception {+ jettyHttpClient.stop();+ }++ @Test+ public void testGetMetricsDefault()+ throws IOException,+ InterruptedException,+ ExecutionException,+ TimeoutException,+ SolrServerException {+ ContentResponse response = null;+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .method(HttpMethod.GET)+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ Assert.assertEquals(200, response.getStatus());++ String str = readMaxOut(response.getContent());+ Assert.assertTrue(str.contains("# HELP"));+ Assert.assertTrue(str.contains("# TYPE"));+ }++ @Test+ public void testGetMetricsPrometheus()+ throws IOException,+ InterruptedException,+ TimeoutException,+ ExecutionException,+ SolrServerException {+ ContentResponse response = null;+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .method(HttpMethod.GET)+ .headers(+ new Consumer<Mutable>() {++ @Override+ public void accept(Mutable arg0) {+ arg0.add(HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS);+ }+ })+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ Assert.assertEquals(200, response.getStatus());+ Assert.assertEquals("text/plain", response.getMediaType());+ }++ @Test+ public void testGetMetricsOpenMetrics()+ throws IOException,+ InterruptedException,+ TimeoutException,+ ExecutionException,+ SolrServerException {+ ContentResponse response = null;+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .method(HttpMethod.GET)+ .headers(+ new Consumer<Mutable>() {++ @Override+ public void accept(Mutable arg0) {+ arg0.add(+ HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_OPEN_METRICS);+ }+ })+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ Assert.assertEquals(200, response.getStatus());+ Assert.assertEquals("application/openmetrics-text", response.getMediaType());+ }++ @Test+ public void testGetMetricsCategoryParams() throws IOException {+ String expected = """+ category="QUERY"+ """;++ ContentResponse response = null;+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .param(MetricUtils.CATEGORY_PARAM, "QUERY")+ .method(HttpMethod.GET)+ .headers(+ new Consumer<Mutable>() {++ @Override+ public void accept(Mutable arg0) {+ arg0.add(HttpHeader.ACCEPT, PrometheusResponseWriter.CONTENT_TYPE_PROMETHEUS);+ }+ })+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ Assert.assertEquals(200, response.getStatus());++ String str = readMaxOut(response.getContent());+ Assert.assertTrue(str.contains(expected.trim()));+ Assert.assertFalse(str.contains("category=\"CORE\""));+ Assert.assertFalse(str.contains("category=\"UPDATE\""));+ }++ @Test+ public void testGetMetricsProxyToNode() throws IOException {+ URL otherUrl = cluster.getJettySolrRunner(1).getBaseURLV2();+ String otherNode = otherUrl.getHost() + ":" + otherUrl.getPort() + "_solr";++ ContentResponse response = null;+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .param(MetricUtils.NODE_PARAM, otherNode)+ .method(HttpMethod.GET)+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ // HTTP 204: no content to test+ Assert.assertEquals(204, response.getStatus());++ String unknownNode = "unknown.host:1234_solr";+ try {+ response =+ jettyHttpClient+ .newRequest(metricsV2Url)+ .timeout(TIMEOUT, TimeUnit.MILLISECONDS)+ .param(MetricUtils.NODE_PARAM, unknownNode)+ .method(HttpMethod.GET)+ .send();+ } catch (InterruptedException | ExecutionException | TimeoutException e) {+ Assert.fail("Should not throw exception: " + e.getClass() + ". message: " + e.getMessage());+ return;+ }+ // Unknown host is ignored, returns the default response+ Assert.assertEquals(200, response.getStatus());+ }++ private static String readMaxOut(byte[] bytes) throws IOException {+ int max = bytes.length > MAX_OUTPUT ? MAX_OUTPUT : bytes.length;+ String str = "";+ try (ByteArrayOutputStream out = new ByteArrayOutputStream(max); ) {+ out.write(bytes, 0, max);+ str = out.toString(StandardCharsets.UTF_8);+ }+ return str;+ }+}diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.javaindex d9d2493eb3a..530a2ba6009 100644--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java@@ -67,8 +67,7 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {public void testPrometheusStructureOutput() throws Exception {ModifiableSolrParams params = new ModifiableSolrParams();params.set("wt", "prometheus");- var req = new MetricsRequest(params);- req.setResponseParser(new InputStreamResponseParser("prometheus"));+ var req = new MetricsRequest(params); // response parser set in MetricsRequest constructortry (SolrClient adminClient = getHttpSolrClient(solrTestRule.getBaseUrl())) {NamedList<Object> res = adminClient.request(req);diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.javaindex ffc254723e1..508bd9d04d5 100644--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java@@ -22,7 +22,6 @@ import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.MetricsRequest;import org.apache.solr.client.solrj.request.SolrQuery;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;@@ -68,7 +67,6 @@ public class TestPrometheusResponseWriterCloud extends SolrCloudTestCase {solrClient.query("collection1", query);var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = solrClient.request(req);try (InputStream in = (InputStream) resp.get("stream")) {@@ -98,7 +96,6 @@ public class TestPrometheusResponseWriterCloud extends SolrCloudTestCase {solrClient.query("collection2", query);var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = solrClient.request(req);diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.javaindex 97916c007ca..e3eb646896c 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java@@ -17,8 +17,6 @@package org.apache.solr.opentelemetry;-import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;-import io.opentelemetry.api.GlobalOpenTelemetry;import io.opentelemetry.api.trace.TracerProvider;import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter;@@ -43,6 +41,7 @@ import org.apache.solr.client.solrj.response.V2Response;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.SolrDocumentList;import org.apache.solr.common.util.NamedList;+import org.apache.solr.util.stats.MetricUtils;import org.apache.solr.util.tracing.TraceUtils;import org.junit.AfterClass;import org.junit.Before;@@ -136,7 +135,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {CloudSolrClient cloudClient = cluster.getSolrClient();MetricsRequest request = new MetricsRequest();- request.setResponseParser(new InputStreamResponseParser(PROMETHEUS_METRICS_WT));+ request.setResponseParser(new InputStreamResponseParser(MetricUtils.PROMETHEUS_METRICS_WT));NamedList<Object> rsp = cloudClient.request(request);((InputStream) rsp.get("stream")).close();var finishedSpans = getAndClearSpans();diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.javaindex 6eaf0c07e17..0c8ba50078f 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java@@ -26,7 +26,6 @@ import java.nio.charset.StandardCharsets;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.MetricsRequest;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.cloud.SolrCloudTestCase;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;@@ -79,7 +78,6 @@ public class TestMetricExemplars extends SolrCloudTestCase {var expectedTrace = getRootTraceId(spans);var req = new MetricsRequest(new ModifiableSolrParams().set("wt", "openmetrics"));- req.setResponseParser(new InputStreamResponseParser("openmetrics"));NamedList<Object> resp = cloudClient.request(req);try (InputStream in = (InputStream) resp.get("stream")) {diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adocindex 09590b44e72..7d0e79a2c0a 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/metrics-reporting.adoc@@ -132,7 +132,13 @@ Metrics collection for index merges can be configured in the `<metrics>` section== Metrics API-The `/admin/metrics` endpoint natively provides access to all metrics in Prometheus format by default. You can also specify `wt=prometheus` as a parameter for Prometheus format or `wt=openmetrics` for OpenMetrics format. More information on the data models is provided in the sections below.+The `/metrics` endpoint natively provides access to all metrics in Prometheus format by default. You can also specify `wt=prometheus` as a parameter for Prometheus format or `wt=openmetrics` for OpenMetrics format. More information on the data models is provided in the sections below.++[NOTE]+====+The V2 `/metrics` endpoint is equivalent to the V1 `/admin/metrics` endpoint.+Examples on this page show only the V2 endpoint.+======= Prometheus@@ -148,14 +154,14 @@ The `prometheus-config.yml` file needs to be configured for a Prometheus server----scrape_configs:- job_name: 'solr'- metrics_path: "/solr/admin/metrics"+ metrics_path: "/api/metrics"static_configs:- targets: ['localhost:8983', 'localhost:7574']----=== OpenMetrics-OpenMetrics format is available from the `/admin/metrics` endpoint by providing the `wt=openmetrics` parameter or by passing the Accept header `application/openmetrics-text;version=1.0.0`. OpenMetrics is an extension of the Prometheus format that adds additional metadata and exemplars.+OpenMetrics format is available from the `/metrics` endpoint by providing the `wt=openmetrics` parameter or by passing the Accept header `application/openmetrics-text;version=1.0.0`. OpenMetrics is an extension of the Prometheus format that adds additional metadata and exemplars.See https://prometheus.io/docs/specs/om/open_metrics_spec/[OpenMetrics Spec] documentation for more information.@@ -173,7 +179,7 @@ A basic `prometheus-config.yml` configuration for a Prometheus server in SolrClo----scrape_configs:- job_name: 'solr'- metrics_path: "/solr/admin/metrics"+ metrics_path: "/api/metrics"static_configs:- targets: ['localhost:8983', 'localhost:7574']params:@@ -256,26 +262,26 @@ The replica type to filter on. Valid values are NRT, TLOG, or PULL. This attribuRequest only metrics from the `foobar` collection:[source,text]-http://localhost:8983/solr/admin/metrics?collection=foobar+http://localhost:8983/api/metrics?collection=foobarRequest only the metrics with a category label of QUERY or UPDATE:[source,text]-http://localhost:8983/solr/admin/metrics?category=QUERY,UPDATE+http://localhost:8983/api/metrics?category=QUERY,UPDATERequest only `solr_core_requests_total` metrics from the `foobar_shard1_replica_n1` core:[source,text]-http://localhost:8983/solr/admin/metrics?name=solr_core_requests_total&core=foobar_shard1_replica_n1+http://localhost:8983/api/metrics?name=solr_core_requests_total&core=foobar_shard1_replica_n1Request only the core index size `solr_core_index_size_bytes` metrics from collections labeled `foo` and `bar`:[source,text]-http://localhost:8983/solr/admin/metrics?name=solr_core_index_size_bytes&collection=foo,bar+http://localhost:8983/api/metrics?name=solr_core_index_size_bytes&collection=foo,bar== OTLP-For users who do not use or support pulling metrics in Prometheus format with the `/admin/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing metrics via gRPC or HTTP.+For users who do not use or support pulling metrics in Prometheus format with the `/metrics` API, Solr also supports pushing metrics natively with https://opentelemetry.io/docs/specs/otlp/[OTLP], which is a vendor-agnostic protocol for pushing metrics via gRPC or HTTP.OTLP is widely supported by many tools, vendors, and pipelines. See the OpenTelemetry https://opentelemetry.io/ecosystem/vendors/[vendors list] for more details on available and compatible options.@@ -339,7 +345,7 @@ Endpoint to send OTLP metrics to using the HTTP protocol.=== OpenTelemetry Collector setup-The https://opentelemetry.io/docs/collector/[OpenTelemetry Collector] is a powerful process that allows users to decouple their metrics pipeline and route to their preferred backend. It natively supports metrics being pushed to it via OTLP and/or scraping the `/admin/metrics` Prometheus endpoint supported by Solr. You can push both metrics and traces to the collector via OTLP as a single pipeline.+The https://opentelemetry.io/docs/collector/[OpenTelemetry Collector] is a powerful process that allows users to decouple their metrics pipeline and route to their preferred backend. It natively supports metrics being pushed to it via OTLP and/or scraping the `/metrics` Prometheus endpoint supported by Solr. You can push both metrics and traces to the collector via OTLP as a single pipeline.A simple setup to route metrics from Solr -> OpenTelemetry Collector -> Prometheus can be configured with the following OpenTelemetry Collector configuration file:diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.javaindex ef6bb5dbd21..f036d294f60 100644--- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java@@ -35,7 +35,6 @@ import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.cloud.NodeStateProvider;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.client.solrj.request.MetricsRequest;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.client.solrj.response.JavaBinResponseParser;import org.apache.solr.client.solrj.response.SimpleSolrResponse;import org.apache.solr.common.MapWriter;@@ -214,7 +213,6 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriterparams.add("name", String.join(",", metricNames));var req = new MetricsRequest(params);- req.setResponseParser(new InputStreamResponseParser("prometheus"));String baseUrl =ctx.zkClientClusterStateProvider.getZkStateReader().getBaseUrlForNodeName(solrNode);diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.javaindex 82c10659586..c849bf16999 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/MetricsRequest.java@@ -17,15 +17,16 @@package org.apache.solr.client.solrj.request;import org.apache.solr.client.solrj.SolrRequest;-import org.apache.solr.client.solrj.SolrResponse;-import org.apache.solr.client.solrj.response.SolrResponseBase;+import org.apache.solr.client.solrj.response.InputStreamResponse;+import org.apache.solr.client.solrj.response.InputStreamResponseParser;+import org.apache.solr.common.SolrException;import org.apache.solr.common.params.CommonParams;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;-/** Request to "/admin/metrics" */-public class MetricsRequest extends SolrRequest<SolrResponse> {+/** Request to V1 "/admin/metrics" or V2 "/metrics" */+public class MetricsRequest extends SolrRequest<InputStreamResponse> {private static final long serialVersionUID = 1L;@@ -36,12 +37,37 @@ public class MetricsRequest extends SolrRequest<SolrResponse> {this(new ModifiableSolrParams());}+ /**+ * @param path the HTTP path to use for this request. Supports V1 "/admin/metrics" (default) or V2+ * "/metrics"+ */+ public MetricsRequest(String path) {+ this(path, new ModifiableSolrParams());+ }+/*** @param params the Solr parameters to use for this request.*/public MetricsRequest(SolrParams params) {- super(METHOD.GET, CommonParams.METRICS_PATH, SolrRequestType.ADMIN);+ this(CommonParams.METRICS_PATH, params);+ }++ /**+ * @param params the Solr parameters to use for this request.+ */+ public MetricsRequest(String path, SolrParams params) {+ super(METHOD.GET, path, SolrRequestType.ADMIN);+ if (!path.endsWith("/metrics")) {+ throw new SolrException(+ SolrException.ErrorCode.BAD_REQUEST, "Request path not supported: " + path);+ }this.params = params;+ // Set response parser according to "wt".+ if ("openmetrics".equals(params.get(CommonParams.WT))) {+ setResponseParser(new InputStreamResponseParser("openmetrics"));+ } else {+ setResponseParser(new InputStreamResponseParser("prometheus"));+ }}@Override@@ -50,8 +76,17 @@ public class MetricsRequest extends SolrRequest<SolrResponse> {}@Override- protected SolrResponse createResponse(NamedList<Object> namedList) {- SolrResponseBase resp = new SolrResponseBase();- return (SolrResponse) resp;+ protected InputStreamResponse createResponse(NamedList<Object> namedList) {+ return new InputStreamResponse();+ }++ @Override+ public ApiVersion getApiVersion() {+ if (CommonParams.METRICS_PATH.equals(getPath())) {+ // (/solr) /admin/metrics+ return ApiVersion.V1;+ }+ // Ref. org.apache.solr.client.api.endpoint.MetricsApi : /metrics+ return ApiVersion.V2;}}diff --git a/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xml b/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xmlnew file mode 100644index 00000000000..a48c29f7e27--- /dev/null+++ b/solr/solrj/src/test-files/solrj/solr/solr-metrics-enabled.xml@@ -0,0 +1,50 @@+<?xml version="1.0" encoding="UTF-8" ?>+<!--+ Licensed to the Apache Software Foundation (ASF) under one or more+ contributor license agreements. See the NOTICE file distributed with+ this work for additional information regarding copyright ownership.+ The ASF licenses this file to You under the Apache License, Version 2.0+ (the "License"); you may not use this file except in compliance with+ the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->+++<!--+ All (relative) paths are relative to the installation path+-->+<solr>++ <metrics enabled="${metricsEnabled:true}"/>++ <str name="shareSchema">${shareSchema:false}</str>+ <str name="configSetBaseDir">${configSetBaseDir:configsets}</str>+ <str name="coreRootDirectory">${coreRootDirectory:.}</str>+ <str name="allowUrls">${solr.tests.security.allow.urls:}</str>++ <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">+ <str name="urlScheme">${urlScheme:}</str>+ <int name="socketTimeout">${socketTimeout:90000}</int>+ <int name="connTimeout">${connTimeout:15000}</int>+ </shardHandlerFactory>++ <solrcloud>+ <str name="host">127.0.0.1</str>+ <int name="hostPort">${hostPort:8983}</int>+ <int name="zkClientTimeout">${solr.zookeeper.client.timeout:30000}</int>+ <int name="leaderVoteWait">0</int>+ <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:45000}</int>+ <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:340000}</int>+ <str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>+ <str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>+ <str name="zkCredentialsInjector">${zkCredentialsInjector:org.apache.solr.common.cloud.DefaultZkCredentialsInjector}</str>+ </solrcloud>++</solr>diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.javanew file mode 100644index 00000000000..c360ddb85d4--- /dev/null+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestMetricsRequest.java@@ -0,0 +1,147 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.solrj.request;++import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.StandardCopyOption;+import java.util.Map;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.TimeoutException;+import org.apache.commons.io.file.PathUtils;+import org.apache.solr.client.solrj.SolrClient.SolrClientFunction;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;+import org.apache.solr.client.solrj.response.InputStreamResponseParser;+import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.util.stats.MetricUtils;+import org.junit.AfterClass;+import org.junit.Assert;+import org.junit.BeforeClass;+import org.junit.Test;++/** Test {@link MetricsRequest}. */+public class TestMetricsRequest extends SolrCloudTestCase {++ private static final String METRICS_V2_PATH = "/metrics";++ private static HttpJettySolrClient httpClient;+ private static MiniSolrCloudCluster cluster;++ @BeforeClass+ public static void beforeClass() throws Exception {+ Path tempDir = createTempDir();+ Path testFilesDir = getFile("solrj/solr/collection1").getParent();+ PathUtils.copyDirectory(testFilesDir, tempDir);+ Files.copy(+ testFilesDir.resolve("solr-metrics-enabled.xml"),+ tempDir.resolve("solr.xml"),+ StandardCopyOption.REPLACE_EXISTING);+ MiniSolrCloudCluster.Builder clusterBuilder = new MiniSolrCloudCluster.Builder(2, tempDir);+ cluster = clusterBuilder.withSolrXml(tempDir.resolve("solr.xml")).build();++ HttpJettySolrClient.Builder clientBuilder =+ new HttpJettySolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString());+ httpClient = clientBuilder.build();+ }++ @AfterClass+ public static void afterClass() throws Exception {+ httpClient.close();+ cluster.shutdown();+ }++ @Test+ public void testGetMetricsV2()+ throws IOException,+ InterruptedException,+ ExecutionException,+ TimeoutException,+ SolrServerException {+ MetricsRequest solrRequest = new MetricsRequest(METRICS_V2_PATH);+ String str =+ httpClient.requestWithBaseUrl(+ cluster.getJettySolrRunner(0).getBaseURLV2().toString(),+ new SolrClientFunction<HttpJettySolrClient, String>() {++ @Override+ public String apply(HttpJettySolrClient c) throws IOException, SolrServerException {+ return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest));+ }+ });++ Assert.assertTrue(str.contains("# HELP"));+ Assert.assertTrue(str.contains("# TYPE"));+ }++ @Test+ public void testGetMetricsV2ParamName()+ throws IOException,+ InterruptedException,+ ExecutionException,+ TimeoutException,+ SolrServerException {+ String requestedName = "solr_disk_space_megabytes";+ String notRequested = "solr_client_request_duration_milliseconds_bucket";+ SolrParams params =+ new ModifiableSolrParams(+ Map.of(MetricUtils.METRIC_NAME_PARAM, new String[] {requestedName}));+ MetricsRequest solrRequest = new MetricsRequest(METRICS_V2_PATH, params);+ String str =+ httpClient.requestWithBaseUrl(+ cluster.getJettySolrRunner(0).getBaseURLV2().toString(),+ new SolrClientFunction<HttpJettySolrClient, String>() {++ @Override+ public String apply(HttpJettySolrClient c) throws IOException, SolrServerException {+ return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest));+ }+ });++ Assert.assertTrue(str.contains("# HELP"));+ Assert.assertTrue(str.contains("# TYPE"));+ Assert.assertTrue(str.contains(requestedName));+ Assert.assertFalse(str.contains(notRequested));+ }++ @Test+ public void testGetMetricsV1()+ throws IOException,+ InterruptedException,+ ExecutionException,+ TimeoutException,+ SolrServerException {+ MetricsRequest solrRequest = new MetricsRequest();+ String str =+ httpClient.requestWithBaseUrl(+ cluster.getJettySolrRunner(0).getBaseUrl().toString(),+ new SolrClientFunction<HttpJettySolrClient, String>() {++ @Override+ public String apply(HttpJettySolrClient c) throws IOException, SolrServerException {+ return InputStreamResponseParser.consumeResponseToString(c.request(solrRequest));+ }+ });++ Assert.assertTrue(str.contains("# HELP"));+ Assert.assertTrue(str.contains("# TYPE"));+ }+}diff --git a/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java b/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.javaindex 1ec456202c6..7669c3755cf 100644--- a/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java+++ b/solr/test-framework/src/java/org/apache/solr/util/SolrJMetricTestUtils.java@@ -24,7 +24,6 @@ import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.jetty.HttpJettySolrClient;import org.apache.solr.client.solrj.request.MetricsRequest;-import org.apache.solr.client.solrj.response.InputStreamResponseParser;import org.apache.solr.common.params.SolrParams;import org.apache.solr.common.util.NamedList;@@ -33,7 +32,6 @@ public final class SolrJMetricTestUtils {public static double getPrometheusMetricValue(SolrClient solrClient, String metricName)throws SolrServerException, IOException {var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = solrClient.request(req);try (InputStream in = (InputStream) resp.get("stream")) {@@ -52,7 +50,6 @@ public final class SolrJMetricTestUtils {try (var client = new HttpJettySolrClient.Builder(baseUrl).build()) {var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = client.request(req);try (InputStream in = (InputStream) resp.get("stream")) {@@ -80,7 +77,6 @@ public final class SolrJMetricTestUtils {try (var client = new HttpJettySolrClient.Builder(baseUrl).build()) {var req = new MetricsRequest(SolrParams.of("wt", "prometheus"));- req.setResponseParser(new InputStreamResponseParser("prometheus"));NamedList<Object> resp = client.request(req);try (InputStream in = (InputStream) resp.get("stream")) {