Apache Solr open-source search software
SOLR-17854: Support filters to /admin/metrics endpoint for Prometheus metrics (#3499)
Matthew Biscocho046b206de344
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex ab4db7d04f2..a8cb4800fde 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -1422,14 +1422,13 @@ public class SolrCore implements SolrInfoBean, Closeable {}),OtelUnit.BYTES));- observables.add(- parentContext.observableLongGauge(- "solr_core_segment_count",- "Number of segments in a Solr core",- (observableLongMeasurement -> {- if (isReady())- observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes);- })));+ parentContext.observableLongGauge(+ "solr_core_segments",+ "Number of segments in a Solr core",+ (observableLongMeasurement -> {+ if (isReady())+ observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes);+ }));// NOCOMMIT: Do we need these start_time metrics? I think at minimum it should be optional// otherwise we fall into metric bloat for something people may not care about.diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex 633ccc7bb00..143cf453f41 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -25,14 +25,23 @@ import com.codahale.metrics.Metric;import com.codahale.metrics.MetricFilter;import com.codahale.metrics.MetricRegistry;import com.codahale.metrics.Timer;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;+import io.prometheus.metrics.model.snapshots.InfoSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.util.ArrayList;import java.util.Collections;import java.util.EnumSet;+import java.util.HashMap;import java.util.HashSet;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.SortedMap;+import java.util.TreeMap;import java.util.TreeSet;import java.util.function.BiConsumer;import java.util.function.Predicate;@@ -49,6 +58,7 @@ import org.apache.solr.common.util.StrUtils;import org.apache.solr.core.CoreContainer;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.metrics.SolrMetricManager;+import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.response.SolrQueryResponse;@@ -69,6 +79,16 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamepublic static final String KEY_PARAM = "key";public static final String EXPR_PARAM = "expr";public static final String TYPE_PARAM = "type";+ // Prometheus filtering parameters+ public static final String CATEGORY_PARAM = "category";+ public static final String CORE_PARAM = "core";+ public static final String COLLECTION_PARAM = "collection";+ public static final String SHARD_PARAM = "shard";+ public static final String REPLICA_PARAM = "replica";+ public static final String METRIC_NAME_PARAM = "name";+ private static final Set<String> labelFilterKeys =+ Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_PARAM);+// NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metricspublic static final String PROMETHEUS_METRICS_WT = "prometheus";public static final String OPEN_METRICS_WT = "openmetrics";@@ -131,7 +151,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName// NOCOMMIT SOLR-17458: Make this the default option after dropwizard removalif (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))|| OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) {- consumer.accept("metrics", metricManager.getPrometheusMetricReaders());+ handlePrometheusRequest(params, consumer);return;}@@ -151,6 +171,58 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNameconsumer.accept("metrics", response);}+ private void handlePrometheusRequest(SolrParams params, BiConsumer<String, Object> consumer) {+ Set<String> metricNames = readParamsAsSet(params, METRIC_NAME_PARAM);+ SortedMap<String, Set<String>> labelFilters = labelFilters(params);++ if (metricNames.isEmpty() && labelFilters.isEmpty()) {+ consumer.accept(+ "metrics",+ mergeSnapshots(+ metricManager.getPrometheusMetricReaders().values().stream()+ .flatMap(r -> r.collect().stream())+ .toList()));+ return;+ }++ List<MetricSnapshot> allSnapshots = new ArrayList<>();+ for (FilterablePrometheusMetricReader reader :+ metricManager.getPrometheusMetricReaders().values()) {+ MetricSnapshots filteredSnapshots = reader.collect(metricNames, labelFilters);+ filteredSnapshots.forEach(allSnapshots::add);+ }++ // Merge all filtered snapshots and return the merged result+ MetricSnapshots mergedSnapshots = mergeSnapshots(allSnapshots);+ consumer.accept("metrics", mergedSnapshots);+ }++ private SortedMap<String, Set<String>> labelFilters(SolrParams params) {+ SortedMap<String, Set<String>> labelFilters = new TreeMap<>();+ labelFilterKeys.forEach(+ (paramName) -> {+ Set<String> filterValues = readParamsAsSet(params, paramName);+ if (!filterValues.isEmpty()) {+ labelFilters.put(paramName, filterValues);+ }+ });++ return labelFilters;+ }++ private Set<String> readParamsAsSet(SolrParams params, String paramName) {+ String[] paramValues = params.getParams(paramName);+ if (paramValues == null || paramValues.length == 0) {+ return Set.of();+ }++ List<String> paramSet = new ArrayList<>();+ for (String param : paramValues) {+ paramSet.addAll(StrUtils.splitSmart(param, ','));+ }+ return Set.copyOf(paramSet);+ }+private NamedList<Object> handleDropwizardRegistry(SolrParams params) {boolean compact = params.getBool(COMPACT_PARAM, true);MetricFilter mustMatchFilter = parseMustMatchFilter(params);@@ -182,6 +254,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn response;}+ // NOCOMMIT: Remove this filtering logicprivate static class MetricsExpr {Pattern registryRegex;MetricFilter metricFilter;@@ -499,9 +572,87 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn metricTypes;}- private String getCoreNameFromRegistry(String registryName) {- String coreName = registryName.substring(registryName.indexOf('.') + 1);- return coreName.replace(".", "_");+ /**+ * Merge a collection of individual {@link MetricSnapshot} instances into one {@link+ * MetricSnapshots}. This is necessary because we create a {@link+ * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric+ * names across cores which is an illegal format if under the same prometheus grouping.+ */+ private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {+ Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();+ Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();+ Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();+ InfoSnapshot otelInfoSnapshots = null;++ for (MetricSnapshot snapshot : snapshots) {+ String metricName = snapshot.getMetadata().getPrometheusName();++ switch (snapshot) {+ case CounterSnapshot counterSnapshot -> {+ CounterSnapshot.Builder builder =+ counterSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ CounterSnapshot.builder()+ .name(counterSnapshot.getMetadata().getName())+ .help(counterSnapshot.getMetadata().getHelp());+ return counterSnapshot.getMetadata().hasUnit()+ ? base.unit(counterSnapshot.getMetadata().getUnit())+ : base;+ });+ counterSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case GaugeSnapshot gaugeSnapshot -> {+ GaugeSnapshot.Builder builder =+ gaugeSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ GaugeSnapshot.builder()+ .name(gaugeSnapshot.getMetadata().getName())+ .help(gaugeSnapshot.getMetadata().getHelp());+ return gaugeSnapshot.getMetadata().hasUnit()+ ? base.unit(gaugeSnapshot.getMetadata().getUnit())+ : base;+ });+ gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case HistogramSnapshot histogramSnapshot -> {+ HistogramSnapshot.Builder builder =+ histogramSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ HistogramSnapshot.builder()+ .name(histogramSnapshot.getMetadata().getName())+ .help(histogramSnapshot.getMetadata().getHelp());+ return histogramSnapshot.getMetadata().hasUnit()+ ? base.unit(histogramSnapshot.getMetadata().getUnit())+ : base;+ });+ histogramSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case InfoSnapshot infoSnapshot -> {+ // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate+ // metric called target_info containing OTEL SDK metadata. Only one of these need to be+ // kept+ if (otelInfoSnapshots == null)+ otelInfoSnapshots =+ new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());+ }+ default -> {+ // Handle unexpected snapshot types gracefully+ }+ }+ }++ MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();+ counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);+ return snapshotsBuilder.build();}@Overridediff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.javaindex bda5fa4af39..4ddc6443f80 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java@@ -52,7 +52,6 @@ import io.opentelemetry.api.metrics.ObservableLongGauge;import io.opentelemetry.api.metrics.ObservableLongMeasurement;import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;import io.opentelemetry.api.metrics.ObservableMeasurement;-import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;import io.opentelemetry.sdk.metrics.SdkMeterProvider;import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter;@@ -86,6 +85,7 @@ import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrInfoBean;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.logging.MDCLoggingContext;+import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;import org.apache.solr.metrics.otel.OtelUnit;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -753,7 +753,7 @@ public class SolrMetricManager {.computeIfAbsent(providerName,key -> {- var reader = new PrometheusMetricReader(true, null);+ var reader = new FilterablePrometheusMetricReader(true, null);// NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP// with an exportervar provider = SdkMeterProvider.builder().registerMetricReader(reader);@@ -1640,17 +1640,17 @@ public class SolrMetricManager {return metricsConfig;}- /** Get a shallow copied map of {@link PrometheusMetricReader}. */- public Map<String, PrometheusMetricReader> getPrometheusMetricReaders() {+ /** Get a shallow copied map of {@link FilterablePrometheusMetricReader}. */+ public Map<String, FilterablePrometheusMetricReader> getPrometheusMetricReaders() {return meterProviderAndReaders.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().prometheusMetricReader()));}- public PrometheusMetricReader getPrometheusMetricReader(String providerName) {+ public FilterablePrometheusMetricReader getPrometheusMetricReader(String providerName) {MeterProviderAndReaders mpr = meterProviderAndReaders.get(enforcePrefix(providerName));return (mpr != null) ? mpr.prometheusMetricReader() : null;}private record MeterProviderAndReaders(- SdkMeterProvider sdkMeterProvider, PrometheusMetricReader prometheusMetricReader) {}+ SdkMeterProvider sdkMeterProvider, FilterablePrometheusMetricReader prometheusMetricReader) {}}diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.javanew file mode 100644index 00000000000..4cd321c112f--- /dev/null+++ b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java@@ -0,0 +1,185 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel;++import static java.util.stream.Collectors.toList;++import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.DataPointSnapshot;+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;+import io.prometheus.metrics.model.snapshots.InfoSnapshot;+import io.prometheus.metrics.model.snapshots.Labels;+import io.prometheus.metrics.model.snapshots.MetricSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshots;+import java.lang.invoke.MethodHandles;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.SortedMap;+import java.util.function.Predicate;+import java.util.stream.Collectors;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++public class FilterablePrometheusMetricReader extends PrometheusMetricReader {++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private static final Set<String> PROM_SUFFIXES =+ Set.of("_total", "_sum", "_count", "_bucket", "_gcount", "_gsum", "_created", "_info");++ public FilterablePrometheusMetricReader(+ boolean otelScopeEnabled, Predicate<String> allowedResourceAttributesFilter) {+ super(otelScopeEnabled, allowedResourceAttributesFilter);+ }++ /**+ * Collect metrics with filtering support for metric names and labels.+ *+ * @param includedNames Set of metric names to include. If empty, all metric names are included.+ * @param requiredLabels Map of label names to their allowed values. If empty, no label filtering+ * is applied.+ * @return Filtered MetricSnapshots+ */+ public MetricSnapshots collect(+ Set<String> includedNames, SortedMap<String, Set<String>> requiredLabels) {++ // If no filtering is requested then return all metrics+ if (includedNames.isEmpty() && requiredLabels.isEmpty()) {+ return super.collect();+ }++ // Prometheus appends a suffix to the metrics depending on the metric type. We need to sanitize+ // the suffix off if they filter by Prometheus name instead of OTEL name.+ Set<String> sanitizedNames =+ includedNames.stream()+ .map(+ (name) -> {+ for (String suffix : PROM_SUFFIXES) {+ if (name.endsWith(suffix)) {+ return name.substring(0, name.lastIndexOf(suffix));+ }+ }+ return name;+ })+ .collect(Collectors.toSet());++ MetricSnapshots snapshotsToFilter;+ if (sanitizedNames.isEmpty()) {+ snapshotsToFilter = super.collect();+ } else {+ snapshotsToFilter = super.collect(sanitizedNames::contains);+ }++ // Return named filtered snapshots if not label filters provided+ if (requiredLabels.isEmpty()) {+ return snapshotsToFilter;+ }++ MetricSnapshots.Builder filteredSnapshots = MetricSnapshots.builder();+ for (MetricSnapshot metricSnapshot : snapshotsToFilter) {+ switch (metricSnapshot) {+ case CounterSnapshot snapshot -> {+ List<CounterSnapshot.CounterDataPointSnapshot> filtered =+ filterDatapoint(+ snapshot, requiredLabels, CounterSnapshot.CounterDataPointSnapshot.class);+ if (!filtered.isEmpty()) {+ filteredSnapshots.metricSnapshot(new CounterSnapshot(snapshot.getMetadata(), filtered));+ }+ }+ case HistogramSnapshot snapshot -> {+ List<HistogramSnapshot.HistogramDataPointSnapshot> filtered =+ filterDatapoint(+ snapshot, requiredLabels, HistogramSnapshot.HistogramDataPointSnapshot.class);+ if (!filtered.isEmpty()) {+ filteredSnapshots.metricSnapshot(+ new HistogramSnapshot(snapshot.getMetadata(), filtered));+ }+ }+ case GaugeSnapshot snapshot -> {+ List<GaugeSnapshot.GaugeDataPointSnapshot> filtered =+ filterDatapoint(snapshot, requiredLabels, GaugeSnapshot.GaugeDataPointSnapshot.class);+ if (!filtered.isEmpty()) {+ filteredSnapshots.metricSnapshot(new GaugeSnapshot(snapshot.getMetadata(), filtered));+ }+ }+ case InfoSnapshot ignored -> {+ // Do nothing for InfoSnapshots. Always filter it out+ }+ default -> {+ log.error("Unknown metric snapshot type {}", metricSnapshot.getClass());+ }+ }+ }+ return filteredSnapshots.build();+ }++ private <D extends DataPointSnapshot> List<D> filterDatapoint(+ MetricSnapshot snapshot,+ SortedMap<String, Set<String>> requiredLabels,+ Class<D> dataPointClass) {+ return snapshot.getDataPoints().stream()+ .filter(dp -> requiredLabelsFilter(dp.getLabels(), requiredLabels))+ .map(dataPointClass::cast)+ .collect(toList());+ }++ static boolean requiredLabelsFilter(+ Labels labels, SortedMap<String, Set<String>> requiredLabels) {+ // Both Labels and requiredLabels are name-sorted with unique names.+ // For each required label, scan forward through labels until we find it.+ int labelIdx = 0;+ requireLoop:+ for (Map.Entry<String, Set<String>> entry : requiredLabels.entrySet()) {+ String requiredLabelName = entry.getKey();+ Set<String> allowedValues = entry.getValue();++ // Labels are sorted, so we can continue from the last position+ while (labelIdx < labels.size()) {+ String labelName = labels.getName(labelIdx);++ int comparison = labelName.compareTo(requiredLabelName);++ if (comparison < 0) {+ // a label before requiredLabelName; we don't care about this one+ assert !requiredLabels.containsKey(labelName);+ labelIdx++;+ continue; // inspect the next input label+ }+ if (comparison > 0) {+ // We've passed where requiredLabelName should be, so it doesn't exist+ assert !labels.contains(requiredLabelName);+ return false;+ }++ assert labels.contains(requiredLabelName);+ if (!allowedValues.contains(labels.getValue(labelIdx))) {+ return false;+ }+ // we satisfied requiredLabelName with its allowedValues. Move onto next+ labelIdx++;+ continue requireLoop;+ } // end while loop on input Labels+ // didn't find requiredLabelName; ran out of names+ assert !labels.contains(requiredLabelName);+ return false;+ }+ return true; // found all requiredLabels in Labels+ }+}diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.javaindex 070ed662dc0..a5f69e1632b 100644--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java@@ -18,27 +18,14 @@ package org.apache.solr.response;import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;-import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;-import io.opentelemetry.sdk.metrics.SdkMeterProvider;import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;-import io.prometheus.metrics.model.snapshots.CounterSnapshot;-import io.prometheus.metrics.model.snapshots.GaugeSnapshot;-import io.prometheus.metrics.model.snapshots.HistogramSnapshot;-import io.prometheus.metrics.model.snapshots.InfoSnapshot;-import io.prometheus.metrics.model.snapshots.MetricSnapshot;import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.io.IOException;import java.io.OutputStream;-import java.lang.invoke.MethodHandles;-import java.util.HashMap;-import java.util.List;-import java.util.Map;import org.apache.solr.common.params.CommonParams;import org.apache.solr.handler.admin.MetricsHandler;import org.apache.solr.request.SolrQueryRequest;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */@SuppressWarnings(value = "unchecked")@@ -49,23 +36,17 @@ public class PrometheusResponseWriter implements QueryResponseWriter {private static final String CONTENT_TYPE_OPEN_METRICS ="application/openmetrics-text; version=1.0.0; charset=utf-8";- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());-@Overridepublic void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType)throws IOException {- Map<String, PrometheusMetricReader> readers =- (Map<String, PrometheusMetricReader>) response.getValues().get("metrics");-- List<MetricSnapshot> snapshots =- readers.values().stream().flatMap(r -> r.collect().stream()).toList();-+ var metrics = response.getValues().get("metrics");+ MetricSnapshots snapshots = (MetricSnapshots) metrics;if (writeOpenMetricsFormat(request)) {- new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots));+ new OpenMetricsTextFormatWriter(false, true).write(out, snapshots);} else {- new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));+ new PrometheusTextFormatWriter(false).write(out, snapshots);}}@@ -92,90 +73,4 @@ public class PrometheusResponseWriter implements QueryResponseWriter {return acceptHeader.contains("application/openmetrics-text")&& (acceptHeader.contains("version=1.0.0"));}-- /**- * Merge a collection of individual {@link MetricSnapshot} instances into one {@link- * MetricSnapshots}. This is necessary because we create a {@link SdkMeterProvider} per Solr core- * resulting in duplicate metric names across cores which is an illegal format if not under the- * same prometheus grouping.- */- private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {- Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();- Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();- Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();- InfoSnapshot otelInfoSnapshots = null;-- for (MetricSnapshot snapshot : snapshots) {- String metricName = snapshot.getMetadata().getPrometheusName();-- switch (snapshot) {- case CounterSnapshot counterSnapshot -> {- CounterSnapshot.Builder builder =- counterSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- CounterSnapshot.builder()- .name(counterSnapshot.getMetadata().getName())- .help(counterSnapshot.getMetadata().getHelp());- return counterSnapshot.getMetadata().hasUnit()- ? base.unit(counterSnapshot.getMetadata().getUnit())- : base;- });- counterSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case GaugeSnapshot gaugeSnapshot -> {- GaugeSnapshot.Builder builder =- gaugeSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- GaugeSnapshot.builder()- .name(gaugeSnapshot.getMetadata().getName())- .help(gaugeSnapshot.getMetadata().getHelp());- return gaugeSnapshot.getMetadata().hasUnit()- ? base.unit(gaugeSnapshot.getMetadata().getUnit())- : base;- });- gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case HistogramSnapshot histogramSnapshot -> {- HistogramSnapshot.Builder builder =- histogramSnapshotMap.computeIfAbsent(- metricName,- k -> {- var base =- HistogramSnapshot.builder()- .name(histogramSnapshot.getMetadata().getName())- .help(histogramSnapshot.getMetadata().getHelp());- return histogramSnapshot.getMetadata().hasUnit()- ? base.unit(histogramSnapshot.getMetadata().getUnit())- : base;- });- histogramSnapshot.getDataPoints().forEach(builder::dataPoint);- }- case InfoSnapshot infoSnapshot -> {- // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate- // metric called target_info containing OTEL SDK metadata. Only one of these need to be- // kept- if (otelInfoSnapshots == null)- otelInfoSnapshots =- new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());- }- default -> {- log.warn(- "Unexpected snapshot type: {} for metric {}",- snapshot.getClass().getName(),- snapshot.getMetadata().getName());- }- }- }-- MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();- counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));- if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);- return snapshotsBuilder.build();- }}diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.javaindex 0878bf2c8cd..6cb09d3d95d 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java@@ -17,33 +17,19 @@package org.apache.solr.handler.admin;-import com.codahale.metrics.Counter;import io.opentelemetry.api.common.Attributes;-import io.prometheus.metrics.model.snapshots.CounterSnapshot;-import io.prometheus.metrics.model.snapshots.GaugeSnapshot;-import io.prometheus.metrics.model.snapshots.Labels;-import io.prometheus.metrics.model.snapshots.MetricSnapshot;import io.prometheus.metrics.model.snapshots.MetricSnapshots;-import io.prometheus.metrics.model.snapshots.SummarySnapshot;-import java.util.Arrays;-import java.util.HashMap;-import java.util.List;import java.util.Map;+import java.util.Set;+import java.util.stream.Collectors;import org.apache.solr.SolrTestCaseJ4;-import org.apache.solr.common.MapWriter;import org.apache.solr.common.params.CommonParams;-import org.apache.solr.common.util.NamedList;-import org.apache.solr.common.util.SimpleOrderedMap;-import org.apache.solr.core.PluginBag;-import org.apache.solr.core.PluginInfo;import org.apache.solr.handler.RequestHandlerBase;import org.apache.solr.metrics.MetricsMap;import org.apache.solr.metrics.SolrMetricsContext;import org.apache.solr.request.SolrQueryRequest;-import org.apache.solr.request.SolrRequestHandler;import org.apache.solr.response.SolrQueryResponse;import org.apache.solr.security.AuthorizationContext;-import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;@@ -51,699 +37,228 @@ import org.junit.Test;public class MetricsHandlerTest extends SolrTestCaseJ4 {@BeforeClasspublic static void beforeClass() throws Exception {-initCore("solrconfig-minimal.xml", "schema.xml");h.getCoreContainer().waitForLoadingCoresToFinish(30000);-- // manually register & seed some metrics in solr.jvm and solr.jetty for testing via handler- // (use "solrtest_" prefix just in case the jvm or jetty adds a "foo" metric at some point)- Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "solrtest_foo");- c.inc();- c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo");- c.inc(2);- // test escapes- c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo:bar");- c.inc(3);-- h.getCoreContainer()- .getMetricManager()- .meter(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses");- h.getCoreContainer()- .getMetricManager()- .counter(- null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.active-requests");- h.getCoreContainer()- .getMetricManager()- .timer(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.dispatches");- }-- @AfterClass- public static void cleanupMetrics() {- if (null != h) {- h.getCoreContainer().getMetricManager().registry("solr.jvm").remove("solrtest_foo");- h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");- h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");- }}- // NOCOMMIT: This test does a bunch of /admin/metrics calls with various params, with various- // filters and parameters. We have not migrated all the metrics to otel yet or even created any- // filters. Once that is done, we should revisit this test and assert the prometheus response.@Test- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")- public void test() throws Exception {+ public void testMetricNamesFiltering() throws Exception {+ String expectedRequestsMetricName = "solr_core_requests";MetricsHandler handler = new MetricsHandler(h.getCoreContainer());+ assertQ(req("*:*"), "//result[@numFound='0']");+SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",CommonParams.WT,- "json"),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.METRIC_NAME_PARAM,+ expectedRequestsMetricName),resp);- NamedList<?> values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1");- assertNotNull(nl);- Object o = nl.get("SEARCHER.new.errors");- assertNotNull(o); // counter type- assertTrue(o instanceof MapWriter);- // response wasn't serialized, so we get here whatever MetricUtils produced instead of NamedList- assertNotNull(((MapWriter) o)._get("count"));- assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count"));- assertNotNull(nl.get("INDEX.segments")); // int gauge- assertTrue((int) ((MapWriter) nl.get("INDEX.segments"))._get("value") >= 0);- assertNotNull(nl.get("INDEX.sizeInBytes")); // long gauge- assertTrue((long) ((MapWriter) nl.get("INDEX.sizeInBytes"))._get("value") >= 0);- nl = (NamedList<?>) values.get("solr.node");- assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge- assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value"));- assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type- Map<String, Object> map = new HashMap<>();- ((MapWriter) nl.get("ADMIN./admin/authorization.clientErrors")).toMap(map);- assertEquals(5, map.size());-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "jvm,jetty"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(2, values.size());- assertNotNull(values.get("solr.jetty"));- assertNotNull(values.get("solr.jvm"));-- resp = new SolrQueryResponse();- // "collection" works too, because it's a prefix for "collection1"- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "registry",- "solr.core.collection,solr.jvm"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(2, values.size());- assertNotNull(values.get("solr.core.collection1"));- assertNotNull(values.get("solr.jvm"));-- resp = new SolrQueryResponse();- // "collection" works too, because it's a prefix for "collection1"- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "registry",- "solr.core.collection",- "registry",- "solr.jvm"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(2, values.size());- assertNotNull(values.get("solr.core.collection1"));- assertNotNull(values.get("solr.jvm"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "jvm,jetty"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(2, values.size());- assertNotNull(values.get("solr.jetty"));- assertNotNull(values.get("solr.jvm"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "jvm",- "group",- "jetty"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(2, values.size());- assertNotNull(values.get("solr.jetty"));- assertNotNull(values.get("solr.jvm"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "node",- "type",- "counter"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(1, values.size());- values = (NamedList<?>) values.get("solr.node");- assertNotNull(values);- assertNull(values.get("ADMIN./admin/authorization.errors")); // this is a timer node-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "prefix",- "CONTAINER.cores,CONTAINER.threadPool"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(1, values.size());- assertNotNull(values.get("solr.node"));- values = (NamedList<?>) values.get("solr.node");- assertEquals(15, values.size());- assertNotNull(values.get("CONTAINER.cores.lazy")); // this is a gauge node- assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "prefix",- "CONTAINER.cores",- "regex",- "C.*thread.*completed"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertNotNull(values.get("solr.node"));- values = (NamedList<?>) values.get("solr.node");- assertEquals(5, values.size());- assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- "prefix",- "CACHE.core.fieldCache",- "property",- "entries_count",- MetricsHandler.COMPACT_PARAM,- "true"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertNotNull(values.get("solr.core.collection1"));- values = (NamedList<?>) values.get("solr.core.collection1");- assertEquals(1, values.size());- MapWriter writer = (MapWriter) values.get("CACHE.core.fieldCache");- assertNotNull(writer);- assertNotNull(writer._get("entries_count"));-- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "jvm",- "prefix",- "CONTAINER.cores"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- assertEquals(0, values.size());+ var metrics = resp.getValues().get("metrics");+ MetricSnapshots snapshots = (MetricSnapshots) metrics;+ assertEquals(1, snapshots.size());+ assertEquals(expectedRequestsMetricName, snapshots.get(0).getMetadata().getPrometheusName());- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- MetricsHandler.COMPACT_PARAM,- "false",- CommonParams.WT,- "json",- "group",- "node",- "type",- "timer",- "prefix",- "CONTAINER.cores"),- resp);- values = resp.getValues();- assertNotNull(values.get("metrics"));- SimpleOrderedMap<?> map1 = (SimpleOrderedMap<?>) values.get("metrics");- assertEquals(0, map1.size());handler.close();}@Test- public void testPropertyFilter() throws Exception {- assertQ(req("*:*"), "//result[@numFound='0']");-+ public void testMultipleMetricNamesFiltering() throws Exception {+ String expectedRequestsMetricName = "solr_core_requests";+ String expectedSearcherMetricName = "solr_core_searcher_new";+ var expected = Set.of(expectedRequestsMetricName, expectedSearcherMetricName);MetricsHandler handler = new MetricsHandler(h.getCoreContainer());+ assertQ(req("*:*"), "//result[@numFound='0']");+SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.COMPACT_PARAM,- "true",- "group",- "core",- "prefix",- "CACHE.searcher"),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.METRIC_NAME_PARAM,+ expectedRequestsMetricName + "," + expectedSearcherMetricName),resp);- NamedList<?> values = resp.getValues();- assertNotNull(values.get("metrics"));- values = (NamedList<?>) values.get("metrics");- NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1");- assertNotNull(nl);- assertTrue(nl.size() > 0);- nl.forEach(- (k, v) -> {- assertTrue(v instanceof MapWriter);- Map<String, Object> map = new HashMap<>();- ((MapWriter) v).toMap(map);- assertTrue(map.size() > 2);- });- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.COMPACT_PARAM,- "true",- "group",- "core",- "prefix",- "CACHE.searcher",- "property",- "inserts",- "property",- "size"),- resp);- values = resp.getValues();- values = (NamedList<?>) values.get("metrics");- nl = (NamedList<?>) values.get("solr.core.collection1");- assertNotNull(nl);- assertTrue(nl.size() > 0);- nl.forEach(- (k, v) -> {- assertTrue(v instanceof MapWriter);- Map<String, Object> map = new HashMap<>();- ((MapWriter) v).toMap(map);- assertEquals("k=" + k + ", v=" + map, 2, map.size());- assertNotNull(map.get("inserts"));- assertNotNull(map.get("size"));- });+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");+ assertEquals(2, metrics.size());+ Set<String> actual =+ metrics.stream().map(m -> m.getMetadata().getPrometheusName()).collect(Collectors.toSet());+ assertEquals(expected, actual);+handler.close();}- // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet@Test- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")- public void testKeyMetrics() throws Exception {+ public void testNonExistentMetricNameFiltering() throws Exception {+ String nonexistentMetricName = "nonexistent_metric_name";MetricsHandler handler = new MetricsHandler(h.getCoreContainer());- String key1 = "solr.core.collection1:CACHE.core.fieldCache";SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- key1),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.METRIC_NAME_PARAM,+ nonexistentMetricName),resp);- NamedList<?> values = resp.getValues();- Object val = values._get(List.of("metrics", key1), null);- assertNotNull(val);- assertTrue(val instanceof MapWriter);- assertTrue(((MapWriter) val)._size() >= 2);-- String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- key2),- resp);- val = resp.getValues()._get("metrics/" + key2);- assertNotNull(val);- assertTrue(val instanceof Number);+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");+ assertEquals(0, metrics.size());+ handler.close();+ }- String key3 = "solr.jetty:solrtest_foo\\:bar";- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- key3),- resp);+ @Test+ public void testLabelFiltering() throws Exception {+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());- val = resp.getValues()._get("metrics/" + key3);- assertNotNull(val);- assertTrue(val instanceof Number);- assertEquals(3, ((Number) val).intValue());+ assertQ(req("*:*"), "//result[@numFound='0']");- // test multiple keys- resp = new SolrQueryResponse();+ SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- key1,- MetricsHandler.KEY_PARAM,- key2,- MetricsHandler.KEY_PARAM,- key3),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.CATEGORY_PARAM,+ "QUERY"),resp);+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");++ metrics.forEach(+ (ms) -> {+ ms.getDataPoints()+ .forEach(+ (dp) -> {+ assertEquals("QUERY", dp.getLabels().get(MetricsHandler.CATEGORY_PARAM));+ });+ });- val = resp.getValues()._get("metrics/" + key1);- assertNotNull(val);- val = resp.getValues()._get("metrics/" + key2);- assertNotNull(val);- val = resp.getValues()._get("metrics/" + key3);- assertNotNull(val);+ handler.close();+ }- String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- key4),- resp);- // the key contains a slash, need explicit list of path elements- val = resp.getValues()._get(Arrays.asList("metrics", key4), null);- assertNotNull(val);- assertTrue(val instanceof Number);+ @Test+ public void testMultipleLabelFiltering() throws Exception {+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());- // test errors+ assertQ(req("*:*"), "//result[@numFound='0']");- // invalid keys- resp = new SolrQueryResponse();+ SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- "foo",- MetricsHandler.KEY_PARAM,- "foo:bar:baz:xyz"),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.CATEGORY_PARAM,+ "QUERY" + "," + "SEARCHER"),resp);- values = resp.getValues();- NamedList<?> metrics = (NamedList<?>) values.get("metrics");- assertEquals(0, metrics.size());- assertNotNull(values._get(List.of("errors", "foo"), null));- assertNotNull(values._get(List.of("errors", "foo:bar:baz:xyz"), null));- // unknown registry- resp = new SolrQueryResponse();+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");+ metrics.forEach(+ (ms) -> {+ ms.getDataPoints()+ .forEach(+ (dp) -> {+ assertTrue(+ dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("QUERY")+ || dp.getLabels()+ .get(MetricsHandler.CATEGORY_PARAM)+ .equals("SEARCHER"));+ });+ });++ handler.close();+ }++ @Test+ public void testNonExistentLabelFiltering() throws Exception {+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());++ SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- "foo:bar:baz"),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.CORE_PARAM,+ "nonexistent_core_name"),resp);- values = resp.getValues();- metrics = (NamedList<?>) values.get("metrics");++ var metrics = (MetricSnapshots) resp.getValues().get("metrics");assertEquals(0, metrics.size());- assertNotNull(values._get(List.of("errors", "foo:bar:baz"), null));+ handler.close();+ }++ @Test+ public void testMixedLabelFiltering() throws Exception {+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());++ assertQ(req("*:*"), "//result[@numFound='0']");- // unknown metric- resp = new SolrQueryResponse();+ SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.KEY_PARAM,- "solr.jetty:unknown:baz"),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.CORE_PARAM,+ "collection1",+ MetricsHandler.CATEGORY_PARAM,+ "SEARCHER"),resp);- values = resp.getValues();- metrics = (NamedList<?>) values.get("metrics");- assertEquals(0, metrics.size());- assertNotNull(values._get(List.of("errors", "solr.jetty:unknown:baz"), null));++ var metrics = (MetricSnapshots) resp.getValues().get("metrics");+ metrics.forEach(+ (ms) -> {+ ms.getDataPoints()+ .forEach(+ (dp) -> {+ assertTrue(+ dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("SEARCHER")+ && dp.getLabels().get(MetricsHandler.CORE_PARAM).equals("collection1"));+ });+ });handler.close();}- // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet@Test- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")- @SuppressWarnings("unchecked")- public void testExprMetrics() throws Exception {+ public void testMetricNamesAndLabelFiltering() throws Exception {+ String expectedMetricName = "solr_core_segments";MetricsHandler handler = new MetricsHandler(h.getCoreContainer());- String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate";+ assertQ(req("*:*"), "//result[@numFound='0']");+SolrQueryResponse resp = new SolrQueryResponse();handler.handleRequestBody(req(CommonParams.QT,"/admin/metrics",CommonParams.WT,- "json",- MetricsHandler.EXPR_PARAM,- key1),- resp);- // response structure is like in the case of non-key params- Object val =- resp.getValues()- ._get(List.of("metrics", "solr.core.collection1", "QUERY./select.requestTimes"), null);- assertNotNull(val);- assertTrue(val instanceof MapWriter);- Map<String, Object> map = new HashMap<>();- ((MapWriter) val).toMap(map);- assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15- assertNotNull(map.toString(), map.get("meanRate"));- assertNotNull(map.toString(), map.get("1minRate"));- assertNotNull(map.toString(), map.get("5minRate"));- assertNotNull(map.toString(), map.get("15minRate"));- assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);- map.clear();-- String key2 = "solr\\.core\\..*:.*/select\\.request.*";- resp = new SolrQueryResponse();- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.EXPR_PARAM,- key2),+ MetricsHandler.PROMETHEUS_METRICS_WT,+ MetricsHandler.CATEGORY_PARAM,+ "CORE",+ MetricsHandler.METRIC_NAME_PARAM,+ expectedMetricName),resp);- // response structure is like in the case of non-key params- val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null);- assertNotNull(val);- Object v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requestTimes");- assertNotNull(v);- assertTrue(v instanceof MapWriter);- ((MapWriter) v).toMap(map);- assertEquals(map.toString(), 14, map.size());- assertNotNull(map.toString(), map.get("1minRate"));- assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);- map.clear();- // select requests counter- v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests");- assertNotNull(v);- assertTrue(v instanceof Number);-- // test multiple expressions producing overlapping metrics - should be no dupes-- // this key matches also sub-metrics of /select, eg. /select[shard], ...- String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";- resp = new SolrQueryResponse();- // ORDER OF PARAMS MATTERS HERE! see the refguide- handler.handleRequestBody(- req(- CommonParams.QT,- "/admin/metrics",- CommonParams.WT,- "json",- MetricsHandler.EXPR_PARAM,- key2,- MetricsHandler.EXPR_PARAM,- key1,- MetricsHandler.EXPR_PARAM,- key3),- resp);- val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null);- assertNotNull(val);- // for requestTimes only the full set of values from the first expr should be present- assertNotNull(val);- SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;- assertEquals(values.jsonStr(), 3, values.size());- v = values.get("QUERY./select.requestTimes");- assertTrue(v instanceof MapWriter);- ((MapWriter) v).toMap(map);- assertTrue(map.toString(), map.containsKey("count"));- map.clear();- v = values.get("QUERY./select[shard].requestTimes");- assertTrue(v instanceof MapWriter);- ((MapWriter) v).toMap(map);- assertTrue(map.toString(), map.containsKey("count"));- }- private MetricSnapshot getMetricSnapshot(MetricSnapshots snapshots, String metricName) {- return snapshots.stream()- .filter(ss -> ss.getMetadata().getPrometheusName().equals(metricName))- .findAny()- .get();- }-- private GaugeSnapshot.GaugeDataPointSnapshot getGaugeDatapointSnapshot(- MetricSnapshot snapshot, Labels labels) {- return (GaugeSnapshot.GaugeDataPointSnapshot)- snapshot.getDataPoints().stream()- .filter(ss -> ss.getLabels().hasSameValues(labels))- .findAny()- .get();- }-- private CounterSnapshot.CounterDataPointSnapshot getCounterDatapointSnapshot(- MetricSnapshot snapshot, Labels labels) {- return (CounterSnapshot.CounterDataPointSnapshot)- snapshot.getDataPoints().stream()- .filter(ss -> ss.getLabels().hasSameValues(labels))- .findAny()- .get();- }-- private SummarySnapshot.SummaryDataPointSnapshot getSummaryDataPointSnapshot(- MetricSnapshot snapshot, Labels labels) {- return (SummarySnapshot.SummaryDataPointSnapshot)- snapshot.getDataPoints().stream()- .filter(ss -> ss.getLabels().hasSameValues(labels))- .findAny()- .get();- }-- static class RefreshablePluginHolder extends PluginBag.PluginHolder<SolrRequestHandler> {-- private DumpRequestHandler rh;- private SolrMetricsContext metricsInfo;-- public RefreshablePluginHolder(PluginInfo info, DumpRequestHandler rh) {- super(info);- this.rh = rh;- }-- @Override- public boolean isLoaded() {- return true;- }-- void closeHandler() throws Exception {- this.metricsInfo = rh.getSolrMetricsContext();- // if(metricsInfo.tag.contains(String.valueOf(rh.hashCode()))){- // //this created a new child metrics- // metricsInfo = metricsInfo.getParent();- // }- this.rh.close();- }-- void reset(DumpRequestHandler rh) {- this.rh = rh;- if (metricsInfo != null)- this.rh.initializeMetrics(metricsInfo, Attributes.empty(), "/dumphandler");- }-- @Override- public SolrRequestHandler get() {- return rh;- }+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");+ assertEquals(1, metrics.size());+ var actualDatapoint = metrics.get(0).getDataPoints().getFirst();+ assertEquals(expectedMetricName, metrics.get(0).getMetadata().getPrometheusName());+ assertEquals("CORE", actualDatapoint.getLabels().get(MetricsHandler.CATEGORY_PARAM));+ handler.close();}public static class DumpRequestHandler extends RequestHandlerBase {diff --git a/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.javanew file mode 100644index 00000000000..ec91e149b4b--- /dev/null+++ b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java@@ -0,0 +1,95 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.metrics.otel;++import io.prometheus.metrics.model.snapshots.Labels;+import java.util.Map;+import java.util.Set;+import java.util.SortedMap;+import java.util.TreeMap;+import org.apache.solr.SolrTestCaseJ4;+import org.junit.Test;++/**+ * Test class for FilterablePrometheusMetricReader which focuses on requiredLabelsFilter method+ * which filters metric data points based on label requirements.+ */+public class FilterablePrometheusMetricReaderTest extends SolrTestCaseJ4 {++ Labels actualLabels = Labels.of("key1", "value1", "key2", "value2", "key3", "value3");++ @Test+ public void testFilterMatchingLabel() {+ SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of("value1")));+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterOneMatchingLabelValue() {+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(Map.of("key1", Set.of("value1", "value123", "value456", "value789")));+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterNoMatchingLabel() {+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(Map.of("dummyKey", Set.of("dummyValue")));+ assertFalse(+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterAllMultipleMatchingLabels() {+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value2")));+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterMultipleWithOneLabelValueNotMatching() {+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value999")));+ assertFalse(+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterMultipleWithOneLabelMissing() {+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key999", Set.of("value2")));+ assertFalse(+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterLabelsWithMixedValues() {+ // Scenario with multiple label values some matching and some not+ SortedMap<String, Set<String>> requiredLabels =+ new TreeMap<>(+ Map.of("key1", Set.of("value1", "value123"), "key2", Set.of("value456", "value2")));+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }++ @Test+ public void testFilterEmptyLabelValues() {+ // Edge case where label has empty set of allowed values+ SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of()));+ assertFalse(+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));+ }+}diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.javaindex 09aa0489949..ecfd83c02d4 100644--- a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java@@ -102,7 +102,7 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 {var segmentSize =SolrMetricTestUtils.getGaugeDatapoint(core,- "solr_core_segment_count",+ "solr_core_segments",SolrMetricTestUtils.newStandaloneLabelsBuilder(core).label("category", "CORE").build());