Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17854: Support filters to /admin/metrics endpoint for Prometheus metrics (#3499)

Matthew Biscocho046b206de344
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index ab4db7d04f2..a8cb4800fde 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -1422,14 +1422,13 @@ public class SolrCore implements SolrInfoBean, Closeable {
}),
OtelUnit.BYTES));
- observables.add(
- parentContext.observableLongGauge(
- "solr_core_segment_count",
- "Number of segments in a Solr core",
- (observableLongMeasurement -> {
- if (isReady())
- observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes);
- })));
+ parentContext.observableLongGauge(
+ "solr_core_segments",
+ "Number of segments in a Solr core",
+ (observableLongMeasurement -> {
+ if (isReady())
+ observableLongMeasurement.record(getSegmentCount(), baseGaugeCoreAttributes);
+ }));
// NOCOMMIT: Do we need these start_time metrics? I think at minimum it should be optional
// otherwise we fall into metric bloat for something people may not care about.
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 633ccc7bb00..143cf453f41 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -25,14 +25,23 @@ import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
+import io.prometheus.metrics.model.snapshots.CounterSnapshot;
+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
+import io.prometheus.metrics.model.snapshots.InfoSnapshot;
+import io.prometheus.metrics.model.snapshots.MetricSnapshot;
+import io.prometheus.metrics.model.snapshots.MetricSnapshots;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.function.BiConsumer;
import java.util.function.Predicate;
@@ -49,6 +58,7 @@ import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.SolrMetricManager;
+import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
@@ -69,6 +79,16 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String KEY_PARAM = "key";
public static final String EXPR_PARAM = "expr";
public static final String TYPE_PARAM = "type";
+ // Prometheus filtering parameters
+ public static final String CATEGORY_PARAM = "category";
+ public static final String CORE_PARAM = "core";
+ public static final String COLLECTION_PARAM = "collection";
+ public static final String SHARD_PARAM = "shard";
+ public static final String REPLICA_PARAM = "replica";
+ public static final String METRIC_NAME_PARAM = "name";
+ private static final Set<String> labelFilterKeys =
+ Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_PARAM);
+
// NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metrics
public static final String PROMETHEUS_METRICS_WT = "prometheus";
public static final String OPEN_METRICS_WT = "openmetrics";
@@ -131,7 +151,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
// NOCOMMIT SOLR-17458: Make this the default option after dropwizard removal
if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))
|| OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) {
- consumer.accept("metrics", metricManager.getPrometheusMetricReaders());
+ handlePrometheusRequest(params, consumer);
return;
}
@@ -151,6 +171,58 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
consumer.accept("metrics", response);
}
+ private void handlePrometheusRequest(SolrParams params, BiConsumer<String, Object> consumer) {
+ Set<String> metricNames = readParamsAsSet(params, METRIC_NAME_PARAM);
+ SortedMap<String, Set<String>> labelFilters = labelFilters(params);
+
+ if (metricNames.isEmpty() && labelFilters.isEmpty()) {
+ consumer.accept(
+ "metrics",
+ mergeSnapshots(
+ metricManager.getPrometheusMetricReaders().values().stream()
+ .flatMap(r -> r.collect().stream())
+ .toList()));
+ return;
+ }
+
+ List<MetricSnapshot> allSnapshots = new ArrayList<>();
+ for (FilterablePrometheusMetricReader reader :
+ metricManager.getPrometheusMetricReaders().values()) {
+ MetricSnapshots filteredSnapshots = reader.collect(metricNames, labelFilters);
+ filteredSnapshots.forEach(allSnapshots::add);
+ }
+
+ // Merge all filtered snapshots and return the merged result
+ MetricSnapshots mergedSnapshots = mergeSnapshots(allSnapshots);
+ consumer.accept("metrics", mergedSnapshots);
+ }
+
+ private SortedMap<String, Set<String>> labelFilters(SolrParams params) {
+ SortedMap<String, Set<String>> labelFilters = new TreeMap<>();
+ labelFilterKeys.forEach(
+ (paramName) -> {
+ Set<String> filterValues = readParamsAsSet(params, paramName);
+ if (!filterValues.isEmpty()) {
+ labelFilters.put(paramName, filterValues);
+ }
+ });
+
+ return labelFilters;
+ }
+
+ private Set<String> readParamsAsSet(SolrParams params, String paramName) {
+ String[] paramValues = params.getParams(paramName);
+ if (paramValues == null || paramValues.length == 0) {
+ return Set.of();
+ }
+
+ List<String> paramSet = new ArrayList<>();
+ for (String param : paramValues) {
+ paramSet.addAll(StrUtils.splitSmart(param, ','));
+ }
+ return Set.copyOf(paramSet);
+ }
+
private NamedList<Object> handleDropwizardRegistry(SolrParams params) {
boolean compact = params.getBool(COMPACT_PARAM, true);
MetricFilter mustMatchFilter = parseMustMatchFilter(params);
@@ -182,6 +254,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return response;
}
+ // NOCOMMIT: Remove this filtering logic
private static class MetricsExpr {
Pattern registryRegex;
MetricFilter metricFilter;
@@ -499,9 +572,87 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return metricTypes;
}
- private String getCoreNameFromRegistry(String registryName) {
- String coreName = registryName.substring(registryName.indexOf('.') + 1);
- return coreName.replace(".", "_");
+ /**
+ * Merge a collection of individual {@link MetricSnapshot} instances into one {@link
+ * MetricSnapshots}. This is necessary because we create a {@link
+ * io.opentelemetry.sdk.metrics.SdkMeterProvider} per Solr core resulting in duplicate metric
+ * names across cores which is an illegal format if under the same prometheus grouping.
+ */
+ private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {
+ Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();
+ Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();
+ Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();
+ InfoSnapshot otelInfoSnapshots = null;
+
+ for (MetricSnapshot snapshot : snapshots) {
+ String metricName = snapshot.getMetadata().getPrometheusName();
+
+ switch (snapshot) {
+ case CounterSnapshot counterSnapshot -> {
+ CounterSnapshot.Builder builder =
+ counterSnapshotMap.computeIfAbsent(
+ metricName,
+ k -> {
+ var base =
+ CounterSnapshot.builder()
+ .name(counterSnapshot.getMetadata().getName())
+ .help(counterSnapshot.getMetadata().getHelp());
+ return counterSnapshot.getMetadata().hasUnit()
+ ? base.unit(counterSnapshot.getMetadata().getUnit())
+ : base;
+ });
+ counterSnapshot.getDataPoints().forEach(builder::dataPoint);
+ }
+ case GaugeSnapshot gaugeSnapshot -> {
+ GaugeSnapshot.Builder builder =
+ gaugeSnapshotMap.computeIfAbsent(
+ metricName,
+ k -> {
+ var base =
+ GaugeSnapshot.builder()
+ .name(gaugeSnapshot.getMetadata().getName())
+ .help(gaugeSnapshot.getMetadata().getHelp());
+ return gaugeSnapshot.getMetadata().hasUnit()
+ ? base.unit(gaugeSnapshot.getMetadata().getUnit())
+ : base;
+ });
+ gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);
+ }
+ case HistogramSnapshot histogramSnapshot -> {
+ HistogramSnapshot.Builder builder =
+ histogramSnapshotMap.computeIfAbsent(
+ metricName,
+ k -> {
+ var base =
+ HistogramSnapshot.builder()
+ .name(histogramSnapshot.getMetadata().getName())
+ .help(histogramSnapshot.getMetadata().getHelp());
+ return histogramSnapshot.getMetadata().hasUnit()
+ ? base.unit(histogramSnapshot.getMetadata().getUnit())
+ : base;
+ });
+ histogramSnapshot.getDataPoints().forEach(builder::dataPoint);
+ }
+ case InfoSnapshot infoSnapshot -> {
+ // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate
+ // metric called target_info containing OTEL SDK metadata. Only one of these need to be
+ // kept
+ if (otelInfoSnapshots == null)
+ otelInfoSnapshots =
+ new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());
+ }
+ default -> {
+ // Handle unexpected snapshot types gracefully
+ }
+ }
+ }
+
+ MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();
+ counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
+ gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
+ histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
+ if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);
+ return snapshotsBuilder.build();
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index bda5fa4af39..4ddc6443f80 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -52,7 +52,6 @@ import io.opentelemetry.api.metrics.ObservableLongGauge;
import io.opentelemetry.api.metrics.ObservableLongMeasurement;
import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
import io.opentelemetry.api.metrics.ObservableMeasurement;
-import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;
import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter;
@@ -86,6 +85,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.logging.MDCLoggingContext;
+import org.apache.solr.metrics.otel.FilterablePrometheusMetricReader;
import org.apache.solr.metrics.otel.OtelUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -753,7 +753,7 @@ public class SolrMetricManager {
.computeIfAbsent(
providerName,
key -> {
- var reader = new PrometheusMetricReader(true, null);
+ var reader = new FilterablePrometheusMetricReader(true, null);
// NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP
// with an exporter
var provider = SdkMeterProvider.builder().registerMetricReader(reader);
@@ -1640,17 +1640,17 @@ public class SolrMetricManager {
return metricsConfig;
}
- /** Get a shallow copied map of {@link PrometheusMetricReader}. */
- public Map<String, PrometheusMetricReader> getPrometheusMetricReaders() {
+ /** Get a shallow copied map of {@link FilterablePrometheusMetricReader}. */
+ public Map<String, FilterablePrometheusMetricReader> getPrometheusMetricReaders() {
return meterProviderAndReaders.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().prometheusMetricReader()));
}
- public PrometheusMetricReader getPrometheusMetricReader(String providerName) {
+ public FilterablePrometheusMetricReader getPrometheusMetricReader(String providerName) {
MeterProviderAndReaders mpr = meterProviderAndReaders.get(enforcePrefix(providerName));
return (mpr != null) ? mpr.prometheusMetricReader() : null;
}
private record MeterProviderAndReaders(
- SdkMeterProvider sdkMeterProvider, PrometheusMetricReader prometheusMetricReader) {}
+ SdkMeterProvider sdkMeterProvider, FilterablePrometheusMetricReader prometheusMetricReader) {}
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java
new file mode 100644
index 00000000000..4cd321c112f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/metrics/otel/FilterablePrometheusMetricReader.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel;
+
+import static java.util.stream.Collectors.toList;
+
+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
+import io.prometheus.metrics.model.snapshots.CounterSnapshot;
+import io.prometheus.metrics.model.snapshots.DataPointSnapshot;
+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
+import io.prometheus.metrics.model.snapshots.InfoSnapshot;
+import io.prometheus.metrics.model.snapshots.Labels;
+import io.prometheus.metrics.model.snapshots.MetricSnapshot;
+import io.prometheus.metrics.model.snapshots.MetricSnapshots;
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FilterablePrometheusMetricReader extends PrometheusMetricReader {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Set<String> PROM_SUFFIXES =
+ Set.of("_total", "_sum", "_count", "_bucket", "_gcount", "_gsum", "_created", "_info");
+
+ public FilterablePrometheusMetricReader(
+ boolean otelScopeEnabled, Predicate<String> allowedResourceAttributesFilter) {
+ super(otelScopeEnabled, allowedResourceAttributesFilter);
+ }
+
+ /**
+ * Collect metrics with filtering support for metric names and labels.
+ *
+ * @param includedNames Set of metric names to include. If empty, all metric names are included.
+ * @param requiredLabels Map of label names to their allowed values. If empty, no label filtering
+ * is applied.
+ * @return Filtered MetricSnapshots
+ */
+ public MetricSnapshots collect(
+ Set<String> includedNames, SortedMap<String, Set<String>> requiredLabels) {
+
+ // If no filtering is requested then return all metrics
+ if (includedNames.isEmpty() && requiredLabels.isEmpty()) {
+ return super.collect();
+ }
+
+ // Prometheus appends a suffix to the metrics depending on the metric type. We need to sanitize
+ // the suffix off if they filter by Prometheus name instead of OTEL name.
+ Set<String> sanitizedNames =
+ includedNames.stream()
+ .map(
+ (name) -> {
+ for (String suffix : PROM_SUFFIXES) {
+ if (name.endsWith(suffix)) {
+ return name.substring(0, name.lastIndexOf(suffix));
+ }
+ }
+ return name;
+ })
+ .collect(Collectors.toSet());
+
+ MetricSnapshots snapshotsToFilter;
+ if (sanitizedNames.isEmpty()) {
+ snapshotsToFilter = super.collect();
+ } else {
+ snapshotsToFilter = super.collect(sanitizedNames::contains);
+ }
+
+ // Return named filtered snapshots if not label filters provided
+ if (requiredLabels.isEmpty()) {
+ return snapshotsToFilter;
+ }
+
+ MetricSnapshots.Builder filteredSnapshots = MetricSnapshots.builder();
+ for (MetricSnapshot metricSnapshot : snapshotsToFilter) {
+ switch (metricSnapshot) {
+ case CounterSnapshot snapshot -> {
+ List<CounterSnapshot.CounterDataPointSnapshot> filtered =
+ filterDatapoint(
+ snapshot, requiredLabels, CounterSnapshot.CounterDataPointSnapshot.class);
+ if (!filtered.isEmpty()) {
+ filteredSnapshots.metricSnapshot(new CounterSnapshot(snapshot.getMetadata(), filtered));
+ }
+ }
+ case HistogramSnapshot snapshot -> {
+ List<HistogramSnapshot.HistogramDataPointSnapshot> filtered =
+ filterDatapoint(
+ snapshot, requiredLabels, HistogramSnapshot.HistogramDataPointSnapshot.class);
+ if (!filtered.isEmpty()) {
+ filteredSnapshots.metricSnapshot(
+ new HistogramSnapshot(snapshot.getMetadata(), filtered));
+ }
+ }
+ case GaugeSnapshot snapshot -> {
+ List<GaugeSnapshot.GaugeDataPointSnapshot> filtered =
+ filterDatapoint(snapshot, requiredLabels, GaugeSnapshot.GaugeDataPointSnapshot.class);
+ if (!filtered.isEmpty()) {
+ filteredSnapshots.metricSnapshot(new GaugeSnapshot(snapshot.getMetadata(), filtered));
+ }
+ }
+ case InfoSnapshot ignored -> {
+ // Do nothing for InfoSnapshots. Always filter it out
+ }
+ default -> {
+ log.error("Unknown metric snapshot type {}", metricSnapshot.getClass());
+ }
+ }
+ }
+ return filteredSnapshots.build();
+ }
+
+ private <D extends DataPointSnapshot> List<D> filterDatapoint(
+ MetricSnapshot snapshot,
+ SortedMap<String, Set<String>> requiredLabels,
+ Class<D> dataPointClass) {
+ return snapshot.getDataPoints().stream()
+ .filter(dp -> requiredLabelsFilter(dp.getLabels(), requiredLabels))
+ .map(dataPointClass::cast)
+ .collect(toList());
+ }
+
+ static boolean requiredLabelsFilter(
+ Labels labels, SortedMap<String, Set<String>> requiredLabels) {
+ // Both Labels and requiredLabels are name-sorted with unique names.
+ // For each required label, scan forward through labels until we find it.
+ int labelIdx = 0;
+ requireLoop:
+ for (Map.Entry<String, Set<String>> entry : requiredLabels.entrySet()) {
+ String requiredLabelName = entry.getKey();
+ Set<String> allowedValues = entry.getValue();
+
+ // Labels are sorted, so we can continue from the last position
+ while (labelIdx < labels.size()) {
+ String labelName = labels.getName(labelIdx);
+
+ int comparison = labelName.compareTo(requiredLabelName);
+
+ if (comparison < 0) {
+ // a label before requiredLabelName; we don't care about this one
+ assert !requiredLabels.containsKey(labelName);
+ labelIdx++;
+ continue; // inspect the next input label
+ }
+ if (comparison > 0) {
+ // We've passed where requiredLabelName should be, so it doesn't exist
+ assert !labels.contains(requiredLabelName);
+ return false;
+ }
+
+ assert labels.contains(requiredLabelName);
+ if (!allowedValues.contains(labels.getValue(labelIdx))) {
+ return false;
+ }
+ // we satisfied requiredLabelName with its allowedValues. Move onto next
+ labelIdx++;
+ continue requireLoop;
+ } // end while loop on input Labels
+ // didn't find requiredLabelName; ran out of names
+ assert !labels.contains(requiredLabelName);
+ return false;
+ }
+ return true; // found all requiredLabels in Labels
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
index 070ed662dc0..a5f69e1632b 100644
--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
@@ -18,27 +18,14 @@ package org.apache.solr.response;
import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;
-import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
-import io.opentelemetry.sdk.metrics.SdkMeterProvider;
import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;
-import io.prometheus.metrics.model.snapshots.CounterSnapshot;
-import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
-import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
-import io.prometheus.metrics.model.snapshots.InfoSnapshot;
-import io.prometheus.metrics.model.snapshots.MetricSnapshot;
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
import java.io.IOException;
import java.io.OutputStream;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.handler.admin.MetricsHandler;
import org.apache.solr.request.SolrQueryRequest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */
@SuppressWarnings(value = "unchecked")
@@ -49,23 +36,17 @@ public class PrometheusResponseWriter implements QueryResponseWriter {
private static final String CONTENT_TYPE_OPEN_METRICS =
"application/openmetrics-text; version=1.0.0; charset=utf-8";
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
@Override
public void write(
OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType)
throws IOException {
- Map<String, PrometheusMetricReader> readers =
- (Map<String, PrometheusMetricReader>) response.getValues().get("metrics");
-
- List<MetricSnapshot> snapshots =
- readers.values().stream().flatMap(r -> r.collect().stream()).toList();
-
+ var metrics = response.getValues().get("metrics");
+ MetricSnapshots snapshots = (MetricSnapshots) metrics;
if (writeOpenMetricsFormat(request)) {
- new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots));
+ new OpenMetricsTextFormatWriter(false, true).write(out, snapshots);
} else {
- new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));
+ new PrometheusTextFormatWriter(false).write(out, snapshots);
}
}
@@ -92,90 +73,4 @@ public class PrometheusResponseWriter implements QueryResponseWriter {
return acceptHeader.contains("application/openmetrics-text")
&& (acceptHeader.contains("version=1.0.0"));
}
-
- /**
- * Merge a collection of individual {@link MetricSnapshot} instances into one {@link
- * MetricSnapshots}. This is necessary because we create a {@link SdkMeterProvider} per Solr core
- * resulting in duplicate metric names across cores which is an illegal format if not under the
- * same prometheus grouping.
- */
- private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {
- Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();
- Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();
- Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();
- InfoSnapshot otelInfoSnapshots = null;
-
- for (MetricSnapshot snapshot : snapshots) {
- String metricName = snapshot.getMetadata().getPrometheusName();
-
- switch (snapshot) {
- case CounterSnapshot counterSnapshot -> {
- CounterSnapshot.Builder builder =
- counterSnapshotMap.computeIfAbsent(
- metricName,
- k -> {
- var base =
- CounterSnapshot.builder()
- .name(counterSnapshot.getMetadata().getName())
- .help(counterSnapshot.getMetadata().getHelp());
- return counterSnapshot.getMetadata().hasUnit()
- ? base.unit(counterSnapshot.getMetadata().getUnit())
- : base;
- });
- counterSnapshot.getDataPoints().forEach(builder::dataPoint);
- }
- case GaugeSnapshot gaugeSnapshot -> {
- GaugeSnapshot.Builder builder =
- gaugeSnapshotMap.computeIfAbsent(
- metricName,
- k -> {
- var base =
- GaugeSnapshot.builder()
- .name(gaugeSnapshot.getMetadata().getName())
- .help(gaugeSnapshot.getMetadata().getHelp());
- return gaugeSnapshot.getMetadata().hasUnit()
- ? base.unit(gaugeSnapshot.getMetadata().getUnit())
- : base;
- });
- gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);
- }
- case HistogramSnapshot histogramSnapshot -> {
- HistogramSnapshot.Builder builder =
- histogramSnapshotMap.computeIfAbsent(
- metricName,
- k -> {
- var base =
- HistogramSnapshot.builder()
- .name(histogramSnapshot.getMetadata().getName())
- .help(histogramSnapshot.getMetadata().getHelp());
- return histogramSnapshot.getMetadata().hasUnit()
- ? base.unit(histogramSnapshot.getMetadata().getUnit())
- : base;
- });
- histogramSnapshot.getDataPoints().forEach(builder::dataPoint);
- }
- case InfoSnapshot infoSnapshot -> {
- // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate
- // metric called target_info containing OTEL SDK metadata. Only one of these need to be
- // kept
- if (otelInfoSnapshots == null)
- otelInfoSnapshots =
- new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());
- }
- default -> {
- log.warn(
- "Unexpected snapshot type: {} for metric {}",
- snapshot.getClass().getName(),
- snapshot.getMetadata().getName());
- }
- }
- }
-
- MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();
- counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
- gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
- histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));
- if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);
- return snapshotsBuilder.build();
- }
}
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0878bf2c8cd..6cb09d3d95d 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -17,33 +17,19 @@
package org.apache.solr.handler.admin;
-import com.codahale.metrics.Counter;
import io.opentelemetry.api.common.Attributes;
-import io.prometheus.metrics.model.snapshots.CounterSnapshot;
-import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
-import io.prometheus.metrics.model.snapshots.Labels;
-import io.prometheus.metrics.model.snapshots.MetricSnapshot;
import io.prometheus.metrics.model.snapshots.MetricSnapshots;
-import io.prometheus.metrics.model.snapshots.SummarySnapshot;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.solr.core.PluginBag;
-import org.apache.solr.core.PluginInfo;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricsContext;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -51,699 +37,228 @@ import org.junit.Test;
public class MetricsHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
-
initCore("solrconfig-minimal.xml", "schema.xml");
h.getCoreContainer().waitForLoadingCoresToFinish(30000);
-
- // manually register & seed some metrics in solr.jvm and solr.jetty for testing via handler
- // (use "solrtest_" prefix just in case the jvm or jetty adds a "foo" metric at some point)
- Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "solrtest_foo");
- c.inc();
- c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo");
- c.inc(2);
- // test escapes
- c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo:bar");
- c.inc(3);
-
- h.getCoreContainer()
- .getMetricManager()
- .meter(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.2xx-responses");
- h.getCoreContainer()
- .getMetricManager()
- .counter(
- null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.active-requests");
- h.getCoreContainer()
- .getMetricManager()
- .timer(null, "solr.jetty", "org.eclipse.jetty.server.handler.DefaultHandler.dispatches");
- }
-
- @AfterClass
- public static void cleanupMetrics() {
- if (null != h) {
- h.getCoreContainer().getMetricManager().registry("solr.jvm").remove("solrtest_foo");
- h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo");
- h.getCoreContainer().getMetricManager().registry("solr.jetty").remove("solrtest_foo:bar");
- }
}
- // NOCOMMIT: This test does a bunch of /admin/metrics calls with various params, with various
- // filters and parameters. We have not migrated all the metrics to otel yet or even created any
- // filters. Once that is done, we should revisit this test and assert the prometheus response.
@Test
- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")
- public void test() throws Exception {
+ public void testMetricNamesFiltering() throws Exception {
+ String expectedRequestsMetricName = "solr_core_requests";
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
CommonParams.WT,
- "json"),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.METRIC_NAME_PARAM,
+ expectedRequestsMetricName),
resp);
- NamedList<?> values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1");
- assertNotNull(nl);
- Object o = nl.get("SEARCHER.new.errors");
- assertNotNull(o); // counter type
- assertTrue(o instanceof MapWriter);
- // response wasn't serialized, so we get here whatever MetricUtils produced instead of NamedList
- assertNotNull(((MapWriter) o)._get("count"));
- assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count"));
- assertNotNull(nl.get("INDEX.segments")); // int gauge
- assertTrue((int) ((MapWriter) nl.get("INDEX.segments"))._get("value") >= 0);
- assertNotNull(nl.get("INDEX.sizeInBytes")); // long gauge
- assertTrue((long) ((MapWriter) nl.get("INDEX.sizeInBytes"))._get("value") >= 0);
- nl = (NamedList<?>) values.get("solr.node");
- assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge
- assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value"));
- assertNotNull(nl.get("ADMIN./admin/authorization.clientErrors")); // timer type
- Map<String, Object> map = new HashMap<>();
- ((MapWriter) nl.get("ADMIN./admin/authorization.clientErrors")).toMap(map);
- assertEquals(5, map.size());
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "jvm,jetty"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(2, values.size());
- assertNotNull(values.get("solr.jetty"));
- assertNotNull(values.get("solr.jvm"));
-
- resp = new SolrQueryResponse();
- // "collection" works too, because it's a prefix for "collection1"
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "registry",
- "solr.core.collection,solr.jvm"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(2, values.size());
- assertNotNull(values.get("solr.core.collection1"));
- assertNotNull(values.get("solr.jvm"));
-
- resp = new SolrQueryResponse();
- // "collection" works too, because it's a prefix for "collection1"
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "registry",
- "solr.core.collection",
- "registry",
- "solr.jvm"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(2, values.size());
- assertNotNull(values.get("solr.core.collection1"));
- assertNotNull(values.get("solr.jvm"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "jvm,jetty"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(2, values.size());
- assertNotNull(values.get("solr.jetty"));
- assertNotNull(values.get("solr.jvm"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "jvm",
- "group",
- "jetty"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(2, values.size());
- assertNotNull(values.get("solr.jetty"));
- assertNotNull(values.get("solr.jvm"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "node",
- "type",
- "counter"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(1, values.size());
- values = (NamedList<?>) values.get("solr.node");
- assertNotNull(values);
- assertNull(values.get("ADMIN./admin/authorization.errors")); // this is a timer node
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "prefix",
- "CONTAINER.cores,CONTAINER.threadPool"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(1, values.size());
- assertNotNull(values.get("solr.node"));
- values = (NamedList<?>) values.get("solr.node");
- assertEquals(15, values.size());
- assertNotNull(values.get("CONTAINER.cores.lazy")); // this is a gauge node
- assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "prefix",
- "CONTAINER.cores",
- "regex",
- "C.*thread.*completed"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertNotNull(values.get("solr.node"));
- values = (NamedList<?>) values.get("solr.node");
- assertEquals(5, values.size());
- assertNotNull(values.get("CONTAINER.threadPool.coreLoadExecutor.completed"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- "prefix",
- "CACHE.core.fieldCache",
- "property",
- "entries_count",
- MetricsHandler.COMPACT_PARAM,
- "true"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertNotNull(values.get("solr.core.collection1"));
- values = (NamedList<?>) values.get("solr.core.collection1");
- assertEquals(1, values.size());
- MapWriter writer = (MapWriter) values.get("CACHE.core.fieldCache");
- assertNotNull(writer);
- assertNotNull(writer._get("entries_count"));
-
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "jvm",
- "prefix",
- "CONTAINER.cores"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- assertEquals(0, values.size());
+ var metrics = resp.getValues().get("metrics");
+ MetricSnapshots snapshots = (MetricSnapshots) metrics;
+ assertEquals(1, snapshots.size());
+ assertEquals(expectedRequestsMetricName, snapshots.get(0).getMetadata().getPrometheusName());
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- MetricsHandler.COMPACT_PARAM,
- "false",
- CommonParams.WT,
- "json",
- "group",
- "node",
- "type",
- "timer",
- "prefix",
- "CONTAINER.cores"),
- resp);
- values = resp.getValues();
- assertNotNull(values.get("metrics"));
- SimpleOrderedMap<?> map1 = (SimpleOrderedMap<?>) values.get("metrics");
- assertEquals(0, map1.size());
handler.close();
}
@Test
- public void testPropertyFilter() throws Exception {
- assertQ(req("*:*"), "//result[@numFound='0']");
-
+ public void testMultipleMetricNamesFiltering() throws Exception {
+ String expectedRequestsMetricName = "solr_core_requests";
+ String expectedSearcherMetricName = "solr_core_searcher_new";
+ var expected = Set.of(expectedRequestsMetricName, expectedSearcherMetricName);
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.COMPACT_PARAM,
- "true",
- "group",
- "core",
- "prefix",
- "CACHE.searcher"),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.METRIC_NAME_PARAM,
+ expectedRequestsMetricName + "," + expectedSearcherMetricName),
resp);
- NamedList<?> values = resp.getValues();
- assertNotNull(values.get("metrics"));
- values = (NamedList<?>) values.get("metrics");
- NamedList<?> nl = (NamedList<?>) values.get("solr.core.collection1");
- assertNotNull(nl);
- assertTrue(nl.size() > 0);
- nl.forEach(
- (k, v) -> {
- assertTrue(v instanceof MapWriter);
- Map<String, Object> map = new HashMap<>();
- ((MapWriter) v).toMap(map);
- assertTrue(map.size() > 2);
- });
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.COMPACT_PARAM,
- "true",
- "group",
- "core",
- "prefix",
- "CACHE.searcher",
- "property",
- "inserts",
- "property",
- "size"),
- resp);
- values = resp.getValues();
- values = (NamedList<?>) values.get("metrics");
- nl = (NamedList<?>) values.get("solr.core.collection1");
- assertNotNull(nl);
- assertTrue(nl.size() > 0);
- nl.forEach(
- (k, v) -> {
- assertTrue(v instanceof MapWriter);
- Map<String, Object> map = new HashMap<>();
- ((MapWriter) v).toMap(map);
- assertEquals("k=" + k + ", v=" + map, 2, map.size());
- assertNotNull(map.get("inserts"));
- assertNotNull(map.get("size"));
- });
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+ assertEquals(2, metrics.size());
+ Set<String> actual =
+ metrics.stream().map(m -> m.getMetadata().getPrometheusName()).collect(Collectors.toSet());
+ assertEquals(expected, actual);
+
handler.close();
}
- // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet
@Test
- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")
- public void testKeyMetrics() throws Exception {
+ public void testNonExistentMetricNameFiltering() throws Exception {
+ String nonexistentMetricName = "nonexistent_metric_name";
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
- String key1 = "solr.core.collection1:CACHE.core.fieldCache";
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- key1),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.METRIC_NAME_PARAM,
+ nonexistentMetricName),
resp);
- NamedList<?> values = resp.getValues();
- Object val = values._get(List.of("metrics", key1), null);
- assertNotNull(val);
- assertTrue(val instanceof MapWriter);
- assertTrue(((MapWriter) val)._size() >= 2);
-
- String key2 = "solr.core.collection1:CACHE.core.fieldCache:entries_count";
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- key2),
- resp);
- val = resp.getValues()._get("metrics/" + key2);
- assertNotNull(val);
- assertTrue(val instanceof Number);
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+ assertEquals(0, metrics.size());
+ handler.close();
+ }
- String key3 = "solr.jetty:solrtest_foo\\:bar";
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- key3),
- resp);
+ @Test
+ public void testLabelFiltering() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
- val = resp.getValues()._get("metrics/" + key3);
- assertNotNull(val);
- assertTrue(val instanceof Number);
- assertEquals(3, ((Number) val).intValue());
+ assertQ(req("*:*"), "//result[@numFound='0']");
- // test multiple keys
- resp = new SolrQueryResponse();
+ SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- key1,
- MetricsHandler.KEY_PARAM,
- key2,
- MetricsHandler.KEY_PARAM,
- key3),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.CATEGORY_PARAM,
+ "QUERY"),
resp);
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+
+ metrics.forEach(
+ (ms) -> {
+ ms.getDataPoints()
+ .forEach(
+ (dp) -> {
+ assertEquals("QUERY", dp.getLabels().get(MetricsHandler.CATEGORY_PARAM));
+ });
+ });
- val = resp.getValues()._get("metrics/" + key1);
- assertNotNull(val);
- val = resp.getValues()._get("metrics/" + key2);
- assertNotNull(val);
- val = resp.getValues()._get("metrics/" + key3);
- assertNotNull(val);
+ handler.close();
+ }
- String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- key4),
- resp);
- // the key contains a slash, need explicit list of path elements
- val = resp.getValues()._get(Arrays.asList("metrics", key4), null);
- assertNotNull(val);
- assertTrue(val instanceof Number);
+ @Test
+ public void testMultipleLabelFiltering() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
- // test errors
+ assertQ(req("*:*"), "//result[@numFound='0']");
- // invalid keys
- resp = new SolrQueryResponse();
+ SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- "foo",
- MetricsHandler.KEY_PARAM,
- "foo:bar:baz:xyz"),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.CATEGORY_PARAM,
+ "QUERY" + "," + "SEARCHER"),
resp);
- values = resp.getValues();
- NamedList<?> metrics = (NamedList<?>) values.get("metrics");
- assertEquals(0, metrics.size());
- assertNotNull(values._get(List.of("errors", "foo"), null));
- assertNotNull(values._get(List.of("errors", "foo:bar:baz:xyz"), null));
- // unknown registry
- resp = new SolrQueryResponse();
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+ metrics.forEach(
+ (ms) -> {
+ ms.getDataPoints()
+ .forEach(
+ (dp) -> {
+ assertTrue(
+ dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("QUERY")
+ || dp.getLabels()
+ .get(MetricsHandler.CATEGORY_PARAM)
+ .equals("SEARCHER"));
+ });
+ });
+
+ handler.close();
+ }
+
+ @Test
+ public void testNonExistentLabelFiltering() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- "foo:bar:baz"),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.CORE_PARAM,
+ "nonexistent_core_name"),
resp);
- values = resp.getValues();
- metrics = (NamedList<?>) values.get("metrics");
+
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
assertEquals(0, metrics.size());
- assertNotNull(values._get(List.of("errors", "foo:bar:baz"), null));
+ handler.close();
+ }
+
+ @Test
+ public void testMixedLabelFiltering() throws Exception {
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ assertQ(req("*:*"), "//result[@numFound='0']");
- // unknown metric
- resp = new SolrQueryResponse();
+ SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.KEY_PARAM,
- "solr.jetty:unknown:baz"),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.CORE_PARAM,
+ "collection1",
+ MetricsHandler.CATEGORY_PARAM,
+ "SEARCHER"),
resp);
- values = resp.getValues();
- metrics = (NamedList<?>) values.get("metrics");
- assertEquals(0, metrics.size());
- assertNotNull(values._get(List.of("errors", "solr.jetty:unknown:baz"), null));
+
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+ metrics.forEach(
+ (ms) -> {
+ ms.getDataPoints()
+ .forEach(
+ (dp) -> {
+ assertTrue(
+ dp.getLabels().get(MetricsHandler.CATEGORY_PARAM).equals("SEARCHER")
+ && dp.getLabels().get(MetricsHandler.CORE_PARAM).equals("collection1"));
+ });
+ });
handler.close();
}
- // NOCOMMIT: Have not implemented any kind of filtering for OTEL yet
@Test
- @BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17458")
- @SuppressWarnings("unchecked")
- public void testExprMetrics() throws Exception {
+ public void testMetricNamesAndLabelFiltering() throws Exception {
+ String expectedMetricName = "solr_core_segments";
MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
- String key1 = "solr\\.core\\..*:.*/select\\.request.*:.*Rate";
+ assertQ(req("*:*"), "//result[@numFound='0']");
+
SolrQueryResponse resp = new SolrQueryResponse();
handler.handleRequestBody(
req(
CommonParams.QT,
"/admin/metrics",
CommonParams.WT,
- "json",
- MetricsHandler.EXPR_PARAM,
- key1),
- resp);
- // response structure is like in the case of non-key params
- Object val =
- resp.getValues()
- ._get(List.of("metrics", "solr.core.collection1", "QUERY./select.requestTimes"), null);
- assertNotNull(val);
- assertTrue(val instanceof MapWriter);
- Map<String, Object> map = new HashMap<>();
- ((MapWriter) val).toMap(map);
- assertEquals(map.toString(), 4, map.size()); // mean, 1, 5, 15
- assertNotNull(map.toString(), map.get("meanRate"));
- assertNotNull(map.toString(), map.get("1minRate"));
- assertNotNull(map.toString(), map.get("5minRate"));
- assertNotNull(map.toString(), map.get("15minRate"));
- assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);
- map.clear();
-
- String key2 = "solr\\.core\\..*:.*/select\\.request.*";
- resp = new SolrQueryResponse();
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.EXPR_PARAM,
- key2),
+ MetricsHandler.PROMETHEUS_METRICS_WT,
+ MetricsHandler.CATEGORY_PARAM,
+ "CORE",
+ MetricsHandler.METRIC_NAME_PARAM,
+ expectedMetricName),
resp);
- // response structure is like in the case of non-key params
- val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null);
- assertNotNull(val);
- Object v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requestTimes");
- assertNotNull(v);
- assertTrue(v instanceof MapWriter);
- ((MapWriter) v).toMap(map);
- assertEquals(map.toString(), 14, map.size());
- assertNotNull(map.toString(), map.get("1minRate"));
- assertEquals(map.toString(), ((Number) map.get("1minRate")).doubleValue(), 0.0, 0.0);
- map.clear();
- // select requests counter
- v = ((SimpleOrderedMap<Object>) val).get("QUERY./select.requests");
- assertNotNull(v);
- assertTrue(v instanceof Number);
-
- // test multiple expressions producing overlapping metrics - should be no dupes
-
- // this key matches also sub-metrics of /select, eg. /select[shard], ...
- String key3 = "solr\\.core\\..*:.*/select.*\\.requestTimes:count";
- resp = new SolrQueryResponse();
- // ORDER OF PARAMS MATTERS HERE! see the refguide
- handler.handleRequestBody(
- req(
- CommonParams.QT,
- "/admin/metrics",
- CommonParams.WT,
- "json",
- MetricsHandler.EXPR_PARAM,
- key2,
- MetricsHandler.EXPR_PARAM,
- key1,
- MetricsHandler.EXPR_PARAM,
- key3),
- resp);
- val = resp.getValues()._get(List.of("metrics", "solr.core.collection1"), null);
- assertNotNull(val);
- // for requestTimes only the full set of values from the first expr should be present
- assertNotNull(val);
- SimpleOrderedMap<Object> values = (SimpleOrderedMap<Object>) val;
- assertEquals(values.jsonStr(), 3, values.size());
- v = values.get("QUERY./select.requestTimes");
- assertTrue(v instanceof MapWriter);
- ((MapWriter) v).toMap(map);
- assertTrue(map.toString(), map.containsKey("count"));
- map.clear();
- v = values.get("QUERY./select[shard].requestTimes");
- assertTrue(v instanceof MapWriter);
- ((MapWriter) v).toMap(map);
- assertTrue(map.toString(), map.containsKey("count"));
- }
- private MetricSnapshot getMetricSnapshot(MetricSnapshots snapshots, String metricName) {
- return snapshots.stream()
- .filter(ss -> ss.getMetadata().getPrometheusName().equals(metricName))
- .findAny()
- .get();
- }
-
- private GaugeSnapshot.GaugeDataPointSnapshot getGaugeDatapointSnapshot(
- MetricSnapshot snapshot, Labels labels) {
- return (GaugeSnapshot.GaugeDataPointSnapshot)
- snapshot.getDataPoints().stream()
- .filter(ss -> ss.getLabels().hasSameValues(labels))
- .findAny()
- .get();
- }
-
- private CounterSnapshot.CounterDataPointSnapshot getCounterDatapointSnapshot(
- MetricSnapshot snapshot, Labels labels) {
- return (CounterSnapshot.CounterDataPointSnapshot)
- snapshot.getDataPoints().stream()
- .filter(ss -> ss.getLabels().hasSameValues(labels))
- .findAny()
- .get();
- }
-
- private SummarySnapshot.SummaryDataPointSnapshot getSummaryDataPointSnapshot(
- MetricSnapshot snapshot, Labels labels) {
- return (SummarySnapshot.SummaryDataPointSnapshot)
- snapshot.getDataPoints().stream()
- .filter(ss -> ss.getLabels().hasSameValues(labels))
- .findAny()
- .get();
- }
-
- static class RefreshablePluginHolder extends PluginBag.PluginHolder<SolrRequestHandler> {
-
- private DumpRequestHandler rh;
- private SolrMetricsContext metricsInfo;
-
- public RefreshablePluginHolder(PluginInfo info, DumpRequestHandler rh) {
- super(info);
- this.rh = rh;
- }
-
- @Override
- public boolean isLoaded() {
- return true;
- }
-
- void closeHandler() throws Exception {
- this.metricsInfo = rh.getSolrMetricsContext();
- // if(metricsInfo.tag.contains(String.valueOf(rh.hashCode()))){
- // //this created a new child metrics
- // metricsInfo = metricsInfo.getParent();
- // }
- this.rh.close();
- }
-
- void reset(DumpRequestHandler rh) {
- this.rh = rh;
- if (metricsInfo != null)
- this.rh.initializeMetrics(metricsInfo, Attributes.empty(), "/dumphandler");
- }
-
- @Override
- public SolrRequestHandler get() {
- return rh;
- }
+ var metrics = (MetricSnapshots) resp.getValues().get("metrics");
+ assertEquals(1, metrics.size());
+ var actualDatapoint = metrics.get(0).getDataPoints().getFirst();
+ assertEquals(expectedMetricName, metrics.get(0).getMetadata().getPrometheusName());
+ assertEquals("CORE", actualDatapoint.getLabels().get(MetricsHandler.CATEGORY_PARAM));
+ handler.close();
}
public static class DumpRequestHandler extends RequestHandlerBase {
diff --git a/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java
new file mode 100644
index 00000000000..ec91e149b4b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/metrics/otel/FilterablePrometheusMetricReaderTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel;
+
+import io.prometheus.metrics.model.snapshots.Labels;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+/**
+ * Test class for FilterablePrometheusMetricReader which focuses on requiredLabelsFilter method
+ * which filters metric data points based on label requirements.
+ */
+public class FilterablePrometheusMetricReaderTest extends SolrTestCaseJ4 {
+
+ Labels actualLabels = Labels.of("key1", "value1", "key2", "value2", "key3", "value3");
+
+ @Test
+ public void testFilterMatchingLabel() {
+ SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of("value1")));
+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterOneMatchingLabelValue() {
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(Map.of("key1", Set.of("value1", "value123", "value456", "value789")));
+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterNoMatchingLabel() {
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(Map.of("dummyKey", Set.of("dummyValue")));
+ assertFalse(
+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterAllMultipleMatchingLabels() {
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value2")));
+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterMultipleWithOneLabelValueNotMatching() {
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key2", Set.of("value999")));
+ assertFalse(
+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterMultipleWithOneLabelMissing() {
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(Map.of("key1", Set.of("value1"), "key999", Set.of("value2")));
+ assertFalse(
+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterLabelsWithMixedValues() {
+ // Scenario with multiple label values some matching and some not
+ SortedMap<String, Set<String>> requiredLabels =
+ new TreeMap<>(
+ Map.of("key1", Set.of("value1", "value123"), "key2", Set.of("value456", "value2")));
+ assertTrue(FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+
+ @Test
+ public void testFilterEmptyLabelValues() {
+ // Edge case where label has empty set of allowed values
+ SortedMap<String, Set<String>> requiredLabels = new TreeMap<>(Map.of("key1", Set.of()));
+ assertFalse(
+ FilterablePrometheusMetricReader.requiredLabelsFilter(actualLabels, requiredLabels));
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
index 09aa0489949..ecfd83c02d4 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrIndexMetricsTest.java
@@ -102,7 +102,7 @@ public class SolrIndexMetricsTest extends SolrTestCaseJ4 {
var segmentSize =
SolrMetricTestUtils.getGaugeDatapoint(
core,
- "solr_core_segment_count",
+ "solr_core_segments",
SolrMetricTestUtils.newStandaloneLabelsBuilder(core)
.label("category", "CORE")
.build());