Apache Solr open-source search software
SOLR-17793: Create dedicated SdkMeterProviders for Solr core metrics (#3402)
Matthew Biscocho50af82fbbc12
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 75e1d43d60b..02b63fe0c16 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -33,7 +33,6 @@ import com.github.benmanes.caffeine.cache.Interner;import com.google.common.annotations.VisibleForTesting;import io.opentelemetry.api.common.Attributes;import io.opentelemetry.api.trace.Tracer;-import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;import jakarta.inject.Singleton;import java.io.IOException;import java.lang.invoke.MethodHandles;@@ -404,9 +403,8 @@ public class CoreContainer {this.solrHome = config.getSolrHome();this.solrCores = SolrCores.newSolrCores(this);this.nodeKeyPair = new SolrNodeKeyPair(cfg.getCloudConfig());- PrometheusMetricReader metricReader = new PrometheusMetricReader(true, null);- OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader, metricReader);- this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig(), metricReader);+ OpenTelemetryConfigurator.initializeOpenTelemetrySdk(cfg, loader);+ this.metricManager = new SolrMetricManager(loader, cfg.getMetricsConfig());this.tracer = TraceUtils.getGlobalTracer();containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(nodeKeyPair));diff --git a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.javaindex 47547b4950f..1be4b0deb6b 100644--- a/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java+++ b/solr/core/src/java/org/apache/solr/core/OpenTelemetryConfigurator.java@@ -19,16 +19,12 @@ package org.apache.solr.core;import com.google.common.annotations.VisibleForTesting;import io.opentelemetry.api.GlobalOpenTelemetry;+import io.opentelemetry.api.OpenTelemetry;import io.opentelemetry.api.trace.Tracer;import io.opentelemetry.context.Context;import io.opentelemetry.context.Scope;import io.opentelemetry.context.propagation.ContextPropagators;import io.opentelemetry.sdk.OpenTelemetrySdk;-import io.opentelemetry.sdk.OpenTelemetrySdkBuilder;-import io.opentelemetry.sdk.metrics.SdkMeterProvider;-import io.opentelemetry.sdk.metrics.export.MetricReader;-import io.opentelemetry.sdk.trace.SdkTracerProvider;-import io.opentelemetry.sdk.trace.samplers.Sampler;import java.lang.invoke.MethodHandles;import java.util.Locale;import java.util.Map;@@ -62,7 +58,7 @@ public abstract class OpenTelemetryConfigurator implements NamedListInitializedP* SDK.*/public static synchronized void initializeOpenTelemetrySdk(- NodeConfig cfg, SolrResourceLoader loader, MetricReader metricReader) {+ NodeConfig cfg, SolrResourceLoader loader) {PluginInfo info = (cfg != null) ? cfg.getTracerConfiguratorPluginInfo() : null;if (info != null && info.isEnabled()) {@@ -71,28 +67,21 @@ public abstract class OpenTelemetryConfigurator implements NamedListInitializedP} else if (OpenTelemetryConfigurator.shouldAutoConfigOTEL()) {OpenTelemetryConfigurator.autoConfigureOpenTelemetrySdk(loader);} else {- // Initializing sampler as always off to replicate no-op Tracer provider- OpenTelemetryConfigurator.configureOpenTelemetrySdk(- SdkMeterProvider.builder().registerMetricReader(metricReader).build(),- SdkTracerProvider.builder().setSampler(Sampler.alwaysOff()).build());+ OpenTelemetryConfigurator.configureOpenTelemetrySdk();}}- private static void configureOpenTelemetrySdk(- SdkMeterProvider sdkMeterProvider, SdkTracerProvider sdkTracerProvider) {+ private static void configureOpenTelemetrySdk() {if (loaded) return;- OpenTelemetrySdkBuilder builder = OpenTelemetrySdk.builder();if (TRACE_ID_GEN_ENABLED) {log.info("OpenTelemetry tracer enabled with simple propagation only.");ExecutorUtil.addThreadLocalProvider(new ContextThreadLocalProvider());- builder.setPropagators(ContextPropagators.create(SimplePropagator.getInstance()));}- if (sdkMeterProvider != null) builder.setMeterProvider(sdkMeterProvider);- if (sdkTracerProvider != null) builder.setTracerProvider(sdkTracerProvider);-- GlobalOpenTelemetry.set(builder.build());+ OpenTelemetry otel =+ OpenTelemetry.propagating(ContextPropagators.create(SimplePropagator.getInstance()));+ GlobalOpenTelemetry.set(otel);loaded = true;}diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.javaindex e633a91113d..6038db647a3 100644--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java@@ -192,7 +192,7 @@ public abstract class RequestHandlerBasenew HandlerMetrics(new SolrMetricsContext(new SolrMetricManager(- null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build(), null),+ null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build()),"NO_OP","NO_OP"),Attributes.empty());diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex f2b501b8a41..ba7f6bba34b 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -128,7 +128,7 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName// TODO SOLR-17458: Make this the default option after dropwizard removalif (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) {- consumer.accept("metrics", metricManager.getMetricReader().collect());+ consumer.accept("metrics", metricManager.getPrometheusMetricReaders());return;}diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.javaindex 62a342180f2..9d79ada8a85 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrCoreMetricManager.java@@ -108,6 +108,7 @@ public class SolrCoreMetricManager implements Closeable {* and will be used under the new core name. This method also reloads reporters so that they use* the new core name.*/+ // NOCOMMIT SOLR-17458: Update for core renamingpublic void afterCoreRename() {assert core.getCoreDescriptor().getCloudDescriptor() == null;String oldRegistryName = solrMetricsContext.getRegistryName();diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.javaindex bcefc0b3ddc..7056a457861 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java@@ -42,8 +42,8 @@ import io.opentelemetry.api.metrics.LongHistogram;import io.opentelemetry.api.metrics.LongHistogramBuilder;import io.opentelemetry.api.metrics.LongUpDownCounter;import io.opentelemetry.api.metrics.LongUpDownCounterBuilder;-import io.opentelemetry.api.metrics.MeterProvider;import io.opentelemetry.api.metrics.ObservableDoubleCounter;+import io.opentelemetry.api.metrics.ObservableDoubleGauge;import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;import io.opentelemetry.api.metrics.ObservableDoubleUpDownCounter;import io.opentelemetry.api.metrics.ObservableLongCounter;@@ -51,6 +51,7 @@ import io.opentelemetry.api.metrics.ObservableLongGauge;import io.opentelemetry.api.metrics.ObservableLongMeasurement;import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;+import io.opentelemetry.sdk.metrics.SdkMeterProvider;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.ArrayList;@@ -81,7 +82,6 @@ import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrInfoBean;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.logging.MDCLoggingContext;-import org.apache.solr.util.stats.MetricUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.slf4j.MDC;@@ -115,6 +115,8 @@ public class SolrMetricManager {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ public static final String OTEL_SCOPE_NAME = "org.apache.solr";+/** Common prefix for all registry names that Solr uses. */public static final String REGISTRY_NAME_PREFIX = "solr.";@@ -147,9 +149,8 @@ public class SolrMetricManager {private final MetricRegistry.MetricSupplier<Timer> timerSupplier;private final MetricRegistry.MetricSupplier<Histogram> histogramSupplier;- private final MeterProvider meterProvider;- private final Map<String, io.opentelemetry.api.metrics.Meter> meters = new ConcurrentHashMap<>();- private final PrometheusMetricReader metricReader;+ private final ConcurrentMap<String, MeterProviderAndReaders> meterProviderAndReaders =+ new ConcurrentHashMap<>();public SolrMetricManager() {metricsConfig = new MetricsConfig.MetricsConfigBuilder().build();@@ -157,15 +158,10 @@ public class SolrMetricManager {meterSupplier = MetricSuppliers.meterSupplier(null, null);timerSupplier = MetricSuppliers.timerSupplier(null, null);histogramSupplier = MetricSuppliers.histogramSupplier(null, null);- meterProvider = MetricUtils.getMeterProvider();- metricReader = null;}- public SolrMetricManager(- SolrResourceLoader loader, MetricsConfig metricsConfig, PrometheusMetricReader metricReader) {+ public SolrMetricManager(SolrResourceLoader loader, MetricsConfig metricsConfig) {this.metricsConfig = metricsConfig;- this.metricReader = metricReader;- this.meterProvider = MetricUtils.getMeterProvider();counterSupplier = MetricSuppliers.counterSupplier(loader, metricsConfig.getCounterSupplier());meterSupplier = MetricSuppliers.meterSupplier(loader, metricsConfig.getMeterSupplier());timerSupplier = MetricSuppliers.timerSupplier(loader, metricsConfig.getTimerSupplier());@@ -176,7 +172,10 @@ public class SolrMetricManager {public LongCounter longCounter(String registry, String counterName, String description, String unit) {LongCounterBuilder builder =- meterProvider.get(registry).counterBuilder(counterName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .counterBuilder(counterName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.build();@@ -185,7 +184,10 @@ public class SolrMetricManager {public LongUpDownCounter longUpDownCounter(String registry, String counterName, String description, String unit) {LongUpDownCounterBuilder builder =- meterProvider.get(registry).upDownCounterBuilder(counterName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .upDownCounterBuilder(counterName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.build();@@ -194,8 +196,8 @@ public class SolrMetricManager {public DoubleUpDownCounter doubleUpDownCounter(String registry, String counterName, String description, String unit) {DoubleUpDownCounterBuilder builder =- meterProvider- .get(registry)+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME).upDownCounterBuilder(counterName).setDescription(description).ofDoubles();@@ -207,8 +209,8 @@ public class SolrMetricManager {public DoubleCounter doubleCounter(String registry, String counterName, String description, String unit) {DoubleCounterBuilder builder =- meterProvider- .get(registry)+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME).counterBuilder(counterName).setDescription(description).ofDoubles();@@ -220,7 +222,10 @@ public class SolrMetricManager {public DoubleHistogram doubleHistogram(String registry, String histogramName, String description, String unit) {DoubleHistogramBuilder builder =- meterProvider.get(registry).histogramBuilder(histogramName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .histogramBuilder(histogramName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.build();@@ -229,8 +234,8 @@ public class SolrMetricManager {public LongHistogram longHistogram(String registry, String histogramName, String description, String unit) {LongHistogramBuilder builder =- meterProvider- .get(registry)+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME).histogramBuilder(histogramName).setDescription(description).ofLongs();@@ -243,7 +248,10 @@ public class SolrMetricManager {public DoubleGauge doubleGauge(String registry, String gaugeName, String description, String unit) {DoubleGaugeBuilder builder =- meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .gaugeBuilder(gaugeName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.build();@@ -251,7 +259,11 @@ public class SolrMetricManager {public LongGauge longGauge(String registry, String gaugeName, String description, String unit) {LongGaugeBuilder builder =- meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description).ofLongs();+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .gaugeBuilder(gaugeName)+ .setDescription(description)+ .ofLongs();if (unit != null) builder.setUnit(unit);return builder.build();@@ -264,7 +276,10 @@ public class SolrMetricManager {Consumer<ObservableLongMeasurement> callback,String unit) {LongCounterBuilder builder =- meterProvider.get(registry).counterBuilder(counterName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .counterBuilder(counterName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.buildWithCallback(callback);@@ -277,8 +292,8 @@ public class SolrMetricManager {Consumer<ObservableDoubleMeasurement> callback,String unit) {DoubleCounterBuilder builder =- meterProvider- .get(registry)+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME).counterBuilder(counterName).setDescription(description).ofDoubles();@@ -294,7 +309,28 @@ public class SolrMetricManager {Consumer<ObservableLongMeasurement> callback,String unit) {LongGaugeBuilder builder =- meterProvider.get(registry).gaugeBuilder(gaugeName).setDescription(description).ofLongs();+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .gaugeBuilder(gaugeName)+ .setDescription(description)+ .ofLongs();+ if (unit != null) builder.setUnit(unit);++ return builder.buildWithCallback(callback);+ }++ public ObservableDoubleGauge observableDoubleGauge(+ String registry,+ String gaugeName,+ String description,+ Consumer<ObservableDoubleMeasurement> callback,+ String unit) {+ DoubleGaugeBuilder builder =+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .gaugeBuilder(gaugeName)+ .setDescription(description);+if (unit != null) builder.setUnit(unit);return builder.buildWithCallback(callback);@@ -307,7 +343,10 @@ public class SolrMetricManager {Consumer<ObservableLongMeasurement> callback,String unit) {LongUpDownCounterBuilder builder =- meterProvider.get(registry).upDownCounterBuilder(counterName).setDescription(description);+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME)+ .upDownCounterBuilder(counterName)+ .setDescription(description);if (unit != null) builder.setUnit(unit);return builder.buildWithCallback(callback);@@ -320,8 +359,8 @@ public class SolrMetricManager {Consumer<ObservableDoubleMeasurement> callback,String unit) {DoubleUpDownCounterBuilder builder =- meterProvider- .get(registry)+ meterProvider(registry)+ .get(OTEL_SCOPE_NAME).upDownCounterBuilder(counterName).setDescription(description).ofDoubles();@@ -330,10 +369,6 @@ public class SolrMetricManager {return builder.buildWithCallback(callback);}- public io.opentelemetry.api.metrics.Meter getOtelRegistry(String registryName) {- return meters.get(registryName);- }-// for unit testspublic MetricRegistry.MetricSupplier<Counter> getCounterSupplier() {return counterSupplier;@@ -578,6 +613,7 @@ public class SolrMetricManager {}/** Return a set of existing registry names. */+ // NOCOMMIT: Remove for OTELpublic Set<String> registryNames() {Set<String> set = new HashSet<>();set.addAll(registries.keySet());@@ -598,6 +634,10 @@ public class SolrMetricManager {return names.contains(name);}+ public boolean hasMeterProvider(String name) {+ return meterProviderAndReaders.containsKey(enforcePrefix(name));+ }+/*** Return set of existing registry names that match a regex pattern*@@ -654,7 +694,7 @@ public class SolrMetricManager {* @param registry name of the registry* @return existing or newly created registry*/- // TODO SOLR-17458: We may not need+ // NOCOMMIT SOLR-17458: We may not needpublic MetricRegistry registry(String registry) {return registry(registry, true);}@@ -684,6 +724,29 @@ public class SolrMetricManager {}}+ /**+ * Get (or create if not present) a named {@link SdkMeterProvider} under {@link+ * MeterProviderAndReaders}. This also registers a corresponding {@link PrometheusMetricReader}+ * Dropwizards's {@link SolrMetricManager#registry(String)} equivalent+ *+ * @param providerName name of the meter provider and prometheus metric reader+ * @return existing or newly created meter provider+ */+ public SdkMeterProvider meterProvider(String providerName) {+ providerName = enforcePrefix(providerName);+ return meterProviderAndReaders+ .computeIfAbsent(+ providerName,+ key -> {+ var reader = new PrometheusMetricReader(true, null);+ // NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP+ // with an exporter+ var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();+ return new MeterProviderAndReaders(provider, reader);+ })+ .sdkMeterProvider();+ }+// TODO SOLR-17458: We may not needprivate static MetricRegistry getOrCreateRegistry(ConcurrentMap<String, MetricRegistry> map, String registry) {@@ -702,12 +765,14 @@ public class SolrMetricManager {}/**- * Remove a named registry.+ * Remove a named registry and close an existing {@link SdkMeterProvider}. Upon closing of+ * provider, all metric readers registered to it are closed.** @param registry name of the registry to remove*/// TODO SOLR-17458: You can't delete OTEL meterspublic void removeRegistry(String registry) {+ // NOCOMMIT Remove all closing Dropwizard registries// close any reporters for this registry firstcloseReporters(registry, null);// make sure we use a name with prefix@@ -722,6 +787,12 @@ public class SolrMetricManager {swapLock.unlock();}}+ meterProviderAndReaders.computeIfPresent(+ registry,+ (key, meterAndReader) -> {+ meterAndReader.sdkMeterProvider().close();+ return null;+ });}/**@@ -827,8 +898,9 @@ public class SolrMetricManager {* start with the prefix will be removed.* @return set of metrics names that have been removed.*/- // TODO SOLR-17458: This is not supported in otel. Metrics are immutable. We can at best filter- // them+ // NOCOMMIT SOLR-17458: This is not supported in otel. Metrics are immutable. We can at best+ // filter+ // them or delete the meterProvider entirelypublic Set<String> clearMetrics(String registry, String... metricPath) {PrefixFilter filter;if (metricPath == null || metricPath.length == 0) {@@ -1603,7 +1675,17 @@ public class SolrMetricManager {return metricsConfig;}- public PrometheusMetricReader getMetricReader() {- return this.metricReader;+ /** Get a shallow copied map of {@link PrometheusMetricReader}. */+ public Map<String, PrometheusMetricReader> getPrometheusMetricReaders() {+ return meterProviderAndReaders.entrySet().stream()+ .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().prometheusMetricReader()));+ }++ public PrometheusMetricReader getPrometheusMetricReader(String providerName) {+ MeterProviderAndReaders mpr = meterProviderAndReaders.get(enforcePrefix(providerName));+ return (mpr != null) ? mpr.prometheusMetricReader() : null;}++ private record MeterProviderAndReaders(+ SdkMeterProvider sdkMeterProvider, PrometheusMetricReader prometheusMetricReader) {}}diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.javaindex 269575798ec..222ce2e0e8a 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricsContext.java@@ -30,6 +30,8 @@ import io.opentelemetry.api.metrics.LongCounter;import io.opentelemetry.api.metrics.LongGauge;import io.opentelemetry.api.metrics.LongHistogram;import io.opentelemetry.api.metrics.LongUpDownCounter;+import io.opentelemetry.api.metrics.ObservableDoubleGauge;+import io.opentelemetry.api.metrics.ObservableDoubleMeasurement;import io.opentelemetry.api.metrics.ObservableLongGauge;import io.opentelemetry.api.metrics.ObservableLongMeasurement;import java.util.Map;@@ -206,17 +208,31 @@ public class SolrMetricsContext {public ObservableLongGauge observableLongGauge(String metricName, String description, Consumer<ObservableLongMeasurement> callback) {- return metricManager.observableLongGauge(registryName, metricName, description, callback, null);+ return observableLongGauge(metricName, description, callback, null);}public ObservableLongGauge observableLongGauge(String metricName,String description,- String unit,- Consumer<ObservableLongMeasurement> callback) {+ Consumer<ObservableLongMeasurement> callback,+ String unit) {return metricManager.observableLongGauge(registryName, metricName, description, callback, unit);}+ public ObservableDoubleGauge observableDoubleGauge(+ String metricName, String description, Consumer<ObservableDoubleMeasurement> callback) {+ return observableDoubleGauge(metricName, description, callback, null);+ }++ public ObservableDoubleGauge observableDoubleGauge(+ String metricName,+ String description,+ Consumer<ObservableDoubleMeasurement> callback,+ String unit) {+ return metricManager.observableDoubleGauge(+ registryName, metricName, description, callback, unit);+ }+/*** Convenience method for {@link SolrMetricManager#meter(SolrMetricsContext, String, String,* String...)}.diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.javaindex 5a375bdb408..77fd3642b2a 100644--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java@@ -16,19 +16,27 @@*/package org.apache.solr.response;+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;+import io.opentelemetry.sdk.metrics.SdkMeterProvider;import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;+import io.prometheus.metrics.model.snapshots.InfoSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshot;import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.io.IOException;import java.io.OutputStream;import java.lang.invoke.MethodHandles;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import org.apache.solr.handler.admin.MetricsHandler;import org.apache.solr.request.SolrQueryRequest;import org.slf4j.Logger;import org.slf4j.LoggerFactory;-/**- * Response writer for Prometheus metrics. This is used only by the {@link- * org.apache.solr.handler.admin.MetricsHandler}- */+/** Response writer for Prometheus metrics. This is used only by the {@link MetricsHandler} */@SuppressWarnings(value = "unchecked")public class PrometheusResponseWriter implements QueryResponseWriter {// not TextQueryResponseWriter because Prometheus libs work with an OutputStream@@ -40,12 +48,104 @@ public class PrometheusResponseWriter implements QueryResponseWriter {public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse response, String contentType)throws IOException {- var prometheusTextFormatWriter = new PrometheusTextFormatWriter(false);- prometheusTextFormatWriter.write(out, (MetricSnapshots) response.getValues().get("metrics"));++ Map<String, PrometheusMetricReader> readers =+ (Map<String, PrometheusMetricReader>) response.getValues().get("metrics");++ List<MetricSnapshot> snapshots =+ readers.values().stream().flatMap(r -> r.collect().stream()).toList();++ new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));}@Overridepublic String getContentType(SolrQueryRequest request, SolrQueryResponse response) {return CONTENT_TYPE_PROMETHEUS;}++ /**+ * Merge a collection of individual {@link MetricSnapshot} instances into one {@link+ * MetricSnapshots}. This is necessary because we create a {@link SdkMeterProvider} per Solr core+ * resulting in duplicate metric names across cores which is an illegal format if not under the+ * same prometheus grouping.+ */+ private MetricSnapshots mergeSnapshots(List<MetricSnapshot> snapshots) {+ Map<String, CounterSnapshot.Builder> counterSnapshotMap = new HashMap<>();+ Map<String, GaugeSnapshot.Builder> gaugeSnapshotMap = new HashMap<>();+ Map<String, HistogramSnapshot.Builder> histogramSnapshotMap = new HashMap<>();+ InfoSnapshot otelInfoSnapshots = null;++ for (MetricSnapshot snapshot : snapshots) {+ String metricName = snapshot.getMetadata().getPrometheusName();++ switch (snapshot) {+ case CounterSnapshot counterSnapshot -> {+ CounterSnapshot.Builder builder =+ counterSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ CounterSnapshot.builder()+ .name(counterSnapshot.getMetadata().getName())+ .help(counterSnapshot.getMetadata().getHelp());+ return counterSnapshot.getMetadata().hasUnit()+ ? base.unit(counterSnapshot.getMetadata().getUnit())+ : base;+ });+ counterSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case GaugeSnapshot gaugeSnapshot -> {+ GaugeSnapshot.Builder builder =+ gaugeSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ GaugeSnapshot.builder()+ .name(gaugeSnapshot.getMetadata().getName())+ .help(gaugeSnapshot.getMetadata().getHelp());+ return gaugeSnapshot.getMetadata().hasUnit()+ ? base.unit(gaugeSnapshot.getMetadata().getUnit())+ : base;+ });+ gaugeSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case HistogramSnapshot histogramSnapshot -> {+ HistogramSnapshot.Builder builder =+ histogramSnapshotMap.computeIfAbsent(+ metricName,+ k -> {+ var base =+ HistogramSnapshot.builder()+ .name(histogramSnapshot.getMetadata().getName())+ .help(histogramSnapshot.getMetadata().getHelp());+ return histogramSnapshot.getMetadata().hasUnit()+ ? base.unit(histogramSnapshot.getMetadata().getUnit())+ : base;+ });+ histogramSnapshot.getDataPoints().forEach(builder::dataPoint);+ }+ case InfoSnapshot infoSnapshot -> {+ // InfoSnapshot is a special case in that each SdkMeterProvider will create a duplicate+ // metric called target_info containing OTEL SDK metadata. Only one of these need to be+ // kept+ if (otelInfoSnapshots == null)+ otelInfoSnapshots =+ new InfoSnapshot(infoSnapshot.getMetadata(), infoSnapshot.getDataPoints());+ }+ default -> {+ log.warn(+ "Unexpected snapshot type: {} for metric {}",+ snapshot.getClass().getName(),+ snapshot.getMetadata().getName());+ }+ }+ }++ MetricSnapshots.Builder snapshotsBuilder = MetricSnapshots.builder();+ counterSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ gaugeSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ histogramSnapshotMap.values().forEach(b -> snapshotsBuilder.metricSnapshot(b.build()));+ if (otelInfoSnapshots != null) snapshotsBuilder.metricSnapshot(otelInfoSnapshots);+ return snapshotsBuilder.build();+ }}diff --git a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.javaindex 8b79fc26b35..9cff065c06f 100644--- a/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java+++ b/solr/core/src/java/org/apache/solr/util/stats/MetricUtils.java@@ -26,10 +26,6 @@ import com.codahale.metrics.MetricFilter;import com.codahale.metrics.MetricRegistry;import com.codahale.metrics.Snapshot;import com.codahale.metrics.Timer;-import io.opentelemetry.api.GlobalOpenTelemetry;-import io.opentelemetry.api.common.Attributes;-import io.opentelemetry.api.common.AttributesBuilder;-import io.opentelemetry.api.metrics.MeterProvider;import java.beans.BeanInfo;import java.beans.IntrospectionException;import java.beans.Introspector;@@ -55,7 +51,6 @@ import java.util.function.Predicate;import org.apache.solr.common.ConditionalKeyMapWriter;import org.apache.solr.common.IteratorWriter;import org.apache.solr.common.MapWriter;-import org.apache.solr.common.SolrException;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.SolrInfoBean;@@ -857,24 +852,4 @@ public class MetricUtils {}}}-- public static MeterProvider getMeterProvider() {- return GlobalOpenTelemetry.getMeterProvider();- }-- public static Attributes createAttributes(String... attributes) {- if (attributes.length % 2 == 1) {- throw new SolrException(- SolrException.ErrorCode.SERVER_ERROR, "Odd number of field/value strings passed");- }-- AttributesBuilder builder = Attributes.builder();- for (int i = 0; i < attributes.length; i += 2) {- String key = attributes[i];- String value = attributes[i + 1];- builder.put(key, value);- }-- return builder.build();- }}diff --git a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.javaindex ba453efc5d4..5aa8ffec1ed 100644--- a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java+++ b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java@@ -41,7 +41,7 @@ public class TestTracerConfigurator extends SolrTestCaseJ4 {public void configuratorClassDoesNotExistTest() {assertTrue(OpenTelemetryConfigurator.shouldAutoConfigOTEL());SolrResourceLoader loader = new SolrResourceLoader(TEST_PATH().resolve("collection1"));- OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, loader, null);+ OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, loader);assertEquals("Expecting noop otel after failure to auto-init",TracerProvider.noop().get(null),diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.javaindex 96001e15b63..07f71176591 100644--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java@@ -48,6 +48,7 @@ import org.junit.BeforeClass;import org.junit.Test;/** Test for {@link MetricsHandler} */+// NOCOMMIT SOLR-17785: Lets move this to SolrCloudTestCasepublic class MetricsHandlerTest extends SolrTestCaseJ4 {@BeforeClasspublic static void beforeClass() throws Exception {diff --git a/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java b/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.javaindex 32245ffb4c6..eff63f4db72 100644--- a/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java+++ b/solr/core/src/test/org/apache/solr/metrics/MetricsConfigTest.java@@ -35,7 +35,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {public void testDefaults() {NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml");SolrMetricManager mgr =- new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null);+ new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.DefaultCounterSupplier);assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.DefaultMeterSupplier);assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.DefaultTimerSupplier);@@ -54,7 +54,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {System.setProperty("histogram.reservoir", SlidingTimeWindowReservoir.class.getName());NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml");SolrMetricManager mgr =- new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null);+ new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.DefaultCounterSupplier);assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.DefaultMeterSupplier);assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.DefaultTimerSupplier);@@ -73,7 +73,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {System.setProperty("histogram.class", MockHistogramSupplier.class.getName());NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml");SolrMetricManager mgr =- new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null);+ new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());assertTrue(mgr.getCounterSupplier() instanceof MockCounterSupplier);assertTrue(mgr.getMeterSupplier() instanceof MockMeterSupplier);assertTrue(mgr.getTimerSupplier() instanceof MockTimerSupplier);@@ -100,7 +100,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {System.setProperty("metricsEnabled", "false");NodeConfig cfg = loadNodeConfig("solr-metricsconfig.xml");SolrMetricManager mgr =- new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null);+ new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());assertTrue(mgr.getCounterSupplier() instanceof MetricSuppliers.NoOpCounterSupplier);assertTrue(mgr.getMeterSupplier() instanceof MetricSuppliers.NoOpMeterSupplier);assertTrue(mgr.getTimerSupplier() instanceof MetricSuppliers.NoOpTimerSupplier);@@ -111,7 +111,7 @@ public class MetricsConfigTest extends SolrTestCaseJ4 {public void testMissingValuesConfig() {NodeConfig cfg = loadNodeConfig("solr-metricsconfig1.xml");SolrMetricManager mgr =- new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig(), null);+ new SolrMetricManager(cfg.getSolrResourceLoader(), cfg.getMetricsConfig());assertNull("nullNumber", mgr.nullNumber());assertEquals("notANumber", -1, mgr.notANumber());assertEquals("nullNumber", "", mgr.nullString());diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.javaindex 730fe0989aa..2c8f51aec6d 100644--- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricManagerTest.java@@ -20,10 +20,28 @@ package org.apache.solr.metrics;import com.codahale.metrics.Counter;import com.codahale.metrics.Metric;import com.codahale.metrics.MetricRegistry;+import com.google.common.util.concurrent.AtomicDouble;+import io.opentelemetry.api.metrics.DoubleCounter;+import io.opentelemetry.api.metrics.DoubleGauge;+import io.opentelemetry.api.metrics.DoubleHistogram;+import io.opentelemetry.api.metrics.DoubleUpDownCounter;+import io.opentelemetry.api.metrics.LongCounter;+import io.opentelemetry.api.metrics.LongGauge;+import io.opentelemetry.api.metrics.LongHistogram;+import io.opentelemetry.api.metrics.LongUpDownCounter;+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;+import io.prometheus.metrics.model.snapshots.CounterSnapshot;+import io.prometheus.metrics.model.snapshots.GaugeSnapshot;+import io.prometheus.metrics.model.snapshots.HistogramSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshot;+import io.prometheus.metrics.model.snapshots.MetricSnapshots;import java.util.HashMap;import java.util.Map;import java.util.Random;import java.util.Set;+import java.util.concurrent.atomic.AtomicLong;+import java.util.concurrent.atomic.DoubleAdder;+import java.util.concurrent.atomic.LongAdder;import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.util.NamedList;@@ -31,10 +49,24 @@ import org.apache.solr.core.PluginInfo;import org.apache.solr.core.SolrInfoBean;import org.apache.solr.core.SolrResourceLoader;import org.apache.solr.metrics.reporters.MockMetricReporter;+import org.junit.Before;import org.junit.Test;public class SolrMetricManagerTest extends SolrTestCaseJ4 {+ final String METER_PROVIDER_NAME = "test_provider_name";+ private SolrMetricManager metricManager;+ private PrometheusMetricReader reader;++ @Before+ public void setUp() throws Exception {+ super.setUp();+ this.metricManager = new SolrMetricManager();+ // Initialize a metric reader for tests+ metricManager.meterProvider(METER_PROVIDER_NAME);+ this.reader = metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME);+ }+ // NOCOMMIT: We might not be supported core swapping in 10. Maybe remove this test@Testpublic void testSwapRegistries() {Random r = random();@@ -76,6 +108,8 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 {}}+ // NOCOMMIT: Migration of this to OTEL isn't possible. You can't register instruments to a+ // meterprovider that the provider itself didn't create@Testpublic void testRegisterAll() throws Exception {Random r = random();@@ -104,6 +138,8 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 {registryName, mr, SolrMetricManager.ResolutionStrategy.ERROR));}+ // NOCOMMIT: Migration of this to OTEL isn't possible. You can only delete the whole+ // sdkMeterProvider and all it's recorded metrics@Testpublic void testClearMetrics() {Random r = random();@@ -270,4 +306,230 @@ public class SolrMetricManagerTest extends SolrTestCaseJ4 {initArgs.add("configurable", "true");return new PluginInfo("SolrMetricReporter", attrs, initArgs, null);}++ @Test+ public void testLongCounter() {+ LongCounter counter =+ metricManager.longCounter(METER_PROVIDER_NAME, "my_counter", "desc", null);+ counter.add(5);+ counter.add(3);+ CounterSnapshot actual = snapshot("my_counter", CounterSnapshot.class);+ assertEquals(8.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testLongUpDownCounter() {+ LongUpDownCounter counter =+ metricManager.longUpDownCounter(METER_PROVIDER_NAME, "long_updown_counter", "desc", null);+ counter.add(10);+ counter.add(-4);+ GaugeSnapshot actual = snapshot("long_updown_counter", GaugeSnapshot.class);+ assertEquals(6.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testDoubleUpDownCounter() {+ DoubleUpDownCounter counter =+ metricManager.doubleUpDownCounter(+ METER_PROVIDER_NAME, "double_updown_counter", "desc", null);+ counter.add(10.0);+ counter.add(-5.5);++ GaugeSnapshot actual = snapshot("double_updown_counter", GaugeSnapshot.class);+ assertEquals(4.5, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testDoubleCounter() {+ DoubleCounter counter =+ metricManager.doubleCounter(METER_PROVIDER_NAME, "double_counter", "desc", null);+ counter.add(10.0);+ counter.add(5.5);++ CounterSnapshot actual = snapshot("double_counter", CounterSnapshot.class);+ assertEquals(15.5, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testDoubleHistogram() {+ DoubleHistogram histogram =+ metricManager.doubleHistogram(METER_PROVIDER_NAME, "double_histogram", "desc", null);+ histogram.record(1.1);+ histogram.record(2.2);+ histogram.record(3.3);++ HistogramSnapshot actual = snapshot("double_histogram", HistogramSnapshot.class);+ assertEquals(3, actual.getDataPoints().getFirst().getCount());+ assertEquals(6.6, actual.getDataPoints().getFirst().getSum(), 0.0);+ }++ @Test+ public void testLongHistogram() {+ LongHistogram histogram =+ metricManager.longHistogram(METER_PROVIDER_NAME, "long_histogram", "desc", null);+ histogram.record(1);+ histogram.record(2);+ histogram.record(3);++ HistogramSnapshot actual = snapshot("long_histogram", HistogramSnapshot.class);+ assertEquals(3, actual.getDataPoints().getFirst().getCount());+ assertEquals(6.0, actual.getDataPoints().getFirst().getSum(), 0.0);+ }++ @Test+ public void testDoubleGauge() {+ DoubleGauge gauge =+ metricManager.doubleGauge(METER_PROVIDER_NAME, "double_gauge", "desc", null);+ gauge.set(10.0);+ gauge.set(5.5);++ GaugeSnapshot actual = snapshot("double_gauge", GaugeSnapshot.class);+ assertEquals(5.5, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testLongGauge() {+ LongGauge gauge = metricManager.longGauge(METER_PROVIDER_NAME, "long_gauge", "desc", null);+ gauge.set(10);+ gauge.set(5);++ GaugeSnapshot actual = snapshot("long_gauge", GaugeSnapshot.class);+ assertEquals(5.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testObservableLongCounter() {+ LongAdder val = new LongAdder();+ metricManager.observableLongCounter(+ METER_PROVIDER_NAME, "obs_long_counter", "desc", m -> m.record(val.longValue()), null);+ val.add(10);++ CounterSnapshot actual = snapshot("obs_long_counter", CounterSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.add(20);+ actual = snapshot("obs_long_counter", CounterSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(30.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testObservableDoubleCounter() {+ DoubleAdder val = new DoubleAdder();+ metricManager.observableDoubleCounter(+ METER_PROVIDER_NAME, "obs_double_counter", "desc", m -> m.record(val.doubleValue()), null);+ val.add(10.0);++ CounterSnapshot actual = snapshot("obs_double_counter", CounterSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.add(0.1);+ actual = snapshot("obs_double_counter", CounterSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(10.1, actual.getDataPoints().getFirst().getValue(), 1e-6);+ }++ @Test+ public void testObservableLongGauge() {+ AtomicLong val = new AtomicLong();+ metricManager.observableLongGauge(+ METER_PROVIDER_NAME, "obs_long_gauge", "desc", m -> m.record(val.get()), null);+ val.set(10L);++ GaugeSnapshot actual = snapshot("obs_long_gauge", GaugeSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.set(20L);+ actual = snapshot("obs_long_gauge", GaugeSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(20.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testObservableDoubleGauge() {+ AtomicDouble val = new AtomicDouble();+ metricManager.observableDoubleGauge(+ METER_PROVIDER_NAME, "obs_double_gauge", "desc", m -> m.record(val.get()), null);+ val.set(10.0);++ GaugeSnapshot actual = snapshot("obs_double_gauge", GaugeSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.set(10.1);+ actual = snapshot("obs_double_gauge", GaugeSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(10.1, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testObservableLongUpDownCounter() {+ LongAdder val = new LongAdder();+ metricManager.observableLongUpDownCounter(+ METER_PROVIDER_NAME, "obs_long_updown_gauge", "desc", m -> m.record(val.longValue()), null);+ val.add(10L);++ GaugeSnapshot actual = snapshot("obs_long_updown_gauge", GaugeSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.add(-20L);+ actual = snapshot("obs_long_updown_gauge", GaugeSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(-10.0, actual.getDataPoints().getFirst().getValue(), 0.0);+ }++ @Test+ public void testObservableDoubleUpDownCounter() {+ DoubleAdder val = new DoubleAdder();+ metricManager.observableDoubleUpDownCounter(+ METER_PROVIDER_NAME,+ "obs_double_updown_gauge",+ "desc",+ m -> m.record(val.doubleValue()),+ null);+ val.add(10.0);+ GaugeSnapshot actual = snapshot("obs_double_updown_gauge", GaugeSnapshot.class);+ assertEquals(10.0, actual.getDataPoints().getFirst().getValue(), 0.0);++ val.add(-20.1);+ actual = snapshot("obs_double_updown_gauge", GaugeSnapshot.class);++ // Observable metrics value changes anytime metricReader collects() to trigger callback+ assertEquals(-10.1, actual.getDataPoints().getFirst().getValue(), 1e-6);+ }++ @Test+ public void testCloseMeterProviders() {+ LongCounter counter =+ metricManager.longCounter(METER_PROVIDER_NAME, "my_counter", "desc", null);+ counter.add(5);++ PrometheusMetricReader reader = metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME);+ MetricSnapshots metrics = reader.collect();+ CounterSnapshot data =+ metrics.stream()+ .filter(m -> m.getMetadata().getPrometheusName().equals("my_counter"))+ .map(CounterSnapshot.class::cast)+ .findFirst()+ .orElseThrow(() -> new AssertionError("LongCounter metric not found"));+ assertEquals(5, data.getDataPoints().getFirst().getValue(), 0.0);++ metricManager.removeRegistry(METER_PROVIDER_NAME);++ assertNull(metricManager.getPrometheusMetricReader(METER_PROVIDER_NAME));+ }++ // Helper to grab any snapshot by name and type+ private <T extends MetricSnapshot> T snapshot(String name, Class<T> cls) {+ return reader.collect().stream()+ .filter(m -> m.getMetadata().getPrometheusName().equals(name))+ .filter(cls::isInstance)+ .map(cls::cast)+ .findFirst()+ .orElseThrow(() -> new AssertionError("MetricSnapshot not found: " + name));+ }}diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.javaindex 4792be1021c..a0010476393 100644--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java@@ -16,10 +16,6 @@*/package org.apache.solr.response;-import com.codahale.metrics.Counter;-import com.codahale.metrics.Gauge;-import com.codahale.metrics.Meter;-import com.codahale.metrics.SettableGauge;import com.codahale.metrics.SharedMetricRegistries;import java.lang.invoke.MethodHandles;import java.util.Arrays;@@ -36,10 +32,8 @@ import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.common.params.ModifiableSolrParams;import org.apache.solr.common.util.NamedList;-import org.apache.solr.metrics.SolrMetricManager;import org.apache.solr.util.ExternalPaths;import org.apache.solr.util.SolrJettyTestRule;-import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.ClassRule;import org.junit.Test;@@ -57,25 +51,22 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {solrClientTestRule.startSolr(LuceneTestCase.createTempDir());solrClientTestRule- .newCollection()+ .newCollection("core1")+ .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET.toString())+ .create();+ solrClientTestRule+ .newCollection("core2").withConfigSet(ExternalPaths.DEFAULT_CONFIGSET.toString()).create();var cc = solrClientTestRule.getCoreContainer();cc.waitForLoadingCoresToFinish(30000);- SolrMetricManager manager = cc.getMetricManager();- Counter c = manager.counter(null, "solr.core.collection1", "QUERY./dummy/metrics.requests");- c.inc(10);- c = manager.counter(null, "solr.node", "ADMIN./dummy/metrics.requests");- c.inc(20);- Meter m = manager.meter(null, "solr.jetty", "dummyMetrics.2xx-responses");- m.mark(30);- registerGauge(manager, "solr.jvm", "gc.dummyMetrics.count");- }+ // Populate request metrics on both cores+ ModifiableSolrParams queryParams = new ModifiableSolrParams();+ queryParams.set("q", "*:*");- @AfterClass- public static void clearMetricsRegistries() {- SharedMetricRegistries.clear();+ solrClientTestRule.getSolrClient("core1").query(queryParams);+ solrClientTestRule.getSolrClient("core2").query(queryParams);}@Test@@ -87,7 +78,6 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {NamedList<Object> res = adminClient.request(req);- assertNotNull("null response from server", res);String output = (String) res.get("response");Set<String> seenTypeInfo = new HashSet<>();@@ -130,20 +120,4 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {});}}-- private static void registerGauge(- SolrMetricManager metricManager, String registry, String metricName) {- Gauge<Number> metric =- new SettableGauge<>() {- @Override- public void setValue(Number value) {}-- @Override- public Number getValue() {- return 0;- }- };- metricManager.registerGauge(- null, registry, metric, "", SolrMetricManager.ResolutionStrategy.IGNORE, metricName, "");- }}diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.javanew file mode 100644index 00000000000..4ed29e43199--- /dev/null+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriterCloud.java@@ -0,0 +1,164 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.response;++import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest.METHOD;+import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++public class TestPrometheusResponseWriterCloud extends SolrCloudTestCase {++ @BeforeClass+ public static void setupCluster() throws Exception {+ System.setProperty("metricsEnabled", "true");+ configureCluster(1)+ .addConfig(+ "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))+ .configure();+ }++ @Before+ public void ensureCollectionsExist() throws Exception {+ SolrClient client = cluster.getSolrClient();+ try {+ CollectionAdminRequest.deleteCollection("collection1").process(client);+ } catch (Exception ignored) {+ }+ try {+ CollectionAdminRequest.deleteCollection("collection2").process(client);+ } catch (Exception ignored) {+ }++ CollectionAdminRequest.createCollection("collection1", "config", 1, 1).process(client);+ CollectionAdminRequest.createCollection("collection2", "config", 1, 1).process(client);+ cluster.waitForActiveCollection("collection1", 1, 1);+ cluster.waitForActiveCollection("collection2", 1, 1);+ }++ @Test+ public void testPrometheusCloudLabels() throws Exception {+ var solrClient = cluster.getSolrClient();++ // Increment solr_metrics_core_requests metric for /select+ SolrQuery query = new SolrQuery("*:*");+ solrClient.query("collection1", query);++ var req =+ new GenericSolrRequest(+ METHOD.GET,+ "/admin/metrics",+ SolrRequestType.ADMIN,+ new ModifiableSolrParams().set("wt", "prometheus"));+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ NamedList<Object> resp = solrClient.request(req);+ try (InputStream in = (InputStream) resp.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ assertTrue(+ "Missing expected Solr cloud mode prometheus metric with cloud labels",+ output+ .lines()+ .anyMatch(+ line ->+ line.startsWith("solr_metrics_core_requests_total")+ && line.contains("handler=\"/select\"")+ && line.contains("collection=\"collection1\"")+ && line.contains("core=\"collection1_shard1_replica_n1\"")+ && line.contains("replica=\"replica_n1\"")+ && line.contains("shard=\"shard1\"")+ && line.contains("type=\"requests\"")));+ }+ }++ @Test+ public void testCollectionDeletePrometheusOutput() throws Exception {+ var solrClient = cluster.getSolrClient();++ // Increment solr_metrics_core_requests metric for /select and assert it exists+ SolrQuery query = new SolrQuery("*:*");+ solrClient.query("collection1", query);+ solrClient.query("collection2", query);++ var req =+ new GenericSolrRequest(+ METHOD.GET,+ "/admin/metrics",+ SolrRequestType.ADMIN,+ new ModifiableSolrParams().set("wt", "prometheus"));+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ NamedList<Object> resp = solrClient.request(req);++ try (InputStream in = (InputStream) resp.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);++ assertTrue(+ "Prometheus output should contains solr_metrics_core_requests for collection1",+ output+ .lines()+ .anyMatch(+ line ->+ line.startsWith("solr_metrics_core_requests")+ && line.contains("collection1")));+ assertTrue(+ "Prometheus output should contains solr_metrics_core_requests for collection2",+ output+ .lines()+ .anyMatch(+ line ->+ line.startsWith("solr_metrics_core_requests")+ && line.contains("collection2")));+ }++ // Delete collection and assert metrics have been removed+ var deleteRequest = CollectionAdminRequest.deleteCollection("collection1");+ deleteRequest.process(solrClient);++ resp = solrClient.request(req);+ try (InputStream in = (InputStream) resp.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ assertFalse(+ "Prometheus output should not contain solr_metrics_core_requests after collection was deleted",+ output+ .lines()+ .anyMatch(+ line ->+ line.startsWith("solr_metrics_core_requests")+ && line.contains("collection1")));+ assertTrue(+ "Prometheus output should contains solr_metrics_core_requests for collection2",+ output+ .lines()+ .anyMatch(+ line ->+ line.startsWith("solr_metrics_core_requests")+ && line.contains("collection2")));+ }+ }+}diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.javaindex c475f2b23d3..53e1915f7c2 100644--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java@@ -25,7 +25,6 @@ import static org.hamcrest.core.StringContains.containsString;import com.carrotsearch.randomizedtesting.RandomizedContext;import com.carrotsearch.randomizedtesting.RandomizedTest;import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;-import io.opentelemetry.api.GlobalOpenTelemetry;import java.io.IOException;import java.io.OutputStreamWriter;import java.io.Reader;@@ -113,6 +112,7 @@ import org.apache.solr.common.util.XML;import org.apache.solr.core.CoreContainer;import org.apache.solr.core.CoresLocator;import org.apache.solr.core.NodeConfig;+import org.apache.solr.core.OpenTelemetryConfigurator;import org.apache.solr.core.SolrConfig;import org.apache.solr.core.SolrCore;import org.apache.solr.core.SolrXmlConfig;@@ -250,7 +250,7 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {@BeforeClasspublic static void setupTestCases() {- GlobalOpenTelemetry.resetForTest();+ OpenTelemetryConfigurator.resetForTest();resetExceptionIgnores();testExecutor =diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex 4a894521e24..b559a233a80 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -1169,6 +1169,7 @@ public class MiniSolrCloudCluster {System.setProperty("solr.cloud.overseer.enabled", Boolean.toString(overseerEnabled));if (!disableTraceIdGeneration && OpenTelemetryConfigurator.TRACE_ID_GEN_ENABLED) {+ OpenTelemetryConfigurator.initializeOpenTelemetrySdk(null, null);injectRandomRecordingFlag();}