Apache Solr open-source search software
SOLR-17818: Expose Open Metrics 1.0 format from /admin/metrics and exemplar support (#3427)
Matthew Biscocho8e02bfc4e871
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex 825e8b11094..bfe73060d22 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -17,6 +17,7 @@package org.apache.solr.core;import static org.apache.solr.common.params.CommonParams.PATH;+import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;@@ -3102,6 +3103,7 @@ public class SolrCore implements SolrInfoBean, Closeable {m.put("schema.xml", new SchemaXmlResponseWriter());m.put("smile", new SmileResponseWriter());m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter());+ m.put(OPEN_METRICS_WT, new PrometheusResponseWriter());m.put(ReplicationAPIBase.FILE_STREAM, getFileStreamWriter());DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);try {diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.javaindex ba7f6bba34b..f55c0c4b177 100644--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java@@ -69,7 +69,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamepublic static final String KEY_PARAM = "key";public static final String EXPR_PARAM = "expr";public static final String TYPE_PARAM = "type";+ // NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metricspublic static final String PROMETHEUS_METRICS_WT = "prometheus";+ public static final String OPEN_METRICS_WT = "openmetrics";public static final String ALL = "all";@@ -126,8 +128,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionNamereturn;}- // TODO SOLR-17458: Make this the default option after dropwizard removal- if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) {+ // NOCOMMIT SOLR-17458: Make this the default option after dropwizard removal+ if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))+ || OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) {consumer.accept("metrics", metricManager.getPrometheusMetricReaders());return;}diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.javaindex 7056a457861..0e6a1db637c 100644--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java@@ -52,6 +52,8 @@ import io.opentelemetry.api.metrics.ObservableLongMeasurement;import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;import io.opentelemetry.sdk.metrics.SdkMeterProvider;+import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;+import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter;import java.io.IOException;import java.lang.invoke.MethodHandles;import java.util.ArrayList;@@ -725,9 +727,7 @@ public class SolrMetricManager {}/**- * Get (or create if not present) a named {@link SdkMeterProvider} under {@link- * MeterProviderAndReaders}. This also registers a corresponding {@link PrometheusMetricReader}- * Dropwizards's {@link SolrMetricManager#registry(String)} equivalent+ * Get (or create if not present) a named {@link SdkMeterProvider}.** @param providerName name of the meter provider and prometheus metric reader* @return existing or newly created meter provider@@ -741,8 +741,9 @@ public class SolrMetricManager {var reader = new PrometheusMetricReader(true, null);// NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP// with an exporter- var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();- return new MeterProviderAndReaders(provider, reader);+ var provider = SdkMeterProvider.builder().registerMetricReader(reader);+ SdkMeterProviderUtil.setExemplarFilter(provider, ExemplarFilter.traceBased());+ return new MeterProviderAndReaders(provider.build(), reader);}).sdkMeterProvider();}diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.javaindex 77fd3642b2a..070ed662dc0 100644--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java@@ -16,8 +16,11 @@*/package org.apache.solr.response;+import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;+import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;import io.opentelemetry.sdk.metrics.SdkMeterProvider;+import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;import io.prometheus.metrics.model.snapshots.CounterSnapshot;import io.prometheus.metrics.model.snapshots.GaugeSnapshot;@@ -31,6 +34,7 @@ import java.lang.invoke.MethodHandles;import java.util.HashMap;import java.util.List;import java.util.Map;+import org.apache.solr.common.params.CommonParams;import org.apache.solr.handler.admin.MetricsHandler;import org.apache.solr.request.SolrQueryRequest;import org.slf4j.Logger;@@ -42,6 +46,9 @@ public class PrometheusResponseWriter implements QueryResponseWriter {// not TextQueryResponseWriter because Prometheus libs work with an OutputStreamprivate static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4";+ private static final String CONTENT_TYPE_OPEN_METRICS =+ "application/openmetrics-text; version=1.0.0; charset=utf-8";+private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());@Override@@ -55,12 +62,35 @@ public class PrometheusResponseWriter implements QueryResponseWriter {List<MetricSnapshot> snapshots =readers.values().stream().flatMap(r -> r.collect().stream()).toList();- new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));+ if (writeOpenMetricsFormat(request)) {+ new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots));+ } else {+ new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));+ }}@Overridepublic String getContentType(SolrQueryRequest request, SolrQueryResponse response) {- return CONTENT_TYPE_PROMETHEUS;+ return writeOpenMetricsFormat(request) ? CONTENT_TYPE_OPEN_METRICS : CONTENT_TYPE_PROMETHEUS;+ }++ private boolean writeOpenMetricsFormat(SolrQueryRequest request) {+ String wt = request.getParams().get(CommonParams.WT);+ if (OPEN_METRICS_WT.equals(wt)) {+ return true;+ }++ String acceptHeader =+ request.getHttpSolrCall() != null+ ? request.getHttpSolrCall().getReq().getHeader("Accept")+ : null;++ if (acceptHeader == null) {+ return false;+ }++ return acceptHeader.contains("application/openmetrics-text")+ && (acceptHeader.contains("version=1.0.0"));}/**diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.javaindex 6d818157b6f..71545ea6628 100644--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java@@ -17,7 +17,9 @@package org.apache.solr.response;import com.codahale.metrics.SharedMetricRegistries;+import java.io.InputStream;import java.lang.invoke.MethodHandles;+import java.nio.charset.StandardCharsets;import java.util.Arrays;import java.util.HashSet;import java.util.List;@@ -28,6 +30,7 @@ import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest.METHOD;import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;import org.apache.solr.client.solrj.impl.NoOpResponseParser;import org.apache.solr.client.solrj.request.GenericSolrRequest;import org.apache.solr.common.params.ModifiableSolrParams;@@ -118,4 +121,63 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {});}}++ @Test+ public void testAcceptHeaderOpenMetricsFormat() throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);++ // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ req.addHeader("Accept", "application/openmetrics-text;version=1.0.0");++ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {+ NamedList<Object> res = adminClient.request(req);++ try (InputStream in = (InputStream) res.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ assertTrue(+ "Should use OpenMetrics format when Accept header is set",+ output.trim().endsWith("# EOF"));+ }+ }+ }++ @Test+ public void testWtParameterOpenMetricsFormat() throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);+ req.setResponseParser(new InputStreamResponseParser("openmetrics"));++ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {+ NamedList<Object> res = adminClient.request(req);++ try (InputStream in = (InputStream) res.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ assertTrue(+ "Should use OpenMetrics format when Accept header is set",+ output.trim().endsWith("# EOF"));+ }+ }+ }++ @Test+ public void testDefaultPrometheusFormat() throws Exception {+ ModifiableSolrParams params = new ModifiableSolrParams();+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);+ // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed+ req.setResponseParser(new InputStreamResponseParser("prometheus"));++ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {+ NamedList<Object> res = adminClient.request(req);++ try (InputStream in = (InputStream) res.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ assertFalse(+ "Should default to Prometheus format when no Accept header or wt=openmetrics is set",+ output.trim().endsWith("# EOF"));+ }+ }+ }}diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.javaindex 501893e99e8..4daba5c87a4 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java@@ -321,7 +321,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {return result;}- private String getRootTraceId(List<SpanData> finishedSpans) {+ static String getRootTraceId(List<SpanData> finishedSpans) {assertEquals(1, finishedSpans.stream().filter(TestDistributedTracing::isRootSpan).count());return finishedSpans.stream().filter(TestDistributedTracing::isRootSpan)diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.javanew file mode 100644index 00000000000..687b86b20cb--- /dev/null+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java@@ -0,0 +1,95 @@+package org.apache.solr.opentelemetry;++import static org.apache.solr.opentelemetry.TestDistributedTracing.getAndClearSpans;+import static org.apache.solr.opentelemetry.TestDistributedTracing.getRootTraceId;++import io.opentelemetry.api.GlobalOpenTelemetry;+import io.opentelemetry.api.trace.TracerProvider;+import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.util.NamedList;+import org.junit.AfterClass;+import org.junit.Before;+import org.junit.BeforeClass;+import org.junit.Test;++public class TestMetricExemplars extends SolrCloudTestCase {++ @BeforeClass+ public static void setupCluster() throws Exception {+ System.setProperty("otel.traces.sampler", "always_on");+ // force early init+ CustomTestOtelTracerConfigurator.prepareForTest();++ configureCluster(1)+ .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf"))+ .withSolrXml(TEST_PATH().resolve("solr.xml"))+ .withTraceIdGenerationDisabled()+ .configure();++ assertNotEquals(+ "Expecting active otel, not noop impl",+ TracerProvider.noop(),+ GlobalOpenTelemetry.get().getTracerProvider());++ CollectionAdminRequest.createCollection("collection1", "config", 1, 1)+ .process(cluster.getSolrClient());+ cluster.waitForActiveCollection("collection1", 1, 1);+ }++ @AfterClass+ public static void afterClass() {+ CustomTestOtelTracerConfigurator.resetForTest();+ }++ @Before+ private void resetSpanData() {+ getAndClearSpans();+ }++ @Test+ public void testOpenMetricExemplars() throws Exception {+ CloudSolrClient cloudClient = cluster.getSolrClient();++ // Generate exemplars+ cloudClient.add("collection1", sdoc("id", "1"));+ var spans = getAndClearSpans();+ var expectedTrace = getRootTraceId(spans);++ var req =+ new GenericSolrRequest(+ SolrRequest.METHOD.GET,+ "/admin/metrics",+ SolrRequest.SolrRequestType.ADMIN,+ new ModifiableSolrParams().set("wt", "openmetrics"));+ req.setResponseParser(new InputStreamResponseParser("openmetrics"));+ NamedList<Object> resp = cloudClient.request(req);++ try (InputStream in = (InputStream) resp.get("stream")) {+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);+ var line =+ output+ .lines()+ .filter(+ l ->+ l.startsWith("solr_metrics_core_requests_total")+ && l.contains("handler=\"/update\"")+ && l.contains("collection=\"collection1\""))+ .findFirst()+ .orElseThrow(+ () ->+ new AssertionError(+ "Could not find /update solr_metrics_core_requests_total"));++ String actualExemplarTrace = line.split("trace_id=\"")[1].split("\"")[0];+ assertEquals(actualExemplarTrace, expectedTrace);+ }+ }+}