Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17818: Expose Open Metrics 1.0 format from /admin/metrics and exemplar support (#3427)

Matthew Biscocho8e02bfc4e871
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 825e8b11094..bfe73060d22 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -17,6 +17,7 @@
package org.apache.solr.core;
import static org.apache.solr.common.params.CommonParams.PATH;
+import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;
import static org.apache.solr.handler.admin.MetricsHandler.PROMETHEUS_METRICS_WT;
import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR;
import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR;
@@ -3102,6 +3103,7 @@ public class SolrCore implements SolrInfoBean, Closeable {
m.put("schema.xml", new SchemaXmlResponseWriter());
m.put("smile", new SmileResponseWriter());
m.put(PROMETHEUS_METRICS_WT, new PrometheusResponseWriter());
+ m.put(OPEN_METRICS_WT, new PrometheusResponseWriter());
m.put(ReplicationAPIBase.FILE_STREAM, getFileStreamWriter());
DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
try {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index ba7f6bba34b..f55c0c4b177 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -69,7 +69,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
public static final String KEY_PARAM = "key";
public static final String EXPR_PARAM = "expr";
public static final String TYPE_PARAM = "type";
+ // NOCOMMIT: This wt=prometheus will be removed as it will become the default for /admin/metrics
public static final String PROMETHEUS_METRICS_WT = "prometheus";
+ public static final String OPEN_METRICS_WT = "openmetrics";
public static final String ALL = "all";
@@ -126,8 +128,9 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
return;
}
- // TODO SOLR-17458: Make this the default option after dropwizard removal
- if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))) {
+ // NOCOMMIT SOLR-17458: Make this the default option after dropwizard removal
+ if (PROMETHEUS_METRICS_WT.equals(params.get(CommonParams.WT))
+ || OPEN_METRICS_WT.equals(params.get(CommonParams.WT))) {
consumer.accept("metrics", metricManager.getPrometheusMetricReaders());
return;
}
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
index 7056a457861..0e6a1db637c 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricManager.java
@@ -52,6 +52,8 @@ import io.opentelemetry.api.metrics.ObservableLongMeasurement;
import io.opentelemetry.api.metrics.ObservableLongUpDownCounter;
import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil;
+import io.opentelemetry.sdk.metrics.internal.exemplar.ExemplarFilter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -725,9 +727,7 @@ public class SolrMetricManager {
}
/**
- * Get (or create if not present) a named {@link SdkMeterProvider} under {@link
- * MeterProviderAndReaders}. This also registers a corresponding {@link PrometheusMetricReader}
- * Dropwizards's {@link SolrMetricManager#registry(String)} equivalent
+ * Get (or create if not present) a named {@link SdkMeterProvider}.
*
* @param providerName name of the meter provider and prometheus metric reader
* @return existing or newly created meter provider
@@ -741,8 +741,9 @@ public class SolrMetricManager {
var reader = new PrometheusMetricReader(true, null);
// NOCOMMIT: We need to add a Periodic Metric Reader here if we want to push with OTLP
// with an exporter
- var provider = SdkMeterProvider.builder().registerMetricReader(reader).build();
- return new MeterProviderAndReaders(provider, reader);
+ var provider = SdkMeterProvider.builder().registerMetricReader(reader);
+ SdkMeterProviderUtil.setExemplarFilter(provider, ExemplarFilter.traceBased());
+ return new MeterProviderAndReaders(provider.build(), reader);
})
.sdkMeterProvider();
}
diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
index 77fd3642b2a..070ed662dc0 100644
--- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java
@@ -16,8 +16,11 @@
*/
package org.apache.solr.response;
+import static org.apache.solr.handler.admin.MetricsHandler.OPEN_METRICS_WT;
+
import io.opentelemetry.exporter.prometheus.PrometheusMetricReader;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;
import io.prometheus.metrics.model.snapshots.CounterSnapshot;
import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
@@ -31,6 +34,7 @@ import java.lang.invoke.MethodHandles;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.handler.admin.MetricsHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.slf4j.Logger;
@@ -42,6 +46,9 @@ public class PrometheusResponseWriter implements QueryResponseWriter {
// not TextQueryResponseWriter because Prometheus libs work with an OutputStream
private static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4";
+ private static final String CONTENT_TYPE_OPEN_METRICS =
+ "application/openmetrics-text; version=1.0.0; charset=utf-8";
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@Override
@@ -55,12 +62,35 @@ public class PrometheusResponseWriter implements QueryResponseWriter {
List<MetricSnapshot> snapshots =
readers.values().stream().flatMap(r -> r.collect().stream()).toList();
- new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));
+ if (writeOpenMetricsFormat(request)) {
+ new OpenMetricsTextFormatWriter(false, true).write(out, mergeSnapshots(snapshots));
+ } else {
+ new PrometheusTextFormatWriter(false).write(out, mergeSnapshots(snapshots));
+ }
}
@Override
public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
- return CONTENT_TYPE_PROMETHEUS;
+ return writeOpenMetricsFormat(request) ? CONTENT_TYPE_OPEN_METRICS : CONTENT_TYPE_PROMETHEUS;
+ }
+
+ private boolean writeOpenMetricsFormat(SolrQueryRequest request) {
+ String wt = request.getParams().get(CommonParams.WT);
+ if (OPEN_METRICS_WT.equals(wt)) {
+ return true;
+ }
+
+ String acceptHeader =
+ request.getHttpSolrCall() != null
+ ? request.getHttpSolrCall().getReq().getHeader("Accept")
+ : null;
+
+ if (acceptHeader == null) {
+ return false;
+ }
+
+ return acceptHeader.contains("application/openmetrics-text")
+ && (acceptHeader.contains("version=1.0.0"));
}
/**
diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java
index 6d818157b6f..71545ea6628 100644
--- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java
@@ -17,7 +17,9 @@
package org.apache.solr.response;
import com.codahale.metrics.SharedMetricRegistries;
+import java.io.InputStream;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@@ -28,6 +30,7 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.SolrRequest.SolrRequestType;
+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -118,4 +121,63 @@ public class TestPrometheusResponseWriter extends SolrTestCaseJ4 {
});
}
}
+
+ @Test
+ public void testAcceptHeaderOpenMetricsFormat() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);
+
+ // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed
+ req.setResponseParser(new InputStreamResponseParser("prometheus"));
+
+ req.addHeader("Accept", "application/openmetrics-text;version=1.0.0");
+
+ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {
+ NamedList<Object> res = adminClient.request(req);
+
+ try (InputStream in = (InputStream) res.get("stream")) {
+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ assertTrue(
+ "Should use OpenMetrics format when Accept header is set",
+ output.trim().endsWith("# EOF"));
+ }
+ }
+ }
+
+ @Test
+ public void testWtParameterOpenMetricsFormat() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);
+ req.setResponseParser(new InputStreamResponseParser("openmetrics"));
+
+ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {
+ NamedList<Object> res = adminClient.request(req);
+
+ try (InputStream in = (InputStream) res.get("stream")) {
+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ assertTrue(
+ "Should use OpenMetrics format when Accept header is set",
+ output.trim().endsWith("# EOF"));
+ }
+ }
+ }
+
+ @Test
+ public void testDefaultPrometheusFormat() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ var req = new GenericSolrRequest(METHOD.GET, "/admin/metrics", SolrRequestType.ADMIN, params);
+ // NOCOMMIT: Remove this prometheus writer type after Dropwizard is removed
+ req.setResponseParser(new InputStreamResponseParser("prometheus"));
+
+ try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) {
+ NamedList<Object> res = adminClient.request(req);
+
+ try (InputStream in = (InputStream) res.get("stream")) {
+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ assertFalse(
+ "Should default to Prometheus format when no Accept header or wt=openmetrics is set",
+ output.trim().endsWith("# EOF"));
+ }
+ }
+ }
}
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
index 501893e99e8..4daba5c87a4 100644
--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java
@@ -321,7 +321,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {
return result;
}
- private String getRootTraceId(List<SpanData> finishedSpans) {
+ static String getRootTraceId(List<SpanData> finishedSpans) {
assertEquals(1, finishedSpans.stream().filter(TestDistributedTracing::isRootSpan).count());
return finishedSpans.stream()
.filter(TestDistributedTracing::isRootSpan)
diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java
new file mode 100644
index 00000000000..687b86b20cb
--- /dev/null
+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestMetricExemplars.java
@@ -0,0 +1,95 @@
+package org.apache.solr.opentelemetry;
+
+import static org.apache.solr.opentelemetry.TestDistributedTracing.getAndClearSpans;
+import static org.apache.solr.opentelemetry.TestDistributedTracing.getRootTraceId;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.trace.TracerProvider;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMetricExemplars extends SolrCloudTestCase {
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ System.setProperty("otel.traces.sampler", "always_on");
+ // force early init
+ CustomTestOtelTracerConfigurator.prepareForTest();
+
+ configureCluster(1)
+ .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf"))
+ .withSolrXml(TEST_PATH().resolve("solr.xml"))
+ .withTraceIdGenerationDisabled()
+ .configure();
+
+ assertNotEquals(
+ "Expecting active otel, not noop impl",
+ TracerProvider.noop(),
+ GlobalOpenTelemetry.get().getTracerProvider());
+
+ CollectionAdminRequest.createCollection("collection1", "config", 1, 1)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection("collection1", 1, 1);
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ CustomTestOtelTracerConfigurator.resetForTest();
+ }
+
+ @Before
+ private void resetSpanData() {
+ getAndClearSpans();
+ }
+
+ @Test
+ public void testOpenMetricExemplars() throws Exception {
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+
+ // Generate exemplars
+ cloudClient.add("collection1", sdoc("id", "1"));
+ var spans = getAndClearSpans();
+ var expectedTrace = getRootTraceId(spans);
+
+ var req =
+ new GenericSolrRequest(
+ SolrRequest.METHOD.GET,
+ "/admin/metrics",
+ SolrRequest.SolrRequestType.ADMIN,
+ new ModifiableSolrParams().set("wt", "openmetrics"));
+ req.setResponseParser(new InputStreamResponseParser("openmetrics"));
+ NamedList<Object> resp = cloudClient.request(req);
+
+ try (InputStream in = (InputStream) resp.get("stream")) {
+ String output = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ var line =
+ output
+ .lines()
+ .filter(
+ l ->
+ l.startsWith("solr_metrics_core_requests_total")
+ && l.contains("handler=\"/update\"")
+ && l.contains("collection=\"collection1\""))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new AssertionError(
+ "Could not find /update solr_metrics_core_requests_total"));
+
+ String actualExemplarTrace = line.split("trace_id=\"")[1].split("\"")[0];
+ assertEquals(actualExemplarTrace, expectedTrace);
+ }
+ }
+}