Apache Solr open-source search software
SOLR-15367 Convert "rid" functionality into a default Tracer (#1854)
Alex Dce6c13e41806
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex ce100a932d6..8c903cc364d 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -19,6 +19,8 @@ Improvements* SOLR-16536: Replace OpenTracing instrumentation with OpenTelemetry (Alex Deparvu, janhoy)+* SOLR-15367: Convert "rid" functionality into a default Tracer (Alex Deparvu, David Smiley)+Optimizations---------------------(No changes)diff --git a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.javaindex bc1798863f7..f9a6846ca91 100644--- a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java+++ b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java@@ -19,17 +19,24 @@ package org.apache.solr.core;import io.opentelemetry.api.trace.Tracer;import org.apache.solr.util.plugin.NamedListInitializedPlugin;+import org.apache.solr.util.tracing.SimplePropagator;import org.apache.solr.util.tracing.TraceUtils;/** Produces a {@link Tracer} from configuration. */public abstract class TracerConfigurator implements NamedListInitializedPlugin {+ public static final boolean TRACE_ID_GEN_ENABLED =+ !Boolean.getBoolean("solr.disable.alwaysOnTraceId");+public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) {if (info != null && info.isEnabled()) {TracerConfigurator configurator =loader.newInstance(info.className, TracerConfigurator.class);configurator.init(info.initArgs);return configurator.getTracer();++ } else if (TRACE_ID_GEN_ENABLED) {+ return SimplePropagator.load();} else {return TraceUtils.noop();}diff --git a/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.javanew file mode 100644index 00000000000..01a72e0a1ff--- /dev/null+++ b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java@@ -0,0 +1,111 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util.tracing;++import io.opentelemetry.api.GlobalOpenTelemetry;+import io.opentelemetry.api.OpenTelemetry;+import io.opentelemetry.api.trace.Tracer;+import io.opentelemetry.context.Context;+import io.opentelemetry.context.ContextKey;+import io.opentelemetry.context.propagation.ContextPropagators;+import io.opentelemetry.context.propagation.TextMapGetter;+import io.opentelemetry.context.propagation.TextMapPropagator;+import io.opentelemetry.context.propagation.TextMapSetter;+import java.lang.invoke.MethodHandles;+import java.util.Collection;+import java.util.List;+import java.util.concurrent.atomic.AtomicLong;+import org.apache.solr.logging.MDCLoggingContext;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;++/**+ * Simple Http Header Propagator. When enabled, this will only propagate the trace id from the+ * client to all internal requests. It is also in charge of generating a trace id if none exists.+ *+ * <p>Note: this is very similar in impl to+ * io.opentelemetry.extension.incubator.propagation.PassThroughPropagator. we should consider+ * replacing/upgrading once that becomes generally available+ */+public class SimplePropagator implements TextMapPropagator {+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ private static final String TRACE_HOST_NAME =+ System.getProperty("trace_host_name", System.getProperty("host"));+ private static final TextMapPropagator INSTANCE = new SimplePropagator();+ private static final ContextKey<String> TRACE_ID_KEY = ContextKey.named("trace_id");++ static final String TRACE_ID = System.getProperty("TRACE_ID", "X-Trace-Id");+ private static final List<String> FIELDS = List.of(TRACE_ID);++ private static final AtomicLong traceCounter = new AtomicLong(0);++ private static volatile boolean loaded = false;++ public static synchronized Tracer load() {+ if (!loaded) {+ log.info("OpenTelemetry tracer enabled with simple propagation only.");+ OpenTelemetry otel =+ OpenTelemetry.propagating(ContextPropagators.create(SimplePropagator.getInstance()));+ GlobalOpenTelemetry.set(otel);+ loaded = true;+ }+ return GlobalOpenTelemetry.getTracer("solr");+ }++ public static TextMapPropagator getInstance() {+ return INSTANCE;+ }++ private SimplePropagator() {}++ @Override+ public Collection<String> fields() {+ return FIELDS;+ }++ @Override+ public <C> void inject(Context context, C carrier, TextMapSetter<C> setter) {+ if (setter == null) {+ return;+ }+ String traceId = context.get(TRACE_ID_KEY);+ if (traceId != null) {+ setter.set(carrier, TRACE_ID, traceId);+ }+ }++ @Override+ public <C> Context extract(Context context, C carrier, TextMapGetter<C> getter) {+ String traceId = getter.get(carrier, TRACE_ID);+ if (traceId == null) {+ traceId = newTraceId();+ }++ MDCLoggingContext.setTracerId(traceId);+ return context.with(TRACE_ID_KEY, traceId);+ }++ private static String newTraceId() {+ return TRACE_HOST_NAME + "-" + traceCounter.incrementAndGet();+ }++ @Override+ public String toString() {+ return "SimplePropagator";+ }+}diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.javanew file mode 100644index 00000000000..0213fce7b9a--- /dev/null+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestSimplePropagatorDistributedTracing.java@@ -0,0 +1,117 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.util.tracing;++import io.opentelemetry.api.GlobalOpenTelemetry;+import io.opentelemetry.api.trace.TracerProvider;+import java.io.IOException;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.util.SuppressForbidden;+import org.apache.solr.core.SolrCore;+import org.apache.solr.logging.MDCLoggingContext;+import org.apache.solr.update.processor.LogUpdateProcessorFactory;+import org.apache.solr.util.LogListener;+import org.junit.BeforeClass;+import org.junit.Test;++@SuppressForbidden(reason = "We need to use log4J2 classes directly to test MDC impacts")+public class TestSimplePropagatorDistributedTracing extends SolrCloudTestCase {++ private static final String COLLECTION = "collection1";++ @BeforeClass+ public static void setupCluster() throws Exception {+ configureCluster(4).addConfig("conf", configset("cloud-minimal")).configure();++ // tracer should be disabled+ assertEquals(+ "Expecting noop otel (propagating only)",+ TracerProvider.noop(),+ GlobalOpenTelemetry.get().getTracerProvider());++ CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)+ .process(cluster.getSolrClient());+ cluster.waitForActiveCollection(COLLECTION, 2, 4);+ }++ @Test+ public void test() throws IOException, SolrServerException {+ CloudSolrClient cloudClient = cluster.getSolrClient();++ // Indexing has trace ids+ try (LogListener reqLog = LogListener.info(LogUpdateProcessorFactory.class.getName())) {+ // verify all indexing events have trace id present+ cloudClient.add(COLLECTION, sdoc("id", "1"));+ cloudClient.add(COLLECTION, sdoc("id", "2"));+ cloudClient.add(COLLECTION, sdoc("id", "3"));+ var queue = reqLog.getQueue();+ assertFalse(queue.isEmpty());+ while (!queue.isEmpty()) {+ var reqEvent = queue.poll();+ String evTraceId = reqEvent.getContextData().getValue(MDCLoggingContext.TRACE_ID);+ assertNotNull(evTraceId);+ }++ // TODO this doesn't work due to solr client creating the UpdateRequest without headers+ // // verify all events have the same 'custom' traceid+ // String traceId = "tidTestSimplePropagatorDistributedTracing0";+ // var doc = sdoc("id", "4");+ // UpdateRequest u = new UpdateRequest();+ // u.add(doc);+ // u.addHeader(SimplePropagator.TRACE_ID, traceId);+ // var r1 = u.process(cloudClient, COLLECTION);+ // assertEquals(0, r1.getStatus());+ // assertSameTraceId(reqLog, traceId);+ }++ // Searching has trace ids+ try (LogListener reqLog = LogListener.info(SolrCore.class.getName() + ".Request")) {+ // verify all query events have the same auto-generated traceid+ var r1 = cloudClient.query(COLLECTION, new SolrQuery("*:*"));+ assertEquals(0, r1.getStatus());+ assertSameTraceId(reqLog, null);++ // verify all query events have the same 'custom' traceid+ String traceId = "tidTestSimplePropagatorDistributedTracing1";+ var q = new QueryRequest(new SolrQuery("*:*"));+ q.addHeader(SimplePropagator.TRACE_ID, traceId);+ var r2 = q.process(cloudClient, COLLECTION);+ assertEquals(0, r2.getStatus());+ assertSameTraceId(reqLog, traceId);+ }+ }++ private void assertSameTraceId(LogListener reqLog, String traceId) {+ var queue = reqLog.getQueue();+ assertFalse(queue.isEmpty());+ if (traceId == null) {+ traceId = queue.poll().getContextData().getValue(MDCLoggingContext.TRACE_ID);+ assertNotNull(traceId);+ }+ while (!queue.isEmpty()) {+ var reqEvent = queue.poll();+ String evTraceId = reqEvent.getContextData().getValue(MDCLoggingContext.TRACE_ID);+ assertEquals(traceId, evTraceId);+ }+ }+}diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.javaindex dea832e87f6..f6e088e5168 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java@@ -54,6 +54,7 @@ public class TestDistributedTracing extends SolrCloudTestCase {configureCluster(4).addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")).withSolrXml(TEST_PATH().resolve("solr.xml"))+ .withTraceIdGenerationDisabled().configure();assertNotEquals(diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex 3a4d41d0420..fdc64237a80 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -78,9 +78,11 @@ import org.apache.solr.common.util.IOUtils;import org.apache.solr.common.util.SolrNamedThreadFactory;import org.apache.solr.common.util.TimeSource;import org.apache.solr.core.CoreContainer;+import org.apache.solr.core.TracerConfigurator;import org.apache.solr.embedded.JettyConfig;import org.apache.solr.embedded.JettySolrRunner;import org.apache.solr.util.TimeOut;+import org.apache.solr.util.tracing.SimplePropagator;import org.apache.zookeeper.KeeperException;import org.eclipse.jetty.server.handler.HandlerWrapper;import org.eclipse.jetty.servlet.ServletHolder;@@ -1044,6 +1046,7 @@ public class MiniSolrCloudCluster {private boolean useDistributedCollectionConfigSetExecution;private boolean useDistributedClusterStateUpdate;private boolean formatZkServer = true;+ private boolean disableTraceIdGeneration = false;/*** Create a builder@@ -1237,6 +1240,11 @@ public class MiniSolrCloudCluster {"solr.distributedClusterStateUpdates",Boolean.toString(useDistributedClusterStateUpdate));+ // eager init to prevent OTEL init races caused by test setup+ if (!disableTraceIdGeneration && TracerConfigurator.TRACE_ID_GEN_ENABLED) {+ SimplePropagator.load();+ }+JettyConfig jettyConfig = jettyConfigBuilder.build();MiniSolrCloudCluster cluster =new MiniSolrCloudCluster(@@ -1279,5 +1287,10 @@ public class MiniSolrCloudCluster {cluster.put(key, value);return this;}++ public Builder withTraceIdGenerationDisabled() {+ this.disableTraceIdGeneration = true;+ return this;+ }}}