Apache Solr open-source search software
SOLR-16938 Auto configure tracer without a <tracerConfig> tag in solr… (#1922)
Alex D5f25f09de465
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 7d5c3776f5c..4e46e13a900 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -128,6 +128,8 @@ Improvements* SOLR-16461: `/solr/coreName/replication?command=backup` now has a v2 equivalent, available at`/api/cores/coreName/replication/backups` (Sanjay Dutt, Jason Gerlowski, Alex Deparvu)+* SOLR-16938: Auto configure tracer without a <tracerConfig> tag in solr.xml (Alex Deparvu)+Optimizations---------------------diff --git a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.javaindex 633201db82f..47e5d32cb7b 100644--- a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java+++ b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java@@ -18,9 +18,16 @@package org.apache.solr.core;import io.opentelemetry.api.trace.Tracer;+import java.lang.invoke.MethodHandles;+import java.util.Locale;+import java.util.Map;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.util.NamedList;import org.apache.solr.util.plugin.NamedListInitializedPlugin;import org.apache.solr.util.tracing.SimplePropagator;import org.apache.solr.util.tracing.TraceUtils;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/** Produces a {@link Tracer} from configuration. */public abstract class TracerConfigurator implements NamedListInitializedPlugin {@@ -28,19 +35,81 @@ public abstract class TracerConfigurator implements NamedListInitializedPlugin {public static final boolean TRACE_ID_GEN_ENABLED =Boolean.parseBoolean(System.getProperty("solr.alwaysOnTraceId", "true"));+ private static final String DEFAULT_CLASS_NAME =+ System.getProperty(+ "solr.otelDefaultConfigurator", "org.apache.solr.opentelemetry.OtelTracerConfigurator");++ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) {if (info != null && info.isEnabled()) {TracerConfigurator configurator =loader.newInstance(info.className, TracerConfigurator.class);configurator.init(info.initArgs);return configurator.getTracer();-- } else if (TRACE_ID_GEN_ENABLED) {+ }+ if (shouldAutoConfigOTEL()) {+ return autoConfigOTEL(loader);+ }+ if (TRACE_ID_GEN_ENABLED) {return SimplePropagator.load();- } else {- return TraceUtils.noop();}+ return TraceUtils.getGlobalTracer();}protected abstract Tracer getTracer();++ private static Tracer autoConfigOTEL(SolrResourceLoader loader) {+ try {+ TracerConfigurator configurator =+ loader.newInstance(DEFAULT_CLASS_NAME, TracerConfigurator.class);+ configurator.init(new NamedList<>());+ return configurator.getTracer();+ } catch (SolrException e) {+ log.error(+ "Unable to auto-config OpenTelemetry with class {}. Make sure you have enabled the 'opentelemetry' module",+ DEFAULT_CLASS_NAME,+ e);+ }+ return TraceUtils.getGlobalTracer();+ }++ /**+ * Best effort way to determine if we should attempt to init OTEL from system properties.+ *+ * @return true if OTEL should be init+ */+ static boolean shouldAutoConfigOTEL() {+ var env = System.getenv();+ boolean isSdkDisabled = Boolean.parseBoolean(getConfig("OTEL_SDK_DISABLED", env));+ if (isSdkDisabled) {+ return false;+ }+ return getConfig("OTEL_SERVICE_NAME", env) != null;+ }++ /**+ * Returns system property if found, else returns environment variable, or null if none found.+ *+ * @param envName the environment variable to look for+ * @param env current env+ * @return the resolved value+ */+ protected static String getConfig(String envName, Map<String, String> env) {+ String sysName = envNameToSyspropName(envName);+ String sysValue = System.getProperty(sysName);+ String envValue = env.get(envName);+ return sysValue != null ? sysValue : envValue;+ }++ /**+ * In OTEL Java SDK there is a convention that the java property name for OTEL_FOO_BAR is+ * otel.foo.bar+ *+ * @param envName the environmnet name to convert+ * @return the corresponding sysprop name+ */+ protected static String envNameToSyspropName(String envName) {+ return envName.toLowerCase(Locale.ROOT).replace("_", ".");+ }}diff --git a/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.javaindex aa029899347..e81657ca2ae 100644--- a/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java+++ b/solr/core/src/java/org/apache/solr/util/tracing/SimplePropagator.java@@ -64,7 +64,7 @@ public class SimplePropagator implements TextMapPropagator {GlobalOpenTelemetry.set(otel);loaded = true;}- return GlobalOpenTelemetry.getTracer("solr");+ return TraceUtils.getGlobalTracer();}public static TextMapPropagator getInstance() {diff --git a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.javaindex e63d8c6cbde..4d305f373fd 100644--- a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java+++ b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java@@ -22,7 +22,6 @@ import io.opentelemetry.api.trace.Span;import io.opentelemetry.api.trace.SpanBuilder;import io.opentelemetry.api.trace.SpanKind;import io.opentelemetry.api.trace.Tracer;-import io.opentelemetry.api.trace.TracerProvider;import io.opentelemetry.context.Context;import io.opentelemetry.context.propagation.TextMapPropagator;import java.util.List;@@ -36,7 +35,6 @@ public class TraceUtils {private static final String REQ_ATTR_TRACING_SPAN = Span.class.getName();private static final String REQ_ATTR_TRACING_TRACER = Tracer.class.getName();- private static final Tracer NOOP_TRACER = TracerProvider.noop().get(null);public static final String DEFAULT_SPAN_NAME = "http.request";public static final String WRITE_QUERY_RESPONSE_SPAN_NAME = "writeQueryResponse";@@ -66,8 +64,8 @@ public class TraceUtils {public static final String TAG_DB_TYPE_SOLR = "solr";- public static Tracer noop() {- return NOOP_TRACER;+ public static Tracer getGlobalTracer() {+ return GlobalOpenTelemetry.getTracer("solr");}public static TextMapPropagator getTextMapPropagator() {diff --git a/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.javanew file mode 100644index 00000000000..4882a295dfa--- /dev/null+++ b/solr/core/src/test/org/apache/solr/core/TestTracerConfigurator.java@@ -0,0 +1,53 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.core;++import io.opentelemetry.api.trace.TracerProvider;+import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.util.tracing.TraceUtils;+import org.junit.Test;++public class TestTracerConfigurator extends SolrTestCaseJ4 {++ @Test+ public void configuratorClassDoesNotExistTest() {+ System.setProperty("otel.service.name", "something");+ System.setProperty("solr.otelDefaultConfigurator", "configuratorClassDoesNotExistTest");+ try {+ assertTrue(TracerConfigurator.shouldAutoConfigOTEL());+ SolrResourceLoader loader = new SolrResourceLoader(TEST_PATH().resolve("collection1"));+ TracerConfigurator.loadTracer(loader, null);+ assertEquals(+ "Expecting noop otel after failure to auto-init",+ TracerProvider.noop().get(null),+ TraceUtils.getGlobalTracer());+ } finally {+ System.clearProperty("solr.otelDefaultConfigurator");+ System.clearProperty("otel.service.name");+ }+ }++ @Test+ public void otelDisabledByProperty() {+ System.setProperty("OTEL_SDK_DISABLED", "true");+ try {+ assertFalse(TracerConfigurator.shouldAutoConfigOTEL());+ } finally {+ System.clearProperty("OTEL_SDK_DISABLED");+ }+ }+}diff --git a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.javaindex 25d819718dc..221cb571b7b 100644--- a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java+++ b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java@@ -16,7 +16,6 @@*/package org.apache.solr.opentelemetry;-import io.opentelemetry.api.GlobalOpenTelemetry;import io.opentelemetry.api.trace.Tracer;import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;import java.lang.invoke.MethodHandles;@@ -28,6 +27,7 @@ import java.util.Objects;import java.util.stream.Collectors;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.TracerConfigurator;+import org.apache.solr.util.tracing.TraceUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -49,17 +49,17 @@ public class OtelTracerConfigurator extends TracerConfigurator {@Overridepublic Tracer getTracer() {- // TODO remove reliance on global- return GlobalOpenTelemetry.getTracer("solr");+ return TraceUtils.getGlobalTracer();}@Overridepublic void init(NamedList<?> args) {- prepareConfiguration();+ prepareConfiguration(args);AutoConfiguredOpenTelemetrySdk.initialize();}- void prepareConfiguration() {+ void prepareConfiguration(NamedList<?> args) {+ injectPluginSettingsIfNotConfigured(args);setDefaultIfNotConfigured("OTEL_SERVICE_NAME", "solr");setDefaultIfNotConfigured("OTEL_TRACES_EXPORTER", "otlp");setDefaultIfNotConfigured("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc");@@ -82,6 +82,20 @@ public class OtelTracerConfigurator extends TracerConfigurator {System.setProperty("otel.logs.exporter", "none");}+ /**+ * Will inject plugin configuration values into system properties if not already setup (existing+ * system properties take precedence)+ */+ private void injectPluginSettingsIfNotConfigured(NamedList<?> args) {+ args.forEach(+ (k, v) -> {+ var asSysName = envNameToSyspropName(k);+ if (asSysName.startsWith("otel.")) {+ setDefaultIfNotConfigured(asSysName, v.toString());+ }+ });+ }+/*** Add explicit tags statically to all traces, independent of request. Attributes with same name* supplied in ENV or SysProp will take precedence over attributes added in code.@@ -138,24 +152,11 @@ public class OtelTracerConfigurator extends TracerConfigurator {/*** Returns system property if found, else returns environment variable, or null if none found.*- * @param envName the environment to look for+ * @param envName the environment variable to look for* @return the resolved value*/String getEnvOrSysprop(String envName) {- String envValue = currentEnv.get(envName);- String propValue = System.getProperty(envNameToSyspropName(envName));- return propValue != null ? propValue : envValue;- }-- /**- * In OTEL Java SDK there is a convention that the java property name for OTEL_FOO_BAR is- * otel.foo.bar- *- * @param envName the environmnet name to convert- * @return the corresponding sysprop name- */- static String envNameToSyspropName(String envName) {- return envName.toLowerCase(Locale.ROOT).replace("_", ".");+ return getConfig(envName, currentEnv);}/**diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.javaindex bad6cce7d9f..cf7723f89ed 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java@@ -62,7 +62,7 @@ public class CustomTestOtelTracerConfigurator extends OtelTracerConfigurator {// force early initCustomTestOtelTracerConfigurator tracer = new CustomTestOtelTracerConfigurator();- tracer.prepareConfiguration();+ tracer.prepareConfiguration(new NamedList<>());bootOtel();}diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.javaindex b4825c4e7fd..c33db57d28f 100644--- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java+++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java@@ -19,6 +19,7 @@ package org.apache.solr.opentelemetry;import java.util.List;import java.util.Map;import org.apache.solr.SolrTestCaseJ4;+import org.apache.solr.common.util.NamedList;import org.junit.After;import org.junit.Before;import org.junit.Test;@@ -88,9 +89,19 @@ public class OtelTracerConfiguratorTest extends SolrTestCaseJ4 {@Testpublic void testResourceAttributes() throws Exception {System.setProperty("otel.resource.attributes", "foo=bar,ILLEGAL-LACKS-VALUE,");- instance.prepareConfiguration();+ instance.prepareConfiguration(new NamedList<>());assertEquals(List.of("host.name=my.solr.host", "foo=bar"),List.of(System.getProperty("otel.resource.attributes").split(",")));}++ @Test+ public void testPluginConfig() throws Exception {+ NamedList<String> conf = new NamedList<>();+ conf.add("OTEL_K1", "conf-k1"); // will be replaced by sys prop+ conf.add("otel.k7", "conf-k7"); // will be kept+ instance.prepareConfiguration(conf);+ assertEquals("prop-k1", instance.getCurrentOtelConfig().get("OTEL_K1"));+ assertEquals("conf-k7", instance.getCurrentOtelConfig().get("OTEL_K7"));+ }}diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xmlindex 752c740514a..006de0c965c 100644--- a/solr/server/resources/log4j2.xml+++ b/solr/server/resources/log4j2.xml@@ -23,7 +23,7 @@<Console name="STDOUT" target="SYSTEM_OUT"><PatternLayout><Pattern>- %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n</Pattern></PatternLayout></Console>@@ -34,7 +34,7 @@filePattern="${sys:solr.log.dir}/solr.log.%i" ><PatternLayout><Pattern>- %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n</Pattern></PatternLayout><Policies>@@ -50,7 +50,7 @@filePattern="${sys:solr.log.dir}/solr_slow_requests.log.%i" ><PatternLayout><Pattern>- %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{t:%X{trace_id}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n+ %maxLen{%d{yyyy-MM-dd HH:mm:ss.SSS} %-5p (%t) [%notEmpty{c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ =>%ex{short}}}{10240}%n</Pattern></PatternLayout><Policies>diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adocindex 18cac7c8695..92fe6c894a1 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc@@ -62,6 +62,8 @@ This module brings support for the industry standard https://opentelemetry.io[Op<tracerConfig name="tracerConfig" class="org.apache.solr.opentelemetry.OtelTracerConfigurator"/>----+As an alternative to changing the `solr.xml` file, the `OTEL` tracer will be enabled if the system property `otel.service.name` or environment variable `OTEL_SERVICE_NAME` is present. The `opentelemetry` module still needs to be enabled for the tracer to work.+Enable the module with either system property `-Dsolr.modules=opentelemetry` or environment variable `SOLR_MODULES=opentelemetry`.=== Configurationdiff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.javaindex fdc64237a80..3da6dd7a23e 100644--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java@@ -1288,6 +1288,13 @@ public class MiniSolrCloudCluster {return this;}+ /**+ * Disables the default/built-in simple trace ID generation/propagation.+ *+ * <p>Tracers are registered as global singletons and if for example a test needs to use a+ * MockTracer or a "real" Tracer, it needs to call this method so that the test setup doesn't+ * accidentally reset the Tracer it wants to use.+ */public Builder withTraceIdGenerationDisabled() {this.disableTraceIdGeneration = true;return this;