Apache Solr open-source search software
SOLR-15283 Tracing overhaul (#52)
David Smiley69b41878551a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 04c237389ab..fd6f6a6d971 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -271,6 +271,12 @@ Other Changes* SOLR-15356: UninvertDocValuesMergePolicy was removed because it's incompatible with Lucene 9.(David Smiley, Christine Poerschke)+* SOLR-15238: Overhaul distributed-tracing. Solr's "samplePercentage" is gone; tracing is either+ enabled and you always have trace IDs or it isn't, based on wether you configure a tracing plugin.+ Reporting of traces to a server like Zipkin is a configuration matter of the tracing plugin you+ use. The Jaeger plugin is now completely configurable via system properties or environment vars,+ not by values in solr.xml.+Bug Fixes---------------------* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order executiondiff --git a/solr/contrib/jaegertracer-configurator/README.md b/solr/contrib/jaegertracer-configurator/README.mdindex 998b65f4567..f6cf87c26bb 100644--- a/solr/contrib/jaegertracer-configurator/README.md+++ b/solr/contrib/jaegertracer-configurator/README.md@@ -5,29 +5,25 @@ Apache Solr Jaeger Tracer Configurator (solr-jaegertracer) provides a way for yo# Setup Jaeger Tracer Configurator-Note that all library of solr-jaegertracer must be included in the classpath of all nodes then Jaeger tracer can be setup in `solr.xml` like this:+Add the solr-jaegertracer JAR file and the other JARs provided with this module to your Solr installation, ideally to each node.+GSON is a dependency that is only used by Jaeger's "remote" sampler,+which is the default. Solr doesn't distribute it, so you'll need to add GSON yourself or configure a different sampler.++The Jaeger Tracer Configurator is added to `solr.xml` like this:```-<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">- <str name="agentHost">localhost</str>- <int name="agentPort">5775</int>- <bool name="logSpans">true</bool>- <int name="flushInterval">1000</int>- <int name="maxQueueSize">10000</int>-</tracerConfig>+<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />```-List of parameters for JaegerTracerConfigurator include:+There are no configuration elements in the XML; instead, this 3rd party system is configured using System Properties or Environment Variables. The full list are listed at [Jaeger-README](https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md).+For example, to use the probabilistic sampler, you could set this environment variable:-|Parameter|Type|Required|Default|Description|-|---------|----|--------|-------|-----------|-|agentHost|string|Yes||The host of Jaeger backend|-|agentPort|int|Yes||The port of Jaeger port|-|logsSpans|bool|No|true|Whether the tracer should also log the spans|-|flushInterval|int|No|5000|The tracer's flush interval (ms)|-|maxQueueSize|int|No|10000|The tracer's maximum queue size|+```+export JAEGER_SAMPLER_TYPE=probabilistic+```-Other parameters which are not listed above can be configured using System Properties or Environment Variables. The full list are listed at [Jaeger-README](https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md).+or System property:-By default the sampling rate is 0.1%, this value can be changed by updating-key `samplePercentage` of cluster properties. I.e: `/admin/collections?action=CLUSTERPROP&name=propertyName&samplePercentage=100`.\ No newline at end of file+```+bin/solr start -DJAEGER_SAMPLER_TYPE=probabilistic+```\ No newline at end of filediff --git a/solr/contrib/jaegertracer-configurator/build.gradle b/solr/contrib/jaegertracer-configurator/build.gradleindex d2094b1bc34..8d145392c36 100644--- a/solr/contrib/jaegertracer-configurator/build.gradle+++ b/solr/contrib/jaegertracer-configurator/build.gradle@@ -24,7 +24,7 @@ dependencies {implementation ("io.jaegertracing:jaeger-thrift", {exclude group: "com.squareup.okhttp3", module: "okhttp"- exclude group: "com.google.code.gson", module: "gson"+ exclude group: "com.google.code.gson", module: "gson" // for "RemoteControlledSampler", optionalexclude group: "javax.annotation", module: "javax.annotation-api"})diff --git a/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java b/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.javaindex fdbb8bbde9a..f52e5806d99 100644--- a/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java+++ b/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java@@ -18,72 +18,20 @@package org.apache.solr.jaeger;import io.jaegertracing.Configuration;-import io.jaegertracing.internal.samplers.ConstSampler;import io.opentracing.Tracer;import org.apache.solr.common.util.NamedList;import org.apache.solr.core.TracerConfigurator;public class JaegerTracerConfigurator extends TracerConfigurator {- public static final String AGENT_HOST = "agentHost";- public static final String AGENT_PORT = "agentPort";- public static final String LOG_SPANS = "logSpans";- public static final String FLUSH_INTERVAL = "flushInterval";- public static final String MAX_QUEUE_SIZE = "maxQueueSize";-- private volatile Tracer tracer;@Overridepublic Tracer getTracer() {- return tracer;+ // See javadocs on Configuration for how to configure via sys props or env vars+ return Configuration.fromEnv("solr").getTracer();}@Overridepublic void init(@SuppressWarnings({"rawtypes"})NamedList args) {- Object host = args.get(AGENT_HOST);- if (!(host instanceof String)) {- throw new IllegalArgumentException("Expected a required string for param '" + AGENT_HOST + "'");- }-- Object portArg = args.get(AGENT_PORT);- if (!(portArg instanceof Integer)) {- throw new IllegalArgumentException("Expected a required int for param '" + AGENT_PORT + "'");- }- int port = (Integer) portArg;-- Boolean logSpans = args.getBooleanArg(LOG_SPANS);- if (logSpans == null)- logSpans = true;-- Object flushIntervalArg = args.get(FLUSH_INTERVAL);- if (flushIntervalArg != null && !(flushIntervalArg instanceof Integer)) {- throw new IllegalArgumentException("Expected a required int for param '" + FLUSH_INTERVAL +"'");- }- int flushInterval = flushIntervalArg == null ? 1000 : (Integer) flushIntervalArg;-- Object maxQueueArgs = args.get(MAX_QUEUE_SIZE);- if (maxQueueArgs != null && !(maxQueueArgs instanceof Integer)) {- throw new IllegalArgumentException("Expected a required int for param '" + MAX_QUEUE_SIZE +"'");- }- int maxQueue = maxQueueArgs == null ? 10000 : (Integer) maxQueueArgs;-- Configuration.SamplerConfiguration samplerConfig = new Configuration.SamplerConfiguration()- .withType(ConstSampler.TYPE)- .withParam(1);-- Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv();- Configuration.SenderConfiguration senderConfig = reporterConfig.getSenderConfiguration()- .withAgentHost(host.toString())- .withAgentPort(port);-- reporterConfig.withLogSpans(logSpans)- .withFlushInterval(flushInterval)- .withMaxQueueSize(maxQueue)- .withSender(senderConfig);- tracer = new Configuration("solr")- .withSampler(samplerConfig)- .withReporter(reporterConfig)- .getTracer();}-}diff --git a/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml b/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xmlindex 34729c300e6..3f2932e784e 100644--- a/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml+++ b/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml@@ -34,13 +34,7 @@<str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str></shardHandlerFactory>- <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">- <str name="agentHost">localhost</str>- <int name="agentPort">5775</int>- <bool name="logSpans">true</bool>- <int name="flushInterval">1000</int>- <int name="maxQueueSize">10000</int>- </tracerConfig>+ <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" /><solrcloud><str name="host">127.0.0.1</str>diff --git a/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java b/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.javaindex a0867f78dd6..9fc8d66de10 100644--- a/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java+++ b/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java@@ -17,43 +17,51 @@package org.apache.solr.jaeger;-import java.io.Closeable;-import java.io.IOException;import java.util.concurrent.TimeUnit;+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;+import io.opentracing.util.GlobalTracer;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrQuery;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.client.solrj.request.UpdateRequest;import org.apache.solr.cloud.MiniSolrCloudCluster;import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.SolrException;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.common.util.TimeSource;import org.apache.solr.util.TimeOut;-import org.apache.solr.util.tracing.GlobalTracer;+import org.junit.Before;+import org.junit.Rule;import org.junit.Test;--import static org.apache.solr.jaeger.JaegerTracerConfigurator.AGENT_HOST;-import static org.apache.solr.jaeger.JaegerTracerConfigurator.AGENT_PORT;-import static org.apache.solr.jaeger.JaegerTracerConfigurator.FLUSH_INTERVAL;-import static org.apache.solr.jaeger.JaegerTracerConfigurator.LOG_SPANS;-import static org.apache.solr.jaeger.JaegerTracerConfigurator.MAX_QUEUE_SIZE;+import org.junit.rules.TestRule;public class TestJaegerConfigurator extends SolrTestCaseJ4 {+ @Rule+ public TestRule solrTestRules = new SystemPropertiesRestoreRule();++ @Before+ public void doBefore() {+ // to be safe because this test tests tracing.+ resetGlobalTracer();+ ExecutorUtil.resetThreadLocalProviders();++ // The default sampler is one that uses GSON but we don't bother with that dependency.+ // So we change it to something simple.+ System.setProperty("JAEGER_SAMPLER_TYPE", "const");+ }+@Test- public void testInjected() throws Exception{+ public void testInjected() throws Exception {MiniSolrCloudCluster cluster = new SolrCloudTestCase.Builder(2, createTempDir()).addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")).withSolrXml(getFile("solr/solr.xml").toPath()).build();- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "100.0")- .process(cluster.getSolrClient());try {TimeOut timeOut = new TimeOut(2, TimeUnit.MINUTES, TimeSource.NANO_TIME);- timeOut.waitFor("Waiting for GlobalTracer is registered", () -> GlobalTracer.getTracer() instanceof io.jaegertracing.internal.JaegerTracer);+ timeOut.waitFor(+ "Waiting for GlobalTracer is registered",+ () -> GlobalTracer.get().toString().contains("JaegerTracer"));//TODO add run Jaeger through Docker and verify spans available after run these commandsCollectionAdminRequest.createCollection("test", 2, 1).process(cluster.getSolrClient());@@ -67,70 +75,4 @@ public class TestJaegerConfigurator extends SolrTestCaseJ4 {}}-- @Test- @SuppressWarnings({"unchecked"})- public void testRequiredParameters() throws IOException {- JaegerTracerConfigurator configurator = new JaegerTracerConfigurator();- @SuppressWarnings({"rawtypes"})- NamedList initArgs = new NamedList();- IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(AGENT_HOST) || exc.getMessage().contains(AGENT_PORT));- initArgs.add(AGENT_HOST, "localhost");-- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(AGENT_PORT));- initArgs.add(AGENT_PORT, 5775);-- // no exception should be thrown- configurator.init(initArgs);- ((Closeable)configurator.getTracer()).close();-- initArgs.add(LOG_SPANS, true);- initArgs.add(FLUSH_INTERVAL, 1000);- initArgs.add(MAX_QUEUE_SIZE, 10000);- configurator.init(initArgs);- ((Closeable)configurator.getTracer()).close();- }-- @Test- @SuppressWarnings({"unchecked"})- public void testIncorrectFormat() {- JaegerTracerConfigurator configurator = new JaegerTracerConfigurator();- @SuppressWarnings({"rawtypes"})- NamedList initArgs = new NamedList();- initArgs.add(AGENT_HOST, 100);- initArgs.add(AGENT_PORT, 5775);-- IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(AGENT_HOST));-- initArgs.clear();- initArgs.add(AGENT_HOST, "localhost");- initArgs.add(AGENT_PORT, "5775");- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(AGENT_PORT));-- initArgs.clear();- initArgs.add(AGENT_HOST, "localhost");- initArgs.add(AGENT_PORT, 5775);- initArgs.add(LOG_SPANS, 10);- SolrException solrExc = expectThrows(SolrException.class, () -> configurator.init(initArgs));- assertTrue(solrExc.getMessage().contains(LOG_SPANS));-- initArgs.clear();- initArgs.add(AGENT_HOST, "localhost");- initArgs.add(AGENT_PORT, 5775);- initArgs.add(FLUSH_INTERVAL, "10");- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(FLUSH_INTERVAL));-- initArgs.clear();- initArgs.add(AGENT_HOST, "localhost");- initArgs.add(AGENT_PORT, 5775);- initArgs.add(MAX_QUEUE_SIZE, "10");- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));- assertTrue(exc.getMessage().contains(MAX_QUEUE_SIZE));-- }}diff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 6bb61993bbd..6eba46c710e 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -77,7 +77,7 @@ dependencies {implementation 'com.tdunning:t-digest'- implementation 'io.opentracing:opentracing-api'+ api 'io.opentracing:opentracing-api' // Tracer is exposed on some methodsimplementation 'io.opentracing:opentracing-noop'implementation 'io.opentracing:opentracing-util'implementation 'org.apache.commons:commons-exec'diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.javaindex 21ffc7497b9..aca17a88993 100644--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java@@ -48,6 +48,9 @@ import java.util.stream.Collectors;import com.google.common.annotations.VisibleForTesting;import com.google.common.collect.ImmutableMap;import com.google.common.collect.Maps;+import io.opentracing.Tracer;+import io.opentracing.noop.NoopTracer;+import io.opentracing.noop.NoopTracerFactory;import org.apache.commons.lang3.StringUtils;import org.apache.http.auth.AuthSchemeProvider;import org.apache.http.client.CredentialsProvider;@@ -165,6 +168,12 @@ public class CoreContainer {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());+ {+ // Declared up top to ensure this is present before anything else.+ // note: will not be re-added if already there+ ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());+ }+final SolrCores solrCores = new SolrCores(this);public static class CoreLoadFailure {@@ -245,6 +254,8 @@ public class CoreContainer {protected volatile SolrMetricsContext solrMetricsContext;+ protected volatile Tracer tracer = NoopTracerFactory.create();+protected MetricsHandler metricsHandler;protected volatile MetricsHistoryHandler metricsHistoryHandler;@@ -647,6 +658,11 @@ public class CoreContainer {return metricsHistoryHandler;}+ /** Never null but may implement {@link NoopTracer}. */+ public Tracer getTracer() {+ return tracer;+ }+public OrderedExecutor getReplayUpdatesExecutor() {return replayUpdatesExecutor;}@@ -719,6 +735,8 @@ public class CoreContainer {String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);solrMetricsContext = new SolrMetricsContext(metricManager, registryName, metricTag);+ tracer = TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo());+coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(coreContainerWorkExecutor, null,metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)),@@ -751,7 +769,6 @@ public class CoreContainer {(PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));// use deprecated API for back-compat, remove in 9.0pkiAuthenticationPlugin.initializeMetrics(solrMetricsContext, "/authentication/pki");- TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader());packageLoader = new PackageLoader(this);containerHandlers.getApiBag().registerObject(packageLoader.getPackageAPI().editAPI);containerHandlers.getApiBag().registerObject(packageLoader.getPackageAPI().readAPI);@@ -1240,6 +1257,8 @@ public class CoreContainer {org.apache.lucene.util.IOUtils.closeWhileHandlingException(packageLoader);}org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort++ tracer.close();}public void cancelCoreRecoveries() {@@ -2247,10 +2266,6 @@ public class CoreContainer {return this.distributedCollectionCommandRunner;}- static {- ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());- }-/*** Run an arbitrary task in it's own thread. This is an expert option and is* a method you should use with great care. It would be bad to run something that never stoppeddiff --git a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.javaindex d5ee7a3ecff..91920443568 100644--- a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java+++ b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java@@ -18,53 +18,85 @@package org.apache.solr.core;import java.lang.invoke.MethodHandles;+import java.util.concurrent.atomic.AtomicReference;+import io.opentracing.Scope;+import io.opentracing.Span;import io.opentracing.Tracer;-import io.opentracing.noop.NoopTracerFactory;-import org.apache.solr.common.cloud.ZkStateReader;+import io.opentracing.util.GlobalTracer;+import org.apache.solr.common.util.ExecutorUtil;import org.apache.solr.util.plugin.NamedListInitializedPlugin;-import org.apache.solr.util.tracing.GlobalTracer;import org.slf4j.Logger;import org.slf4j.LoggerFactory;+/** Produces an OpenTracing {@link Tracer} from configuration. */public abstract class TracerConfigurator implements NamedListInitializedPlugin {private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());public abstract Tracer getTracer();- public static void loadTracer(SolrResourceLoader loader, PluginInfo info, ZkStateReader stateReader) {- if (info == null) {- // in case of a Tracer is registered to OpenTracing through javaagent- if (io.opentracing.util.GlobalTracer.isRegistered()) {- GlobalTracer.setup(io.opentracing.util.GlobalTracer.get());- registerListener(stateReader);- } else {- GlobalTracer.setup(NoopTracerFactory.create());- GlobalTracer.get().setSamplePercentage(0.0);- }- } else {- TracerConfigurator configurator = loader- .newInstance(info.className, TracerConfigurator.class);- configurator.init(info.initArgs);+ public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) {+ // In "normal" Solr operation, loadTracer is called once in Solr's lifetime.+ // In test mode we run many test suites, sometimes multiple servers per test suite.+ // It's important that the tracing config not change throughout a test suite because of the+ // static singleton pattern and assumptions based on this.- GlobalTracer.setup(configurator.getTracer());- registerListener(stateReader);+ if (info != null && info.isEnabled()) {+ GlobalTracer.registerIfAbsent(() -> {+ TracerConfigurator configurator =+ loader.newInstance(info.className, TracerConfigurator.class);+ configurator.init(info.initArgs);+ return configurator.getTracer();+ });+ }+ if (GlobalTracer.isRegistered()) {+ // ideally we would furthermore check that it's not a no-op impl either but+ // GlobalTracer.get() always returns a GlobalTracer implementing Tracer that delegates+ // to the real Tracer (that may or may not be a No-Op impl.+ ExecutorUtil.addThreadLocalProvider(new TracerConfigurator.SpanThreadLocalProvider());}++ return GlobalTracer.get();}- private static void registerListener(ZkStateReader stateReader) {- stateReader.registerClusterPropertiesListener(properties -> {- if (properties.containsKey(ZkStateReader.SAMPLE_PERCENTAGE)) {- try {- double sampleRate = Double.parseDouble(properties.get(ZkStateReader.SAMPLE_PERCENTAGE).toString());- GlobalTracer.get().setSamplePercentage(sampleRate);- } catch (NumberFormatException e) {- log.error("Unable to set sample rate", e);- }- } else {- GlobalTracer.get().setSamplePercentage(0.1);+ /**+ * Propagate the active span across threads. New spans are not created, which means that we're+ * possibly exposing a Span to a thread that may find that it has already finished, depending on+ * how the instrumented thread pool is used. It's probably fine to create new spans related to a+ * finished span? It's not okay to otherwise touch a finished span (e.g. to log or tag).+ *+ * <p>This strategy is also used by {@code+ * brave.propagation.CurrentTraceContext#wrap(java.lang.Runnable)}.+ */+ private static class SpanThreadLocalProvider+ implements ExecutorUtil.InheritableThreadLocalProvider {+ private final Tracer tracer = GlobalTracer.get();++ @SuppressWarnings({"rawtypes", "unchecked"})+ @Override+ public void store(AtomicReference ctx) {+ assert tracer == GlobalTracer.get() : "Tracer changed; not supported!";+ ctx.set(tracer.scopeManager().activeSpan());+ }++ @SuppressWarnings({"rawtypes", "unchecked"})+ @Override+ public void set(AtomicReference ctx) {+ final Span span = (Span) ctx.get();+ if (span != null) {+ log.trace("Thread received span to do async work: {}", span);+ final Scope scope = tracer.scopeManager().activate(span);+ ctx.set(scope);}- return false;- });+ }++ @SuppressWarnings({"rawtypes", "unchecked"})+ @Override+ public void clean(AtomicReference ctx) {+ Scope scope = (Scope) ctx.get();+ if (scope != null) {+ scope.close();+ }+ }}}diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.javaindex 5aa80120ab0..6207a63402b 100644--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java@@ -49,7 +49,6 @@ import org.apache.solr.common.util.NamedList;import org.apache.solr.core.CoreDescriptor;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.request.SolrRequestInfo;-import org.apache.solr.util.tracing.GlobalTracer;import org.apache.solr.util.tracing.SolrRequestCarrier;@SolrThreadUnsafe@@ -125,8 +124,8 @@ public class HttpShardHandler extends ShardHandler {public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {// do this outside of the callable for thread safety reasonsfinal List<String> urls = getURLs(shard);- final Tracer tracer = GlobalTracer.getTracer();- final Span span = tracer != null ? tracer.activeSpan() : null;+ final Tracer tracer = sreq.tracer; // not null+ final Span span = tracer.activeSpan(); // probably not null?params.remove(CommonParams.WT); // use default (currently javabin)params.remove(CommonParams.VERSION);@@ -162,7 +161,7 @@ public class HttpShardHandler extends ShardHandler {@Overridepublic void onStart() {- if (tracer != null && span != null) {+ if (span != null) {tracer.inject(span.context(), Format.Builtin.HTTP_HEADERS, new SolrRequestCarrier(req));}SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.javaindex f7c05d236bf..0e152bde04d 100644--- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java+++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java@@ -16,6 +16,8 @@*/package org.apache.solr.handler.component;+import io.opentracing.Tracer;+import io.opentracing.util.GlobalTracer;import org.apache.solr.common.params.ModifiableSolrParams;import java.util.ArrayList;@@ -59,6 +61,9 @@ public class ShardRequest {/** may be null */public String nodeName;+ /** Not null but may implement {@link io.opentracing.noop.NoopTracer}. */+ public final Tracer tracer = GlobalTracer.get();+// TODO: one could store a list of numbers to correlate where returned docs// go in the top-level response rather than looking up by id...// this would work well if we ever transitioned to using internal ids anddiff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.javaindex 2c5090ba16a..394764868ac 100644--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java@@ -16,6 +16,8 @@*/package org.apache.solr.request;+import io.opentracing.Tracer;+import io.opentracing.util.GlobalTracer;import org.apache.solr.search.SolrIndexSearcher;import org.apache.solr.schema.IndexSchema;import org.apache.solr.common.params.SolrParams;@@ -132,6 +134,14 @@ public interface SolrQueryRequest extends AutoCloseable {default HttpSolrCall getHttpSolrCall() {return null;}++ /**+ * Distributed tracing Tracer. Never null but might implement+ * {@link io.opentracing.noop.NoopTracer}.+ */+ default Tracer getTracer() {+ return GlobalTracer.get(); // default impl is only for some tests+ }}diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.javaindex 76e8733c550..ac3acf37af2 100644--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java@@ -23,6 +23,8 @@ import java.util.HashMap;import java.util.List;import java.util.Map;+import io.opentracing.Tracer;+import io.opentracing.util.GlobalTracer;import org.apache.solr.api.ApiBag;import org.apache.solr.common.SolrException;import org.apache.solr.common.params.SolrParams;@@ -34,6 +36,7 @@ import org.apache.solr.common.util.ValidatingJsonMap;import org.apache.solr.core.SolrCore;import org.apache.solr.schema.IndexSchema;import org.apache.solr.search.SolrIndexSearcher;+import org.apache.solr.servlet.HttpSolrCall;import org.apache.solr.util.RTimerTree;import org.apache.solr.util.RefCounted;@@ -139,6 +142,21 @@ public abstract class SolrQueryRequestBase implements SolrQueryRequest, Closeablreturn schema;}+ @Override+ public Tracer getTracer() {+ if (core != null) {+ return core.getCoreContainer().getTracer(); // this is the common path+ }+ final HttpSolrCall call = getHttpSolrCall();+ if (call != null) {+ final Tracer tracer = (Tracer) call.getReq().getAttribute(Tracer.class.getName());+ if (tracer != null) {+ return tracer;+ }+ }+ return GlobalTracer.get(); // this way is not ideal (particularly in tests) but it's okay+ }+@Overridepublic void updateSchemaToLatest() {schema = core.getLatestSchema();diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.javaindex ee7c9f4b5ac..548b00d5720 100644--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java@@ -109,7 +109,6 @@ import org.apache.solr.servlet.cache.Method;import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;import org.apache.solr.util.RTimerTree;import org.apache.solr.util.TimeOut;-import org.apache.solr.util.tracing.GlobalTracer;import org.apache.zookeeper.KeeperException;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -501,10 +500,8 @@ public class HttpSolrCall {*/public Action call() throws IOException {MDCLoggingContext.reset();- Span activeSpan = GlobalTracer.getTracer().activeSpan();- if (activeSpan != null) {- MDCLoggingContext.setTracerId(activeSpan.context().toTraceId());- }+ Span activeSpan = (Span) req.getAttribute(Span.class.getName()); // never null+ MDCLoggingContext.setTracerId(activeSpan.context().toTraceId()); // handles empty stringMDCLoggingContext.setNode(cores);@@ -706,7 +703,7 @@ public class HttpSolrCall {// encoding issues with Tomcatif (header != null && !header.getName().equalsIgnoreCase(TRANSFER_ENCODING_HEADER)&& !header.getName().equalsIgnoreCase(CONNECTION_HEADER)) {-+// NOTE: explicitly using 'setHeader' instead of 'addHeader' so that// the remote nodes values for any response headers will overide any that// may have already been set locally (ex: by the local jetty's RewriteHandler config)diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.javaindex 3a294c5985f..87ad3d5abde 100644--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java@@ -58,10 +58,9 @@ import com.codahale.metrics.jvm.GarbageCollectorMetricSet;import com.codahale.metrics.jvm.MemoryUsageGaugeSet;import com.codahale.metrics.jvm.ThreadStatesGaugeSet;import com.google.common.annotations.VisibleForTesting;-import io.opentracing.Scope;import io.opentracing.Span;-import io.opentracing.SpanContext;import io.opentracing.Tracer;+import io.opentracing.propagation.Format;import io.opentracing.tag.Tags;import org.apache.commons.lang3.StringUtils;import org.apache.http.HttpHeaders;@@ -90,7 +89,7 @@ import org.apache.solr.security.PKIAuthenticationPlugin;import org.apache.solr.security.PublicKeyHandler;import org.apache.solr.util.StartupLoggingUtils;import org.apache.solr.util.configuration.SSLConfigurationsFactory;-import org.apache.solr.util.tracing.GlobalTracer;+import org.apache.solr.util.tracing.HttpServletCarrier;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -423,7 +422,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {httpClient = null;cc.shutdown();}- GlobalTracer.get().close();}}@@ -436,10 +434,31 @@ public class SolrDispatchFilter extends BaseSolrFilter {if (!(_request instanceof HttpServletRequest)) return;HttpServletRequest request = closeShield((HttpServletRequest)_request, retry);HttpServletResponse response = closeShield((HttpServletResponse)_response, retry);- Scope scope = null;- Span span = null;++ String requestPath = ServletUtils.getPathAfterContext(request);+ // No need to even create the HttpSolrCall object if this path is excluded.+ if (excludePatterns != null) {+ for (Pattern p : excludePatterns) {+ Matcher matcher = p.matcher(requestPath);+ if (matcher.lookingAt()) {+ chain.doFilter(request, response);+ return;+ }+ }+ }++ Tracer tracer = cores.getTracer();+ String hostAndPort = request.getServerName() + "_" + request.getServerPort();+ Span span = tracer.buildSpan(request.getMethod() + ":" + hostAndPort)+ .asChildOf(tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request)))+ .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER)+ .withTag(Tags.HTTP_URL, request.getRequestURL().toString())+ .start();+ request.setAttribute(Tracer.class.getName(), tracer);+ request.setAttribute(Span.class.getName(), span);boolean accepted = false;- try {+ try (var scope = tracer.scopeManager().activate(span)) {+ assert scope != null; // prevent javac warning about scope being unusedif (cores == null || cores.isShutDown()) {try {@@ -453,18 +472,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {}}- String requestPath = ServletUtils.getPathAfterContext(request);- // No need to even create the HttpSolrCall object if this path is excluded.- if (excludePatterns != null) {- for (Pattern p : excludePatterns) {- Matcher matcher = p.matcher(requestPath);- if (matcher.lookingAt()) {- chain.doFilter(request, response);- return;- }- }- }-try {accepted = rateLimitManager.handleRequest(request);} catch (InterruptedException e) {@@ -479,24 +486,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {response.sendError(429, errorMessage);}- SpanContext parentSpan = GlobalTracer.get().extract(request);- Tracer tracer = GlobalTracer.getTracer();-- Tracer.SpanBuilder spanBuilder = null;- String hostAndPort = request.getServerName() + "_" + request.getServerPort();- if (parentSpan == null) {- spanBuilder = tracer.buildSpan(request.getMethod() + ":" + hostAndPort);- } else {- spanBuilder = tracer.buildSpan(request.getMethod() + ":" + hostAndPort)- .asChildOf(parentSpan);- }-- spanBuilder- .withTag(Tags.SPAN_KIND.getKey(), Tags.SPAN_KIND_SERVER)- .withTag(Tags.HTTP_URL.getKey(), request.getRequestURL().toString());- span = spanBuilder.start();- scope = tracer.scopeManager().activate(span);-AtomicReference<HttpServletRequest> wrappedRequest = new AtomicReference<>();if (!authenticateRequest(request, response, wrappedRequest)) { // the response and status code have already been sentreturn;@@ -537,10 +526,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {ExecutorUtil.setServerThreadFlag(null);}} finally {- if (span != null) span.finish();- if (scope != null) scope.close();-- GlobalTracer.get().clearContext();consumeInputFully(request, response);SolrRequestInfo.reset();SolrRequestParsers.cleanupMultipartFiles(request);@@ -548,6 +533,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {if (accepted) {rateLimitManager.decrementActiveRequests(request);}+ span.finish();}}diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.javaindex 96cc66dd2c8..c81c32091d2 100644--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java@@ -41,6 +41,8 @@ import java.util.LinkedList;import java.util.List;import java.util.Map;+import io.opentracing.Span;+import io.opentracing.noop.NoopSpan;import org.apache.commons.io.input.CloseShieldInputStream;import org.apache.lucene.util.IOUtils;import org.apache.solr.api.V2HttpCall;@@ -59,7 +61,6 @@ import org.apache.solr.core.SolrCore;import org.apache.solr.request.SolrQueryRequest;import org.apache.solr.request.SolrQueryRequestBase;import org.apache.solr.util.RTimerTree;-import org.apache.solr.util.tracing.GlobalTracer;import org.eclipse.jetty.http.HttpFields;import org.eclipse.jetty.http.MimeTypes;import org.eclipse.jetty.server.Request;@@ -169,8 +170,10 @@ public class SolrRequestParsers {// Pick the parser from the request...ArrayList<ContentStream> streams = new ArrayList<>(1);SolrParams params = parser.parseParamsAndFillStreams( req, streams );- if (GlobalTracer.get().tracing()) {- GlobalTracer.getTracer().activeSpan().setTag("params", params.toString());++ Span span = (Span) req.getAttribute(Span.class.getName()); // not null but maybe in some tests?+ if (span != null && !(span instanceof NoopSpan)) {+ span.setTag("params", params.toString());}SolrQueryRequest sreq = buildRequestFrom(core, params, streams, getRequestTimer(req), req);diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.javaindex 9d2377efdea..ea2ae7b654e 100644--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java@@ -53,7 +53,6 @@ import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.update.processor.DistributedUpdateProcessor;import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;-import org.apache.solr.util.tracing.GlobalTracer;import org.apache.solr.util.tracing.SolrRequestCarrier;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -294,7 +293,7 @@ public class SolrCmdDistributor implements Closeable {req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal());}- Tracer tracer = GlobalTracer.getTracer();+ Tracer tracer = req.cmd.getTracer();Span parentSpan = tracer.activeSpan();if (parentSpan != null) {tracer.inject(parentSpan.context(), Format.Builtin.HTTP_HEADERS,diff --git a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java b/solr/core/src/java/org/apache/solr/update/UpdateCommand.javaindex b124271d977..29f695314f2 100644--- a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java+++ b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java@@ -16,6 +16,8 @@*/package org.apache.solr.update;+import io.opentracing.Tracer;+import io.opentracing.util.GlobalTracer;import org.apache.solr.request.SolrQueryRequest;@@ -90,6 +92,14 @@ public abstract class UpdateCommand implements Cloneable {this.req = req;}+ /**+ * Distributed tracing Tracer. Never null but might implement+ * {@link io.opentracing.noop.NoopTracer}.+ */+ public Tracer getTracer() {+ return (req != null) ? req.getTracer() : GlobalTracer.get();+ }+@Overridepublic UpdateCommand clone() {try {diff --git a/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.java b/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.javadeleted file mode 100644index bc9ffe51702..00000000000--- a/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.java+++ /dev/null@@ -1,120 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.util.tracing;--import javax.servlet.http.HttpServletRequest;-import java.util.Random;--import com.google.common.annotations.VisibleForTesting;-import io.opentracing.SpanContext;-import io.opentracing.Tracer;-import io.opentracing.noop.NoopTracerFactory;-import io.opentracing.propagation.Format;--public class GlobalTracer {- private static final Tracer NOOP_TRACER = NoopTracerFactory.create();- private static final Random RANDOM;- static {- // We try to make things reproducible in the context of our tests by initializing the random instance- // based on the current seed- String seed = System.getProperty("tests.seed");- if (seed == null) {- RANDOM = new Random();- } else {- RANDOM = new Random(seed.hashCode());- }- }-- private static volatile GlobalTracer INS = new GlobalTracer(NOOP_TRACER);-- public synchronized static void setup(Tracer tracer) {- if (INS != null) {- INS.close();- }- INS = new GlobalTracer(tracer);- }-- public static GlobalTracer get() {- return INS;- }-- public static Tracer getTracer() {- return INS.tracer();- }-- @VisibleForTesting- final Tracer tracer;- private double rate;- private final ThreadLocal<Tracer> threadLocal = new ThreadLocal<>();-- public GlobalTracer(Tracer tracer) {- this.tracer = tracer;- }-- public synchronized void setSamplePercentage(double rate) {- this.rate = rate / 100.0;- }-- @VisibleForTesting- public double getSampleRate() {- return rate;- }-- public boolean tracing() {- return threadLocal.get() != null- && threadLocal.get() != NOOP_TRACER- && threadLocal.get().activeSpan() != null;- }-- public SpanContext extract(HttpServletRequest request) {- SpanContext spanContext = tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request));- if (spanContext != null) {- threadLocal.set(tracer);- }- return spanContext;- }-- private Tracer tracer() {- Tracer tracer = threadLocal.get();- if (tracer != null)- return tracer;-- if (traced()) {- threadLocal.set(this.tracer);- } else {- threadLocal.set(NOOP_TRACER);- }-- return threadLocal.get();- }-- /**- * Clear tracing context for the current thread- */- public void clearContext() {- threadLocal.remove();- }-- private boolean traced() {- return RANDOM.nextDouble() <= rate;- }-- public void close() {- tracer.close();- }-}diff --git a/solr/core/src/test-files/solr/solr-tracing.xml b/solr/core/src/test-files/solr/solr-tracing.xmldeleted file mode 100644index b33a4a85e6a..00000000000--- a/solr/core/src/test-files/solr/solr-tracing.xml+++ /dev/null@@ -1,57 +0,0 @@-<?xml version="1.0" encoding="UTF-8" ?>-<!--- Licensed to the Apache Software Foundation (ASF) under one or more- contributor license agreements. See the NOTICE file distributed with- this work for additional information regarding copyright ownership.- The ASF licenses this file to You under the Apache License, Version 2.0- (the "License"); you may not use this file except in compliance with- the License. You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.--->--<!--- All (relative) paths are relative to the installation path--->-<solr>-- <str name="shareSchema">${shareSchema:false}</str>- <str name="configSetBaseDir">${configSetBaseDir:configsets}</str>- <str name="coreRootDirectory">${coreRootDirectory:.}</str>- <str name="collectionsHandler">${collectionsHandler:solr.CollectionsHandler}</str>-- <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">- <str name="urlScheme">${urlScheme:}</str>- <int name="socketTimeout">${socketTimeout:90000}</int>- <int name="connTimeout">${connTimeout:15000}</int>- <str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str>- </shardHandlerFactory>-- <tracerConfig name="tracerConfig" class="MockTracerConfigurator">- </tracerConfig>-- <solrcloud>- <str name="host">127.0.0.1</str>- <int name="hostPort">${hostPort:8983}</int>- <str name="hostContext">${hostContext:solr}</str>- <int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>- <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>- <int name="leaderVoteWait">${leaderVoteWait:10000}</int>- <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:45000}</int>- <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:340000}</int>- <str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>- <str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>- </solrcloud>- <metrics>- <reporter name="default" class="org.apache.solr.metrics.reporters.SolrJmxReporter">- <str name="rootName">solr_${hostPort:8983}</str>- </reporter>- </metrics>--</solr>\ No newline at end of filediff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.javaindex fc5dcb2815a..8d1ebc2b216 100644--- a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java@@ -18,42 +18,39 @@package org.apache.solr.util.tracing;import java.io.IOException;-import java.lang.invoke.MethodHandles;import java.util.ArrayList;import java.util.List;-import java.util.concurrent.TimeUnit;import java.util.concurrent.TimeoutException;import java.util.stream.Collectors;import io.opentracing.mock.MockSpan;import io.opentracing.mock.MockTracer;+import io.opentracing.util.GlobalTracer;import org.apache.solr.client.solrj.SolrQuery;import org.apache.solr.client.solrj.SolrServerException;import org.apache.solr.client.solrj.impl.CloudSolrClient;import org.apache.solr.client.solrj.request.CollectionAdminRequest;import org.apache.solr.cloud.SolrCloudTestCase;-import org.apache.solr.common.cloud.ZkStateReader;-import org.apache.solr.common.util.TimeSource;-import org.apache.solr.util.TimeOut;+import org.apache.solr.util.LogLevel;+import org.junit.AfterClass;import org.junit.BeforeClass;import org.junit.Test;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;+@LogLevel("org.apache.solr.core.TracerConfigurator=trace")public class TestDistributedTracing extends SolrCloudTestCase {private static final String COLLECTION = "collection1";- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());++ static MockTracer tracer;@BeforeClasspublic static void beforeTest() throws Exception {+ tracer = new MockTracer();+ assertTrue(GlobalTracer.registerIfAbsent(tracer));+configureCluster(4).addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))- .withSolrXml(TEST_PATH().resolve("solr-tracing.xml")).configure();- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "100.0")- .process(cluster.getSolrClient());- waitForSampleRateUpdated(1.0);CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 2).setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)@@ -61,21 +58,18 @@ public class TestDistributedTracing extends SolrCloudTestCase {cluster.waitForActiveCollection(COLLECTION, 2, 4);}- private static void waitForSampleRateUpdated(double rate) throws TimeoutException, InterruptedException {- TimeOut timeOut = new TimeOut(1, TimeUnit.MINUTES, TimeSource.NANO_TIME);- timeOut.waitFor("Waiting for sample rate is updated", () ->- Math.abs(GlobalTracer.get().getSampleRate() - rate) < 0.001- && GlobalTracer.get().tracer instanceof MockTracer);- }-- private List<MockSpan> getFinishedSpans() {- return ((MockTracer)GlobalTracer.get().tracer).finishedSpans();+ @AfterClass+ public static void afterTest() {+ tracer = null;}@Testpublic void test() throws IOException, SolrServerException, TimeoutException, InterruptedException {+ // TODO it would be clearer if we could compare the complete Span tree between reality+ // and what we assert it looks like in a structured visual way.+CloudSolrClient cloudClient = cluster.getSolrClient();- List<MockSpan> allSpans = getFinishedSpans();+ List<MockSpan> allSpans = tracer.finishedSpans();cloudClient.add(COLLECTION, sdoc("id", "1"));List<MockSpan> finishedSpans = getRecentSpans(allSpans);@@ -112,15 +106,6 @@ public class TestDistributedTracing extends SolrCloudTestCase {fail("All spans must belong to single span, but:"+finishedSpans);}}-- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "0.0")- .process(cluster.getSolrClient());- waitForSampleRateUpdated(0);-- getRecentSpans(allSpans);- cloudClient.add(COLLECTION, sdoc("id", "5"));- finishedSpans = getRecentSpans(allSpans);- assertEquals(0, finishedSpans.size());}private void assertOneSpanIsChildOfAnother(List<MockSpan> finishedSpans) {@@ -136,10 +121,10 @@ public class TestDistributedTracing extends SolrCloudTestCase {}private List<MockSpan> getRecentSpans(List<MockSpan> allSpans) {- List<MockSpan> result = new ArrayList<>(getFinishedSpans());+ List<MockSpan> result = new ArrayList<>(tracer.finishedSpans());result.removeAll(allSpans);allSpans.clear();- allSpans.addAll(getFinishedSpans());+ allSpans.addAll(tracer.finishedSpans());return result;}}diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adocindex 85c0b05a819..4768539ec77 100644--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc@@ -192,6 +192,14 @@ set `-Dsolr.storeBaseUrl=true` so that Solr will continue to store the `base_urlSupport for the `solr.storeBaseUrl` system property will be removed in Solr 10.x and `base_url` will no longer be stored.+* Solr's distributed tracing no longer incorporates a special `samplePercentage` SolrCloud cluster property.+Instead, consult the documentation for the tracing system you use on how to sample the traces.+Consequently, if you use a Tracer at all, you will always have traces and thus trace IDs in logs.+What percentage of them get reported to a tracing server is up to you.++* JaegerTracerConfigurator no longer recognizes any configuration in solr.xml.+ It is now completely configured via System properties and/or Environment variables as documented by Jaeger.+=== Schema Changes in 9=== Authentication & Security Changes in Solr 9diff --git a/solr/solr-ref-guide/src/solr-tracing.adoc b/solr/solr-ref-guide/src/solr-tracing.adocindex 47477ed87c6..6a87bcdd3d7 100644--- a/solr/solr-ref-guide/src/solr-tracing.adoc+++ b/solr/solr-ref-guide/src/solr-tracing.adoc@@ -36,96 +36,43 @@ A setup of a `TracerConfigurator` in `solr.xml` looks like this:[source,xml]----<solr>- <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">- <str name="agentHost">localhost</str>- <int name="agentPort">5775</int>- <bool name="logSpans">true</bool>- <int name="flushInterval">1000</int>- <int name="maxQueueSize">10000</int>- </tracerConfig>+ <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" /></solr>----If `<tracerConfig>` is absent, `TracerConfigurator` will try to pick up the `Tracer` instance that was registered in `io.opentracing.util.GlobalTracer`.-By doing this some backends like DataDog are supported out of the box since+By doing this, some backends like DataDog are supported out of the box sincehttps://docs.datadoghq.com/tracing/setup/java/[datadog-java-agent] uses Javaagent to register a `Tracer` in`io.opentracing.util.GlobalTracer`.-=== Configuring Sample Rate-By default only 0.1% of requests are sampled, this ensures that tracing activities do not affect performance of a node.+== Jaeger Tracer Configurator-The rate can be changed without needing to restart Solr nodes by setting the new sample rate as a cluster property using the Collections API's <<cluster-node-management.adoc#clusterprop,CLUSTERPROP>> command.+The `contrib/jagertracer-configurator` provides a default implementation for setting up Jaeger Tracer.-For example, the following sets the `samplePercentage` property to 100%:+Add the solr-jaegertracer JAR file and the other JARs provided with this module to your Solr installation, ideally to each node.+GSON is a dependency that is only used by Jaeger's "remote" sampler,+which is the default. Solr doesn't distribute it, so you'll need to add GSON yourself or configure a different sampler.-[source,text]-/admin/collections?action=CLUSTERPROP&name=samplePercentage&val=100+Then Jaeger tracer is configured in `solr.xml` like this:-== Jaeger Tracer Configurator+[source,xml]+----+<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />+-----The `contrib/jagertracer-configurator` provides a default implementation for setting up Jaeger Tracer.+_There are no configuration elements in the XML_; instead, this 3rd party system is configured using System Properties or Environment Variables. The full list are listed at https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].-Note that all libraries of `jaegertracer-configurator` must be included in the classpath of all nodes.-Then Jaeger tracer can be set up in `solr.xml` like this:+For example, to use the probabilistic sampler, you could set this environment variable:-[source,xml]+[source,bash]-----<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">- <str name="agentHost">localhost</str>- <int name="agentPort">5775</int>- <bool name="logSpans">true</bool>- <int name="flushInterval">1000</int>- <int name="maxQueueSize">10000</int>-</tracerConfig>+export JAEGER_SAMPLER_TYPE=probabilistic-----The parameters for `JaegerTracerConfigurator` include:--`agentHost`::-+-[%autowidth,frame=none]-|===-s|Required |Default: none-|===-+-The Jaeger backend host.--`agentPort`::-+-[%autowidth,frame=none]-|===-s|Required |Default: none-|===-+-The Jaeger port.--`logSpans`::-+-[%autowidth,frame=none]-|===-|Optional |Default: `true`-|===-+-Whether the tracer should also log the spans.--`flushInterval`::-+-[%autowidth,frame=none]-|===-|Optional | Default: `5000` (ms)-|===-+-The tracer's flush interval.--`maxQueueSize`::-+-[%autowidth,frame=none]-|===-|Optional | Default: `10000`-|===-+-The tracer's maximum queue size.--Other parameters which are not listed above can be configured using system properties or environment variables.-The full list of parameters can be found in the https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].+or System property:++[source,bash]+----+bin/solr start -DJAEGER_SAMPLER_TYPE=probabilistic+----\ No newline at end of filediff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.javaindex b26183cea05..7ff0b3ae5c8 100644--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java@@ -129,8 +129,6 @@ public class ZkStateReader implements SolrCloseable {public static final String CONFIGS_ZKNODE = "/configs";public final static String CONFIGNAME_PROP = "configName";- public static final String SAMPLE_PERCENTAGE = "samplePercentage";-/*** @deprecated use {@link org.apache.solr.common.params.CollectionAdminParams#DEFAULTS} instead.*/@@ -227,7 +225,6 @@ public class ZkStateReader implements SolrCloseable {CoreAdminParams.BACKUP_LOCATION,DEFAULT_SHARD_PREFERENCES,MAX_CORES_PER_NODE,- SAMPLE_PERCENTAGE,SOLR_ENVIRONMENT,CollectionAdminParams.DEFAULTS,CONTAINER_PLUGINS,diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.javaindex 6664dbf02b3..cb23d94250a 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java@@ -41,6 +41,14 @@ public class ExecutorUtil {private static volatile List<InheritableThreadLocalProvider> providers = new ArrayList<>();+ /**+ * Resets everything added via {@link #addThreadLocalProvider(InheritableThreadLocalProvider)}.+ * Useful to call at the beginning of tests.+ */+ public static void resetThreadLocalProviders() {+ providers = new ArrayList<>();+ }+public synchronized static void addThreadLocalProvider(InheritableThreadLocalProvider provider) {for (InheritableThreadLocalProvider p : providers) {//this is to avoid accidental multiple addition of providers in testsif (p.getClass().equals(provider.getClass())) return;diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.javaindex 7aa14f421de..38268dcc4ad 100644--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java@@ -31,6 +31,7 @@ import java.lang.annotation.Retention;import java.lang.annotation.RetentionPolicy;import java.lang.annotation.Target;import java.lang.invoke.MethodHandles;+import java.lang.reflect.Field;import java.lang.reflect.InvocationTargetException;import java.lang.reflect.Method;import java.net.MalformedURLException;@@ -40,6 +41,8 @@ import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.Paths;+import java.security.AccessController;+import java.security.PrivilegedAction;import java.security.SecureRandom;import java.time.Instant;import java.util.ArrayList;@@ -70,6 +73,8 @@ import com.carrotsearch.randomizedtesting.RandomizedTest;import com.carrotsearch.randomizedtesting.TraceFormatting;import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;+import io.opentracing.noop.NoopTracerFactory;+import io.opentracing.util.GlobalTracer;import org.apache.commons.io.FileUtils;import org.apache.http.client.HttpClient;import org.apache.logging.log4j.Level;@@ -307,6 +312,35 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {} else {UrlScheme.INSTANCE.setUrlScheme(UrlScheme.HTTP);}++ resetGlobalTracer();+ ExecutorUtil.resetThreadLocalProviders();+ }++ /**+ * GlobalTracer is initialized by org.apache.solr.core.TracerConfigurator by+ * org.apache.solr.core.CoreContainer. Tests may need to reset it in the beginning of a test if+ * it might have differing configuration from other tests in the same suite.+ * It's also important to call {@link ExecutorUtil#resetThreadLocalProviders()}.+ */+ @SuppressForbidden(reason = "Hack to reset internal state of GlobalTracer")+ public static void resetGlobalTracer() {+ AccessController.doPrivileged((PrivilegedAction<Void>) () -> {+ try {+ final Class<GlobalTracer> globalTracerClass = GlobalTracer.class;+ final Field isRegistered = globalTracerClass.getDeclaredField("isRegistered");+ isRegistered.setAccessible(true);+ isRegistered.setBoolean(null, false);+ final Field tracer = globalTracerClass.getDeclaredField("tracer");+ tracer.setAccessible(true);+ tracer.set(null, NoopTracerFactory.create());+ } catch (NoSuchFieldException | IllegalAccessException e) {+ throw new RuntimeException(e);+ }+ return null;+ });++ assert GlobalTracer.isRegistered() == false;}@AfterClassdiff --git a/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.java b/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.javadeleted file mode 100644index b1c7cb9168d..00000000000--- a/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.java+++ /dev/null@@ -1,35 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--package org.apache.solr.core;--import io.opentracing.Tracer;-import io.opentracing.mock.MockTracer;-import org.apache.solr.common.util.NamedList;--public class MockTracerConfigurator extends TracerConfigurator {-- @Override- public Tracer getTracer() {- return new MockTracer();- }-- @Override- public void init(@SuppressWarnings({"rawtypes"})NamedList args) {-- }-}