Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15283 Tracing overhaul (#52)

David Smiley69b41878551a
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 04c237389ab..fd6f6a6d971 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -271,6 +271,12 @@ Other Changes
* SOLR-15356: UninvertDocValuesMergePolicy was removed because it's incompatible with Lucene 9.
(David Smiley, Christine Poerschke)
+* SOLR-15238: Overhaul distributed-tracing. Solr's "samplePercentage" is gone; tracing is either
+ enabled and you always have trace IDs or it isn't, based on wether you configure a tracing plugin.
+ Reporting of traces to a server like Zipkin is a configuration matter of the tracing plugin you
+ use. The Jaeger plugin is now completely configurable via system properties or environment vars,
+ not by values in solr.xml.
+
Bug Fixes
---------------------
* SOLR-14546: Fix for a relatively hard to hit issue in OverseerTaskProcessor that could lead to out of order execution
diff --git a/solr/contrib/jaegertracer-configurator/README.md b/solr/contrib/jaegertracer-configurator/README.md
index 998b65f4567..f6cf87c26bb 100644
--- a/solr/contrib/jaegertracer-configurator/README.md
+++ b/solr/contrib/jaegertracer-configurator/README.md
@@ -5,29 +5,25 @@ Apache Solr Jaeger Tracer Configurator (solr-jaegertracer) provides a way for yo
# Setup Jaeger Tracer Configurator
-Note that all library of solr-jaegertracer must be included in the classpath of all nodes then Jaeger tracer can be setup in `solr.xml` like this:
+Add the solr-jaegertracer JAR file and the other JARs provided with this module to your Solr installation, ideally to each node.
+GSON is a dependency that is only used by Jaeger's "remote" sampler,
+which is the default. Solr doesn't distribute it, so you'll need to add GSON yourself or configure a different sampler.
+
+The Jaeger Tracer Configurator is added to `solr.xml` like this:
```
-<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">
- <str name="agentHost">localhost</str>
- <int name="agentPort">5775</int>
- <bool name="logSpans">true</bool>
- <int name="flushInterval">1000</int>
- <int name="maxQueueSize">10000</int>
-</tracerConfig>
+<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />
```
-List of parameters for JaegerTracerConfigurator include:
+There are no configuration elements in the XML; instead, this 3rd party system is configured using System Properties or Environment Variables. The full list are listed at [Jaeger-README](https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md).
+For example, to use the probabilistic sampler, you could set this environment variable:
-|Parameter|Type|Required|Default|Description|
-|---------|----|--------|-------|-----------|
-|agentHost|string|Yes||The host of Jaeger backend|
-|agentPort|int|Yes||The port of Jaeger port|
-|logsSpans|bool|No|true|Whether the tracer should also log the spans|
-|flushInterval|int|No|5000|The tracer's flush interval (ms)|
-|maxQueueSize|int|No|10000|The tracer's maximum queue size|
+```
+export JAEGER_SAMPLER_TYPE=probabilistic
+```
-Other parameters which are not listed above can be configured using System Properties or Environment Variables. The full list are listed at [Jaeger-README](https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md).
+or System property:
-By default the sampling rate is 0.1%, this value can be changed by updating
-key `samplePercentage` of cluster properties. I.e: `/admin/collections?action=CLUSTERPROP&name=propertyName&samplePercentage=100`.
\ No newline at end of file
+```
+bin/solr start -DJAEGER_SAMPLER_TYPE=probabilistic
+```
\ No newline at end of file
diff --git a/solr/contrib/jaegertracer-configurator/build.gradle b/solr/contrib/jaegertracer-configurator/build.gradle
index d2094b1bc34..8d145392c36 100644
--- a/solr/contrib/jaegertracer-configurator/build.gradle
+++ b/solr/contrib/jaegertracer-configurator/build.gradle
@@ -24,7 +24,7 @@ dependencies {
implementation ("io.jaegertracing:jaeger-thrift", {
exclude group: "com.squareup.okhttp3", module: "okhttp"
- exclude group: "com.google.code.gson", module: "gson"
+ exclude group: "com.google.code.gson", module: "gson" // for "RemoteControlledSampler", optional
exclude group: "javax.annotation", module: "javax.annotation-api"
})
diff --git a/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java b/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java
index fdbb8bbde9a..f52e5806d99 100644
--- a/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java
+++ b/solr/contrib/jaegertracer-configurator/src/java/org/apache/solr/jaeger/JaegerTracerConfigurator.java
@@ -18,72 +18,20 @@
package org.apache.solr.jaeger;
import io.jaegertracing.Configuration;
-import io.jaegertracing.internal.samplers.ConstSampler;
import io.opentracing.Tracer;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.TracerConfigurator;
public class JaegerTracerConfigurator extends TracerConfigurator {
- public static final String AGENT_HOST = "agentHost";
- public static final String AGENT_PORT = "agentPort";
- public static final String LOG_SPANS = "logSpans";
- public static final String FLUSH_INTERVAL = "flushInterval";
- public static final String MAX_QUEUE_SIZE = "maxQueueSize";
-
- private volatile Tracer tracer;
@Override
public Tracer getTracer() {
- return tracer;
+ // See javadocs on Configuration for how to configure via sys props or env vars
+ return Configuration.fromEnv("solr").getTracer();
}
@Override
public void init(@SuppressWarnings({"rawtypes"})NamedList args) {
- Object host = args.get(AGENT_HOST);
- if (!(host instanceof String)) {
- throw new IllegalArgumentException("Expected a required string for param '" + AGENT_HOST + "'");
- }
-
- Object portArg = args.get(AGENT_PORT);
- if (!(portArg instanceof Integer)) {
- throw new IllegalArgumentException("Expected a required int for param '" + AGENT_PORT + "'");
- }
- int port = (Integer) portArg;
-
- Boolean logSpans = args.getBooleanArg(LOG_SPANS);
- if (logSpans == null)
- logSpans = true;
-
- Object flushIntervalArg = args.get(FLUSH_INTERVAL);
- if (flushIntervalArg != null && !(flushIntervalArg instanceof Integer)) {
- throw new IllegalArgumentException("Expected a required int for param '" + FLUSH_INTERVAL +"'");
- }
- int flushInterval = flushIntervalArg == null ? 1000 : (Integer) flushIntervalArg;
-
- Object maxQueueArgs = args.get(MAX_QUEUE_SIZE);
- if (maxQueueArgs != null && !(maxQueueArgs instanceof Integer)) {
- throw new IllegalArgumentException("Expected a required int for param '" + MAX_QUEUE_SIZE +"'");
- }
- int maxQueue = maxQueueArgs == null ? 10000 : (Integer) maxQueueArgs;
-
- Configuration.SamplerConfiguration samplerConfig = new Configuration.SamplerConfiguration()
- .withType(ConstSampler.TYPE)
- .withParam(1);
-
- Configuration.ReporterConfiguration reporterConfig = Configuration.ReporterConfiguration.fromEnv();
- Configuration.SenderConfiguration senderConfig = reporterConfig.getSenderConfiguration()
- .withAgentHost(host.toString())
- .withAgentPort(port);
-
- reporterConfig.withLogSpans(logSpans)
- .withFlushInterval(flushInterval)
- .withMaxQueueSize(maxQueue)
- .withSender(senderConfig);
- tracer = new Configuration("solr")
- .withSampler(samplerConfig)
- .withReporter(reporterConfig)
- .getTracer();
}
-
}
diff --git a/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml b/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml
index 34729c300e6..3f2932e784e 100644
--- a/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml
+++ b/solr/contrib/jaegertracer-configurator/src/test-files/solr/solr.xml
@@ -34,13 +34,7 @@
<str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str>
</shardHandlerFactory>
- <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">
- <str name="agentHost">localhost</str>
- <int name="agentPort">5775</int>
- <bool name="logSpans">true</bool>
- <int name="flushInterval">1000</int>
- <int name="maxQueueSize">10000</int>
- </tracerConfig>
+ <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />
<solrcloud>
<str name="host">127.0.0.1</str>
diff --git a/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java b/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java
index a0867f78dd6..9fc8d66de10 100644
--- a/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java
+++ b/solr/contrib/jaegertracer-configurator/src/test/org/apache/solr/jaeger/TestJaegerConfigurator.java
@@ -17,43 +17,51 @@
package org.apache.solr.jaeger;
-import java.io.Closeable;
-import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.util.TimeOut;
-import org.apache.solr.util.tracing.GlobalTracer;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-
-import static org.apache.solr.jaeger.JaegerTracerConfigurator.AGENT_HOST;
-import static org.apache.solr.jaeger.JaegerTracerConfigurator.AGENT_PORT;
-import static org.apache.solr.jaeger.JaegerTracerConfigurator.FLUSH_INTERVAL;
-import static org.apache.solr.jaeger.JaegerTracerConfigurator.LOG_SPANS;
-import static org.apache.solr.jaeger.JaegerTracerConfigurator.MAX_QUEUE_SIZE;
+import org.junit.rules.TestRule;
public class TestJaegerConfigurator extends SolrTestCaseJ4 {
+ @Rule
+ public TestRule solrTestRules = new SystemPropertiesRestoreRule();
+
+ @Before
+ public void doBefore() {
+ // to be safe because this test tests tracing.
+ resetGlobalTracer();
+ ExecutorUtil.resetThreadLocalProviders();
+
+ // The default sampler is one that uses GSON but we don't bother with that dependency.
+ // So we change it to something simple.
+ System.setProperty("JAEGER_SAMPLER_TYPE", "const");
+ }
+
@Test
- public void testInjected() throws Exception{
+ public void testInjected() throws Exception {
MiniSolrCloudCluster cluster = new SolrCloudTestCase.Builder(2, createTempDir())
.addConfig("config", TEST_PATH().resolve("collection1").resolve("conf"))
.withSolrXml(getFile("solr/solr.xml").toPath())
.build();
- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "100.0")
- .process(cluster.getSolrClient());
try {
TimeOut timeOut = new TimeOut(2, TimeUnit.MINUTES, TimeSource.NANO_TIME);
- timeOut.waitFor("Waiting for GlobalTracer is registered", () -> GlobalTracer.getTracer() instanceof io.jaegertracing.internal.JaegerTracer);
+ timeOut.waitFor(
+ "Waiting for GlobalTracer is registered",
+ () -> GlobalTracer.get().toString().contains("JaegerTracer"));
//TODO add run Jaeger through Docker and verify spans available after run these commands
CollectionAdminRequest.createCollection("test", 2, 1).process(cluster.getSolrClient());
@@ -67,70 +75,4 @@ public class TestJaegerConfigurator extends SolrTestCaseJ4 {
}
}
-
- @Test
- @SuppressWarnings({"unchecked"})
- public void testRequiredParameters() throws IOException {
- JaegerTracerConfigurator configurator = new JaegerTracerConfigurator();
- @SuppressWarnings({"rawtypes"})
- NamedList initArgs = new NamedList();
- IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(AGENT_HOST) || exc.getMessage().contains(AGENT_PORT));
- initArgs.add(AGENT_HOST, "localhost");
-
- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(AGENT_PORT));
- initArgs.add(AGENT_PORT, 5775);
-
- // no exception should be thrown
- configurator.init(initArgs);
- ((Closeable)configurator.getTracer()).close();
-
- initArgs.add(LOG_SPANS, true);
- initArgs.add(FLUSH_INTERVAL, 1000);
- initArgs.add(MAX_QUEUE_SIZE, 10000);
- configurator.init(initArgs);
- ((Closeable)configurator.getTracer()).close();
- }
-
- @Test
- @SuppressWarnings({"unchecked"})
- public void testIncorrectFormat() {
- JaegerTracerConfigurator configurator = new JaegerTracerConfigurator();
- @SuppressWarnings({"rawtypes"})
- NamedList initArgs = new NamedList();
- initArgs.add(AGENT_HOST, 100);
- initArgs.add(AGENT_PORT, 5775);
-
- IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(AGENT_HOST));
-
- initArgs.clear();
- initArgs.add(AGENT_HOST, "localhost");
- initArgs.add(AGENT_PORT, "5775");
- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(AGENT_PORT));
-
- initArgs.clear();
- initArgs.add(AGENT_HOST, "localhost");
- initArgs.add(AGENT_PORT, 5775);
- initArgs.add(LOG_SPANS, 10);
- SolrException solrExc = expectThrows(SolrException.class, () -> configurator.init(initArgs));
- assertTrue(solrExc.getMessage().contains(LOG_SPANS));
-
- initArgs.clear();
- initArgs.add(AGENT_HOST, "localhost");
- initArgs.add(AGENT_PORT, 5775);
- initArgs.add(FLUSH_INTERVAL, "10");
- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(FLUSH_INTERVAL));
-
- initArgs.clear();
- initArgs.add(AGENT_HOST, "localhost");
- initArgs.add(AGENT_PORT, 5775);
- initArgs.add(MAX_QUEUE_SIZE, "10");
- exc = expectThrows(IllegalArgumentException.class, () -> configurator.init(initArgs));
- assertTrue(exc.getMessage().contains(MAX_QUEUE_SIZE));
-
- }
}
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index 6bb61993bbd..6eba46c710e 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -77,7 +77,7 @@ dependencies {
implementation 'com.tdunning:t-digest'
- implementation 'io.opentracing:opentracing-api'
+ api 'io.opentracing:opentracing-api' // Tracer is exposed on some methods
implementation 'io.opentracing:opentracing-noop'
implementation 'io.opentracing:opentracing-util'
implementation 'org.apache.commons:commons-exec'
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 21ffc7497b9..aca17a88993 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -48,6 +48,9 @@ import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+import io.opentracing.Tracer;
+import io.opentracing.noop.NoopTracer;
+import io.opentracing.noop.NoopTracerFactory;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.client.CredentialsProvider;
@@ -165,6 +168,12 @@ public class CoreContainer {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ {
+ // Declared up top to ensure this is present before anything else.
+ // note: will not be re-added if already there
+ ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());
+ }
+
final SolrCores solrCores = new SolrCores(this);
public static class CoreLoadFailure {
@@ -245,6 +254,8 @@ public class CoreContainer {
protected volatile SolrMetricsContext solrMetricsContext;
+ protected volatile Tracer tracer = NoopTracerFactory.create();
+
protected MetricsHandler metricsHandler;
protected volatile MetricsHistoryHandler metricsHistoryHandler;
@@ -647,6 +658,11 @@ public class CoreContainer {
return metricsHistoryHandler;
}
+ /** Never null but may implement {@link NoopTracer}. */
+ public Tracer getTracer() {
+ return tracer;
+ }
+
public OrderedExecutor getReplayUpdatesExecutor() {
return replayUpdatesExecutor;
}
@@ -719,6 +735,8 @@ public class CoreContainer {
String registryName = SolrMetricManager.getRegistryName(SolrInfoBean.Group.node);
solrMetricsContext = new SolrMetricsContext(metricManager, registryName, metricTag);
+ tracer = TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo());
+
coreContainerWorkExecutor = MetricUtils.instrumentedExecutorService(
coreContainerWorkExecutor, null,
metricManager.registry(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)),
@@ -751,7 +769,6 @@ public class CoreContainer {
(PublicKeyHandler) containerHandlers.get(PublicKeyHandler.PATH));
// use deprecated API for back-compat, remove in 9.0
pkiAuthenticationPlugin.initializeMetrics(solrMetricsContext, "/authentication/pki");
- TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader());
packageLoader = new PackageLoader(this);
containerHandlers.getApiBag().registerObject(packageLoader.getPackageAPI().editAPI);
containerHandlers.getApiBag().registerObject(packageLoader.getPackageAPI().readAPI);
@@ -1240,6 +1257,8 @@ public class CoreContainer {
org.apache.lucene.util.IOUtils.closeWhileHandlingException(packageLoader);
}
org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort
+
+ tracer.close();
}
public void cancelCoreRecoveries() {
@@ -2247,10 +2266,6 @@ public class CoreContainer {
return this.distributedCollectionCommandRunner;
}
- static {
- ExecutorUtil.addThreadLocalProvider(SolrRequestInfo.getInheritableThreadLocalProvider());
- }
-
/**
* Run an arbitrary task in it's own thread. This is an expert option and is
* a method you should use with great care. It would be bad to run something that never stopped
diff --git a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java
index d5ee7a3ecff..91920443568 100644
--- a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java
+++ b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java
@@ -18,53 +18,85 @@
package org.apache.solr.core;
import java.lang.invoke.MethodHandles;
+import java.util.concurrent.atomic.AtomicReference;
+import io.opentracing.Scope;
+import io.opentracing.Span;
import io.opentracing.Tracer;
-import io.opentracing.noop.NoopTracerFactory;
-import org.apache.solr.common.cloud.ZkStateReader;
+import io.opentracing.util.GlobalTracer;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
-import org.apache.solr.util.tracing.GlobalTracer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/** Produces an OpenTracing {@link Tracer} from configuration. */
public abstract class TracerConfigurator implements NamedListInitializedPlugin {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public abstract Tracer getTracer();
- public static void loadTracer(SolrResourceLoader loader, PluginInfo info, ZkStateReader stateReader) {
- if (info == null) {
- // in case of a Tracer is registered to OpenTracing through javaagent
- if (io.opentracing.util.GlobalTracer.isRegistered()) {
- GlobalTracer.setup(io.opentracing.util.GlobalTracer.get());
- registerListener(stateReader);
- } else {
- GlobalTracer.setup(NoopTracerFactory.create());
- GlobalTracer.get().setSamplePercentage(0.0);
- }
- } else {
- TracerConfigurator configurator = loader
- .newInstance(info.className, TracerConfigurator.class);
- configurator.init(info.initArgs);
+ public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) {
+ // In "normal" Solr operation, loadTracer is called once in Solr's lifetime.
+ // In test mode we run many test suites, sometimes multiple servers per test suite.
+ // It's important that the tracing config not change throughout a test suite because of the
+ // static singleton pattern and assumptions based on this.
- GlobalTracer.setup(configurator.getTracer());
- registerListener(stateReader);
+ if (info != null && info.isEnabled()) {
+ GlobalTracer.registerIfAbsent(() -> {
+ TracerConfigurator configurator =
+ loader.newInstance(info.className, TracerConfigurator.class);
+ configurator.init(info.initArgs);
+ return configurator.getTracer();
+ });
+ }
+ if (GlobalTracer.isRegistered()) {
+ // ideally we would furthermore check that it's not a no-op impl either but
+ // GlobalTracer.get() always returns a GlobalTracer implementing Tracer that delegates
+ // to the real Tracer (that may or may not be a No-Op impl.
+ ExecutorUtil.addThreadLocalProvider(new TracerConfigurator.SpanThreadLocalProvider());
}
+
+ return GlobalTracer.get();
}
- private static void registerListener(ZkStateReader stateReader) {
- stateReader.registerClusterPropertiesListener(properties -> {
- if (properties.containsKey(ZkStateReader.SAMPLE_PERCENTAGE)) {
- try {
- double sampleRate = Double.parseDouble(properties.get(ZkStateReader.SAMPLE_PERCENTAGE).toString());
- GlobalTracer.get().setSamplePercentage(sampleRate);
- } catch (NumberFormatException e) {
- log.error("Unable to set sample rate", e);
- }
- } else {
- GlobalTracer.get().setSamplePercentage(0.1);
+ /**
+ * Propagate the active span across threads. New spans are not created, which means that we're
+ * possibly exposing a Span to a thread that may find that it has already finished, depending on
+ * how the instrumented thread pool is used. It's probably fine to create new spans related to a
+ * finished span? It's not okay to otherwise touch a finished span (e.g. to log or tag).
+ *
+ * <p>This strategy is also used by {@code
+ * brave.propagation.CurrentTraceContext#wrap(java.lang.Runnable)}.
+ */
+ private static class SpanThreadLocalProvider
+ implements ExecutorUtil.InheritableThreadLocalProvider {
+ private final Tracer tracer = GlobalTracer.get();
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Override
+ public void store(AtomicReference ctx) {
+ assert tracer == GlobalTracer.get() : "Tracer changed; not supported!";
+ ctx.set(tracer.scopeManager().activeSpan());
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Override
+ public void set(AtomicReference ctx) {
+ final Span span = (Span) ctx.get();
+ if (span != null) {
+ log.trace("Thread received span to do async work: {}", span);
+ final Scope scope = tracer.scopeManager().activate(span);
+ ctx.set(scope);
}
- return false;
- });
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ @Override
+ public void clean(AtomicReference ctx) {
+ Scope scope = (Scope) ctx.get();
+ if (scope != null) {
+ scope.close();
+ }
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
index 5aa80120ab0..6207a63402b 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
@@ -49,7 +49,6 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.util.tracing.GlobalTracer;
import org.apache.solr.util.tracing.SolrRequestCarrier;
@SolrThreadUnsafe
@@ -125,8 +124,8 @@ public class HttpShardHandler extends ShardHandler {
public void submit(final ShardRequest sreq, final String shard, final ModifiableSolrParams params) {
// do this outside of the callable for thread safety reasons
final List<String> urls = getURLs(shard);
- final Tracer tracer = GlobalTracer.getTracer();
- final Span span = tracer != null ? tracer.activeSpan() : null;
+ final Tracer tracer = sreq.tracer; // not null
+ final Span span = tracer.activeSpan(); // probably not null?
params.remove(CommonParams.WT); // use default (currently javabin)
params.remove(CommonParams.VERSION);
@@ -162,7 +161,7 @@ public class HttpShardHandler extends ShardHandler {
@Override
public void onStart() {
- if (tracer != null && span != null) {
+ if (span != null) {
tracer.inject(span.context(), Format.Builtin.HTTP_HEADERS, new SolrRequestCarrier(req));
}
SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
index f7c05d236bf..0e152bde04d 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.handler.component;
+import io.opentracing.Tracer;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.common.params.ModifiableSolrParams;
import java.util.ArrayList;
@@ -59,6 +61,9 @@ public class ShardRequest {
/** may be null */
public String nodeName;
+ /** Not null but may implement {@link io.opentracing.noop.NoopTracer}. */
+ public final Tracer tracer = GlobalTracer.get();
+
// TODO: one could store a list of numbers to correlate where returned docs
// go in the top-level response rather than looking up by id...
// this would work well if we ever transitioned to using internal ids and
diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
index 2c5090ba16a..394764868ac 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.request;
+import io.opentracing.Tracer;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.common.params.SolrParams;
@@ -132,6 +134,14 @@ public interface SolrQueryRequest extends AutoCloseable {
default HttpSolrCall getHttpSolrCall() {
return null;
}
+
+ /**
+ * Distributed tracing Tracer. Never null but might implement
+ * {@link io.opentracing.noop.NoopTracer}.
+ */
+ default Tracer getTracer() {
+ return GlobalTracer.get(); // default impl is only for some tests
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
index 76e8733c550..ac3acf37af2 100644
--- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
+++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java
@@ -23,6 +23,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import io.opentracing.Tracer;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.api.ApiBag;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
@@ -34,6 +36,7 @@ import org.apache.solr.common.util.ValidatingJsonMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.servlet.HttpSolrCall;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.RefCounted;
@@ -139,6 +142,21 @@ public abstract class SolrQueryRequestBase implements SolrQueryRequest, Closeabl
return schema;
}
+ @Override
+ public Tracer getTracer() {
+ if (core != null) {
+ return core.getCoreContainer().getTracer(); // this is the common path
+ }
+ final HttpSolrCall call = getHttpSolrCall();
+ if (call != null) {
+ final Tracer tracer = (Tracer) call.getReq().getAttribute(Tracer.class.getName());
+ if (tracer != null) {
+ return tracer;
+ }
+ }
+ return GlobalTracer.get(); // this way is not ideal (particularly in tests) but it's okay
+ }
+
@Override
public void updateSchemaToLatest() {
schema = core.getLatestSchema();
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index ee7c9f4b5ac..548b00d5720 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -109,7 +109,6 @@ import org.apache.solr.servlet.cache.Method;
import org.apache.solr.update.processor.DistributingUpdateProcessorFactory;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.TimeOut;
-import org.apache.solr.util.tracing.GlobalTracer;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -501,10 +500,8 @@ public class HttpSolrCall {
*/
public Action call() throws IOException {
MDCLoggingContext.reset();
- Span activeSpan = GlobalTracer.getTracer().activeSpan();
- if (activeSpan != null) {
- MDCLoggingContext.setTracerId(activeSpan.context().toTraceId());
- }
+ Span activeSpan = (Span) req.getAttribute(Span.class.getName()); // never null
+ MDCLoggingContext.setTracerId(activeSpan.context().toTraceId()); // handles empty string
MDCLoggingContext.setNode(cores);
@@ -706,7 +703,7 @@ public class HttpSolrCall {
// encoding issues with Tomcat
if (header != null && !header.getName().equalsIgnoreCase(TRANSFER_ENCODING_HEADER)
&& !header.getName().equalsIgnoreCase(CONNECTION_HEADER)) {
-
+
// NOTE: explicitly using 'setHeader' instead of 'addHeader' so that
// the remote nodes values for any response headers will overide any that
// may have already been set locally (ex: by the local jetty's RewriteHandler config)
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
index 3a294c5985f..87ad3d5abde 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
@@ -58,10 +58,9 @@ import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.annotations.VisibleForTesting;
-import io.opentracing.Scope;
import io.opentracing.Span;
-import io.opentracing.SpanContext;
import io.opentracing.Tracer;
+import io.opentracing.propagation.Format;
import io.opentracing.tag.Tags;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
@@ -90,7 +89,7 @@ import org.apache.solr.security.PKIAuthenticationPlugin;
import org.apache.solr.security.PublicKeyHandler;
import org.apache.solr.util.StartupLoggingUtils;
import org.apache.solr.util.configuration.SSLConfigurationsFactory;
-import org.apache.solr.util.tracing.GlobalTracer;
+import org.apache.solr.util.tracing.HttpServletCarrier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -423,7 +422,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
httpClient = null;
cc.shutdown();
}
- GlobalTracer.get().close();
}
}
@@ -436,10 +434,31 @@ public class SolrDispatchFilter extends BaseSolrFilter {
if (!(_request instanceof HttpServletRequest)) return;
HttpServletRequest request = closeShield((HttpServletRequest)_request, retry);
HttpServletResponse response = closeShield((HttpServletResponse)_response, retry);
- Scope scope = null;
- Span span = null;
+
+ String requestPath = ServletUtils.getPathAfterContext(request);
+ // No need to even create the HttpSolrCall object if this path is excluded.
+ if (excludePatterns != null) {
+ for (Pattern p : excludePatterns) {
+ Matcher matcher = p.matcher(requestPath);
+ if (matcher.lookingAt()) {
+ chain.doFilter(request, response);
+ return;
+ }
+ }
+ }
+
+ Tracer tracer = cores.getTracer();
+ String hostAndPort = request.getServerName() + "_" + request.getServerPort();
+ Span span = tracer.buildSpan(request.getMethod() + ":" + hostAndPort)
+ .asChildOf(tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request)))
+ .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER)
+ .withTag(Tags.HTTP_URL, request.getRequestURL().toString())
+ .start();
+ request.setAttribute(Tracer.class.getName(), tracer);
+ request.setAttribute(Span.class.getName(), span);
boolean accepted = false;
- try {
+ try (var scope = tracer.scopeManager().activate(span)) {
+ assert scope != null; // prevent javac warning about scope being unused
if (cores == null || cores.isShutDown()) {
try {
@@ -453,18 +472,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
}
}
- String requestPath = ServletUtils.getPathAfterContext(request);
- // No need to even create the HttpSolrCall object if this path is excluded.
- if (excludePatterns != null) {
- for (Pattern p : excludePatterns) {
- Matcher matcher = p.matcher(requestPath);
- if (matcher.lookingAt()) {
- chain.doFilter(request, response);
- return;
- }
- }
- }
-
try {
accepted = rateLimitManager.handleRequest(request);
} catch (InterruptedException e) {
@@ -479,24 +486,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
response.sendError(429, errorMessage);
}
- SpanContext parentSpan = GlobalTracer.get().extract(request);
- Tracer tracer = GlobalTracer.getTracer();
-
- Tracer.SpanBuilder spanBuilder = null;
- String hostAndPort = request.getServerName() + "_" + request.getServerPort();
- if (parentSpan == null) {
- spanBuilder = tracer.buildSpan(request.getMethod() + ":" + hostAndPort);
- } else {
- spanBuilder = tracer.buildSpan(request.getMethod() + ":" + hostAndPort)
- .asChildOf(parentSpan);
- }
-
- spanBuilder
- .withTag(Tags.SPAN_KIND.getKey(), Tags.SPAN_KIND_SERVER)
- .withTag(Tags.HTTP_URL.getKey(), request.getRequestURL().toString());
- span = spanBuilder.start();
- scope = tracer.scopeManager().activate(span);
-
AtomicReference<HttpServletRequest> wrappedRequest = new AtomicReference<>();
if (!authenticateRequest(request, response, wrappedRequest)) { // the response and status code have already been sent
return;
@@ -537,10 +526,6 @@ public class SolrDispatchFilter extends BaseSolrFilter {
ExecutorUtil.setServerThreadFlag(null);
}
} finally {
- if (span != null) span.finish();
- if (scope != null) scope.close();
-
- GlobalTracer.get().clearContext();
consumeInputFully(request, response);
SolrRequestInfo.reset();
SolrRequestParsers.cleanupMultipartFiles(request);
@@ -548,6 +533,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
if (accepted) {
rateLimitManager.decrementActiveRequests(request);
}
+ span.finish();
}
}
diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
index 96cc66dd2c8..c81c32091d2 100644
--- a/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
+++ b/solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
@@ -41,6 +41,8 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import io.opentracing.Span;
+import io.opentracing.noop.NoopSpan;
import org.apache.commons.io.input.CloseShieldInputStream;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.api.V2HttpCall;
@@ -59,7 +61,6 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.util.RTimerTree;
-import org.apache.solr.util.tracing.GlobalTracer;
import org.eclipse.jetty.http.HttpFields;
import org.eclipse.jetty.http.MimeTypes;
import org.eclipse.jetty.server.Request;
@@ -169,8 +170,10 @@ public class SolrRequestParsers {
// Pick the parser from the request...
ArrayList<ContentStream> streams = new ArrayList<>(1);
SolrParams params = parser.parseParamsAndFillStreams( req, streams );
- if (GlobalTracer.get().tracing()) {
- GlobalTracer.getTracer().activeSpan().setTag("params", params.toString());
+
+ Span span = (Span) req.getAttribute(Span.class.getName()); // not null but maybe in some tests?
+ if (span != null && !(span instanceof NoopSpan)) {
+ span.setTag("params", params.toString());
}
SolrQueryRequest sreq = buildRequestFrom(core, params, streams, getRequestTimer(req), req);
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 9d2377efdea..ea2ae7b654e 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -53,7 +53,6 @@ import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;
import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
-import org.apache.solr.util.tracing.GlobalTracer;
import org.apache.solr.util.tracing.SolrRequestCarrier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -294,7 +293,7 @@ public class SolrCmdDistributor implements Closeable {
req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal());
}
- Tracer tracer = GlobalTracer.getTracer();
+ Tracer tracer = req.cmd.getTracer();
Span parentSpan = tracer.activeSpan();
if (parentSpan != null) {
tracer.inject(parentSpan.context(), Format.Builtin.HTTP_HEADERS,
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
index b124271d977..29f695314f2 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java
@@ -16,6 +16,8 @@
*/
package org.apache.solr.update;
+import io.opentracing.Tracer;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.request.SolrQueryRequest;
@@ -90,6 +92,14 @@ public abstract class UpdateCommand implements Cloneable {
this.req = req;
}
+ /**
+ * Distributed tracing Tracer. Never null but might implement
+ * {@link io.opentracing.noop.NoopTracer}.
+ */
+ public Tracer getTracer() {
+ return (req != null) ? req.getTracer() : GlobalTracer.get();
+ }
+
@Override
public UpdateCommand clone() {
try {
diff --git a/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.java b/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.java
deleted file mode 100644
index bc9ffe51702..00000000000
--- a/solr/core/src/java/org/apache/solr/util/tracing/GlobalTracer.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.util.tracing;
-
-import javax.servlet.http.HttpServletRequest;
-import java.util.Random;
-
-import com.google.common.annotations.VisibleForTesting;
-import io.opentracing.SpanContext;
-import io.opentracing.Tracer;
-import io.opentracing.noop.NoopTracerFactory;
-import io.opentracing.propagation.Format;
-
-public class GlobalTracer {
- private static final Tracer NOOP_TRACER = NoopTracerFactory.create();
- private static final Random RANDOM;
- static {
- // We try to make things reproducible in the context of our tests by initializing the random instance
- // based on the current seed
- String seed = System.getProperty("tests.seed");
- if (seed == null) {
- RANDOM = new Random();
- } else {
- RANDOM = new Random(seed.hashCode());
- }
- }
-
- private static volatile GlobalTracer INS = new GlobalTracer(NOOP_TRACER);
-
- public synchronized static void setup(Tracer tracer) {
- if (INS != null) {
- INS.close();
- }
- INS = new GlobalTracer(tracer);
- }
-
- public static GlobalTracer get() {
- return INS;
- }
-
- public static Tracer getTracer() {
- return INS.tracer();
- }
-
- @VisibleForTesting
- final Tracer tracer;
- private double rate;
- private final ThreadLocal<Tracer> threadLocal = new ThreadLocal<>();
-
- public GlobalTracer(Tracer tracer) {
- this.tracer = tracer;
- }
-
- public synchronized void setSamplePercentage(double rate) {
- this.rate = rate / 100.0;
- }
-
- @VisibleForTesting
- public double getSampleRate() {
- return rate;
- }
-
- public boolean tracing() {
- return threadLocal.get() != null
- && threadLocal.get() != NOOP_TRACER
- && threadLocal.get().activeSpan() != null;
- }
-
- public SpanContext extract(HttpServletRequest request) {
- SpanContext spanContext = tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request));
- if (spanContext != null) {
- threadLocal.set(tracer);
- }
- return spanContext;
- }
-
- private Tracer tracer() {
- Tracer tracer = threadLocal.get();
- if (tracer != null)
- return tracer;
-
- if (traced()) {
- threadLocal.set(this.tracer);
- } else {
- threadLocal.set(NOOP_TRACER);
- }
-
- return threadLocal.get();
- }
-
- /**
- * Clear tracing context for the current thread
- */
- public void clearContext() {
- threadLocal.remove();
- }
-
- private boolean traced() {
- return RANDOM.nextDouble() <= rate;
- }
-
- public void close() {
- tracer.close();
- }
-}
diff --git a/solr/core/src/test-files/solr/solr-tracing.xml b/solr/core/src/test-files/solr/solr-tracing.xml
deleted file mode 100644
index b33a4a85e6a..00000000000
--- a/solr/core/src/test-files/solr/solr-tracing.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- All (relative) paths are relative to the installation path
--->
-<solr>
-
- <str name="shareSchema">${shareSchema:false}</str>
- <str name="configSetBaseDir">${configSetBaseDir:configsets}</str>
- <str name="coreRootDirectory">${coreRootDirectory:.}</str>
- <str name="collectionsHandler">${collectionsHandler:solr.CollectionsHandler}</str>
-
- <shardHandlerFactory name="shardHandlerFactory" class="HttpShardHandlerFactory">
- <str name="urlScheme">${urlScheme:}</str>
- <int name="socketTimeout">${socketTimeout:90000}</int>
- <int name="connTimeout">${connTimeout:15000}</int>
- <str name="shardsWhitelist">${solr.tests.shardsWhitelist:}</str>
- </shardHandlerFactory>
-
- <tracerConfig name="tracerConfig" class="MockTracerConfigurator">
- </tracerConfig>
-
- <solrcloud>
- <str name="host">127.0.0.1</str>
- <int name="hostPort">${hostPort:8983}</int>
- <str name="hostContext">${hostContext:solr}</str>
- <int name="zkClientTimeout">${solr.zkclienttimeout:30000}</int>
- <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
- <int name="leaderVoteWait">${leaderVoteWait:10000}</int>
- <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:45000}</int>
- <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:340000}</int>
- <str name="zkCredentialsProvider">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str>
- <str name="zkACLProvider">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str>
- </solrcloud>
- <metrics>
- <reporter name="default" class="org.apache.solr.metrics.reporters.SolrJmxReporter">
- <str name="rootName">solr_${hostPort:8983}</str>
- </reporter>
- </metrics>
-
-</solr>
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
index fc5dcb2815a..8d1ebc2b216 100644
--- a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
+++ b/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java
@@ -18,42 +18,39 @@
package org.apache.solr.util.tracing;
import java.io.IOException;
-import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import io.opentracing.mock.MockSpan;
import io.opentracing.mock.MockTracer;
+import io.opentracing.util.GlobalTracer;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.util.TimeOut;
+import org.apache.solr.util.LogLevel;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+@LogLevel("org.apache.solr.core.TracerConfigurator=trace")
public class TestDistributedTracing extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ static MockTracer tracer;
@BeforeClass
public static void beforeTest() throws Exception {
+ tracer = new MockTracer();
+ assertTrue(GlobalTracer.registerIfAbsent(tracer));
+
configureCluster(4)
.addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(TEST_PATH().resolve("solr-tracing.xml"))
.configure();
- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "100.0")
- .process(cluster.getSolrClient());
- waitForSampleRateUpdated(1.0);
CollectionAdminRequest
.createCollection(COLLECTION, "config", 2, 2)
.setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE)
@@ -61,21 +58,18 @@ public class TestDistributedTracing extends SolrCloudTestCase {
cluster.waitForActiveCollection(COLLECTION, 2, 4);
}
- private static void waitForSampleRateUpdated(double rate) throws TimeoutException, InterruptedException {
- TimeOut timeOut = new TimeOut(1, TimeUnit.MINUTES, TimeSource.NANO_TIME);
- timeOut.waitFor("Waiting for sample rate is updated", () ->
- Math.abs(GlobalTracer.get().getSampleRate() - rate) < 0.001
- && GlobalTracer.get().tracer instanceof MockTracer);
- }
-
- private List<MockSpan> getFinishedSpans() {
- return ((MockTracer)GlobalTracer.get().tracer).finishedSpans();
+ @AfterClass
+ public static void afterTest() {
+ tracer = null;
}
@Test
public void test() throws IOException, SolrServerException, TimeoutException, InterruptedException {
+ // TODO it would be clearer if we could compare the complete Span tree between reality
+ // and what we assert it looks like in a structured visual way.
+
CloudSolrClient cloudClient = cluster.getSolrClient();
- List<MockSpan> allSpans = getFinishedSpans();
+ List<MockSpan> allSpans = tracer.finishedSpans();
cloudClient.add(COLLECTION, sdoc("id", "1"));
List<MockSpan> finishedSpans = getRecentSpans(allSpans);
@@ -112,15 +106,6 @@ public class TestDistributedTracing extends SolrCloudTestCase {
fail("All spans must belong to single span, but:"+finishedSpans);
}
}
-
- CollectionAdminRequest.setClusterProperty(ZkStateReader.SAMPLE_PERCENTAGE, "0.0")
- .process(cluster.getSolrClient());
- waitForSampleRateUpdated(0);
-
- getRecentSpans(allSpans);
- cloudClient.add(COLLECTION, sdoc("id", "5"));
- finishedSpans = getRecentSpans(allSpans);
- assertEquals(0, finishedSpans.size());
}
private void assertOneSpanIsChildOfAnother(List<MockSpan> finishedSpans) {
@@ -136,10 +121,10 @@ public class TestDistributedTracing extends SolrCloudTestCase {
}
private List<MockSpan> getRecentSpans(List<MockSpan> allSpans) {
- List<MockSpan> result = new ArrayList<>(getFinishedSpans());
+ List<MockSpan> result = new ArrayList<>(tracer.finishedSpans());
result.removeAll(allSpans);
allSpans.clear();
- allSpans.addAll(getFinishedSpans());
+ allSpans.addAll(tracer.finishedSpans());
return result;
}
}
diff --git a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
index 85c0b05a819..4768539ec77 100644
--- a/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/src/major-changes-in-solr-9.adoc
@@ -192,6 +192,14 @@ set `-Dsolr.storeBaseUrl=true` so that Solr will continue to store the `base_url
Support for the `solr.storeBaseUrl` system property will be removed in Solr 10.x and `base_url` will no longer be stored.
+* Solr's distributed tracing no longer incorporates a special `samplePercentage` SolrCloud cluster property.
+Instead, consult the documentation for the tracing system you use on how to sample the traces.
+Consequently, if you use a Tracer at all, you will always have traces and thus trace IDs in logs.
+What percentage of them get reported to a tracing server is up to you.
+
+* JaegerTracerConfigurator no longer recognizes any configuration in solr.xml.
+ It is now completely configured via System properties and/or Environment variables as documented by Jaeger.
+
=== Schema Changes in 9
=== Authentication & Security Changes in Solr 9
diff --git a/solr/solr-ref-guide/src/solr-tracing.adoc b/solr/solr-ref-guide/src/solr-tracing.adoc
index 47477ed87c6..6a87bcdd3d7 100644
--- a/solr/solr-ref-guide/src/solr-tracing.adoc
+++ b/solr/solr-ref-guide/src/solr-tracing.adoc
@@ -36,96 +36,43 @@ A setup of a `TracerConfigurator` in `solr.xml` looks like this:
[source,xml]
----
<solr>
- <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">
- <str name="agentHost">localhost</str>
- <int name="agentPort">5775</int>
- <bool name="logSpans">true</bool>
- <int name="flushInterval">1000</int>
- <int name="maxQueueSize">10000</int>
- </tracerConfig>
+ <tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />
</solr>
----
If `<tracerConfig>` is absent, `TracerConfigurator` will try to pick up the `Tracer` instance that was registered in `io.opentracing.util.GlobalTracer`.
-By doing this some backends like DataDog are supported out of the box since
+By doing this, some backends like DataDog are supported out of the box since
https://docs.datadoghq.com/tracing/setup/java/[datadog-java-agent] uses Javaagent to register a `Tracer` in
`io.opentracing.util.GlobalTracer`.
-=== Configuring Sample Rate
-By default only 0.1% of requests are sampled, this ensures that tracing activities do not affect performance of a node.
+== Jaeger Tracer Configurator
-The rate can be changed without needing to restart Solr nodes by setting the new sample rate as a cluster property using the Collections API's <<cluster-node-management.adoc#clusterprop,CLUSTERPROP>> command.
+The `contrib/jagertracer-configurator` provides a default implementation for setting up Jaeger Tracer.
-For example, the following sets the `samplePercentage` property to 100%:
+Add the solr-jaegertracer JAR file and the other JARs provided with this module to your Solr installation, ideally to each node.
+GSON is a dependency that is only used by Jaeger's "remote" sampler,
+which is the default. Solr doesn't distribute it, so you'll need to add GSON yourself or configure a different sampler.
-[source,text]
-/admin/collections?action=CLUSTERPROP&name=samplePercentage&val=100
+Then Jaeger tracer is configured in `solr.xml` like this:
-== Jaeger Tracer Configurator
+[source,xml]
+----
+<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator" />
+----
-The `contrib/jagertracer-configurator` provides a default implementation for setting up Jaeger Tracer.
+_There are no configuration elements in the XML_; instead, this 3rd party system is configured using System Properties or Environment Variables. The full list are listed at https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].
-Note that all libraries of `jaegertracer-configurator` must be included in the classpath of all nodes.
-Then Jaeger tracer can be set up in `solr.xml` like this:
+For example, to use the probabilistic sampler, you could set this environment variable:
-[source,xml]
+[source,bash]
----
-<tracerConfig name="tracerConfig" class="org.apache.solr.jaeger.JaegerTracerConfigurator">
- <str name="agentHost">localhost</str>
- <int name="agentPort">5775</int>
- <bool name="logSpans">true</bool>
- <int name="flushInterval">1000</int>
- <int name="maxQueueSize">10000</int>
-</tracerConfig>
+export JAEGER_SAMPLER_TYPE=probabilistic
----
-The parameters for `JaegerTracerConfigurator` include:
-
-`agentHost`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The Jaeger backend host.
-
-`agentPort`::
-+
-[%autowidth,frame=none]
-|===
-s|Required |Default: none
-|===
-+
-The Jaeger port.
-
-`logSpans`::
-+
-[%autowidth,frame=none]
-|===
-|Optional |Default: `true`
-|===
-+
-Whether the tracer should also log the spans.
-
-`flushInterval`::
-+
-[%autowidth,frame=none]
-|===
-|Optional | Default: `5000` (ms)
-|===
-+
-The tracer's flush interval.
-
-`maxQueueSize`::
-+
-[%autowidth,frame=none]
-|===
-|Optional | Default: `10000`
-|===
-+
-The tracer's maximum queue size.
-
-Other parameters which are not listed above can be configured using system properties or environment variables.
-The full list of parameters can be found in the https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md[Jaeger-README].
+or System property:
+
+[source,bash]
+----
+bin/solr start -DJAEGER_SAMPLER_TYPE=probabilistic
+----
\ No newline at end of file
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index b26183cea05..7ff0b3ae5c8 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -129,8 +129,6 @@ public class ZkStateReader implements SolrCloseable {
public static final String CONFIGS_ZKNODE = "/configs";
public final static String CONFIGNAME_PROP = "configName";
- public static final String SAMPLE_PERCENTAGE = "samplePercentage";
-
/**
* @deprecated use {@link org.apache.solr.common.params.CollectionAdminParams#DEFAULTS} instead.
*/
@@ -227,7 +225,6 @@ public class ZkStateReader implements SolrCloseable {
CoreAdminParams.BACKUP_LOCATION,
DEFAULT_SHARD_PREFERENCES,
MAX_CORES_PER_NODE,
- SAMPLE_PERCENTAGE,
SOLR_ENVIRONMENT,
CollectionAdminParams.DEFAULTS,
CONTAINER_PLUGINS,
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
index 6664dbf02b3..cb23d94250a 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java
@@ -41,6 +41,14 @@ public class ExecutorUtil {
private static volatile List<InheritableThreadLocalProvider> providers = new ArrayList<>();
+ /**
+ * Resets everything added via {@link #addThreadLocalProvider(InheritableThreadLocalProvider)}.
+ * Useful to call at the beginning of tests.
+ */
+ public static void resetThreadLocalProviders() {
+ providers = new ArrayList<>();
+ }
+
public synchronized static void addThreadLocalProvider(InheritableThreadLocalProvider provider) {
for (InheritableThreadLocalProvider p : providers) {//this is to avoid accidental multiple addition of providers in tests
if (p.getClass().equals(provider.getClass())) return;
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 7aa14f421de..38268dcc4ad 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -31,6 +31,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
@@ -40,6 +41,8 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.ArrayList;
@@ -70,6 +73,8 @@ import com.carrotsearch.randomizedtesting.RandomizedTest;
import com.carrotsearch.randomizedtesting.TraceFormatting;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
+import io.opentracing.noop.NoopTracerFactory;
+import io.opentracing.util.GlobalTracer;
import org.apache.commons.io.FileUtils;
import org.apache.http.client.HttpClient;
import org.apache.logging.log4j.Level;
@@ -307,6 +312,35 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
} else {
UrlScheme.INSTANCE.setUrlScheme(UrlScheme.HTTP);
}
+
+ resetGlobalTracer();
+ ExecutorUtil.resetThreadLocalProviders();
+ }
+
+ /**
+ * GlobalTracer is initialized by org.apache.solr.core.TracerConfigurator by
+ * org.apache.solr.core.CoreContainer. Tests may need to reset it in the beginning of a test if
+ * it might have differing configuration from other tests in the same suite.
+ * It's also important to call {@link ExecutorUtil#resetThreadLocalProviders()}.
+ */
+ @SuppressForbidden(reason = "Hack to reset internal state of GlobalTracer")
+ public static void resetGlobalTracer() {
+ AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+ try {
+ final Class<GlobalTracer> globalTracerClass = GlobalTracer.class;
+ final Field isRegistered = globalTracerClass.getDeclaredField("isRegistered");
+ isRegistered.setAccessible(true);
+ isRegistered.setBoolean(null, false);
+ final Field tracer = globalTracerClass.getDeclaredField("tracer");
+ tracer.setAccessible(true);
+ tracer.set(null, NoopTracerFactory.create());
+ } catch (NoSuchFieldException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ return null;
+ });
+
+ assert GlobalTracer.isRegistered() == false;
}
@AfterClass
diff --git a/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.java b/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.java
deleted file mode 100644
index b1c7cb9168d..00000000000
--- a/solr/test-framework/src/java/org/apache/solr/core/MockTracerConfigurator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.core;
-
-import io.opentracing.Tracer;
-import io.opentracing.mock.MockTracer;
-import org.apache.solr.common.util.NamedList;
-
-public class MockTracerConfigurator extends TracerConfigurator {
-
- @Override
- public Tracer getTracer() {
- return new MockTracer();
- }
-
- @Override
- public void init(@SuppressWarnings({"rawtypes"})NamedList args) {
-
- }
-}