Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-18124: Add tracing spans for UpdateLog replay (#4962)

David Smiley4aaf68b40fb0
diff --git a/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml
new file mode 100644
index 00000000000..655aed1a3a4
--- /dev/null
+++ b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml
@@ -0,0 +1,9 @@
+title: Tracing (spans) for replication, recovery, and updatelog replay.
+type: changed
+authors:
+ - name: Gaurav Singh
+ nick: gsingh-code
+ - name: David Smiley
+links:
+ - name: SOLR-18124
+ url: https://issues.apache.org/jira/browse/SOLR-18124
diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
index 2b0f202a4bd..cfe671f36df 100644
--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java
+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java
@@ -22,6 +22,8 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
import com.carrotsearch.hppc.LongHashSet;
import com.carrotsearch.hppc.LongSet;
import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.StatusCode;
import java.io.Closeable;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -93,6 +95,7 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.TestInjection;
import org.apache.solr.util.TimeOut;
import org.apache.solr.util.plugin.PluginInfoInitialized;
+import org.apache.solr.util.tracing.TraceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -2129,7 +2132,16 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
// setting request info will help logging
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
- try {
+ final Span span = TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay").startSpan();
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ TraceUtils.setDbInstance(s, req.getCore().getName());
+ s.setAttribute("updatelog.replay.state", state.toString());
+ s.setAttribute("updatelog.replay.logs", translogs.size());
+ });
+ try (var scope = span.makeCurrent()) {
+ assert scope != null; // prevent javac warning about scope being unused
for (; ; ) {
TransactionLog translog = translogs.pollFirst();
if (translog == null) break;
@@ -2143,9 +2155,11 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
recoveryInfo.errors.incrementAndGet();
log.error("Replay failed due to exception", e);
}
+ span.recordException(e);
} catch (Exception e) {
recoveryInfo.errors.incrementAndGet();
log.error("Replay failed due to exception", e);
+ span.recordException(e);
} finally {
// change the state while updates are still blocked to prevent races
state = State.ACTIVE;
@@ -2159,6 +2173,19 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
log.error("ERROR: didn't get to recover from tlog {}", translog);
translog.decref();
}
+
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ s.setAttribute("updatelog.replay.adds", recoveryInfo.adds);
+ s.setAttribute("updatelog.replay.deletes", recoveryInfo.deletes);
+ s.setAttribute("updatelog.replay.deleteByQuery", recoveryInfo.deleteByQuery);
+ s.setAttribute("updatelog.replay.errors", recoveryInfo.errors.get());
+ });
+ if (recoveryInfo.failed || recoveryInfo.errors.get() > 0) {
+ span.setStatus(StatusCode.ERROR);
+ }
+ span.end();
}
loglog.warn("Log replay finished. recoveryInfo={}", recoveryInfo);
@@ -2169,7 +2196,18 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
}
public void doReplay(TransactionLog translog) {
- try {
+ final Span span =
+ TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay.log").startSpan();
+ TraceUtils.ifNotNoop(
+ span,
+ s -> {
+ if (translog.tlog != null) {
+ s.setAttribute("updatelog.replay.logFile", translog.tlog.getFileName().toString());
+ }
+ s.setAttribute("updatelog.replay.logSizeBytes", translog.getLogSize());
+ });
+ try (var scope = span.makeCurrent()) {
+ assert scope != null; // prevent javac warning about scope being unused
loglog.warn(
"Starting log replay {} active={} starting pos={} inSortedOrder={}",
translog,
@@ -2346,10 +2384,12 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
} catch (ClassCastException cl) {
recoveryInfo.errors.incrementAndGet();
loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log. Entry={}", o, cl);
+ span.recordException(cl);
// would be caused by a corrupt transaction log
} catch (Exception ex) {
recoveryInfo.errors.incrementAndGet();
loglog.warn("REPLAY_ERR: Exception replaying log", ex);
+ span.recordException(ex);
// something wrong with the request?
}
assert TestInjection.injectUpdateLogReplayRandomPause();
@@ -2393,6 +2433,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {
} finally {
if (tlogReader != null) tlogReader.close();
translog.decref();
+ span.end();
}
}