Apache Solr open-source search software
SOLR-18124: Add tracing spans for UpdateLog replay (#4962)
David Smiley4aaf68b40fb0
diff --git a/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.ymlnew file mode 100644index 00000000000..655aed1a3a4--- /dev/null+++ b/changelog/unreleased/SOLR-18124-updatelog-replay-tracing.yml@@ -0,0 +1,9 @@+title: Tracing (spans) for replication, recovery, and updatelog replay.+type: changed+authors:+ - name: Gaurav Singh+ nick: gsingh-code+ - name: David Smiley+links:+ - name: SOLR-18124+ url: https://issues.apache.org/jira/browse/SOLR-18124diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.javaindex 2b0f202a4bd..cfe671f36df 100644--- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java+++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java@@ -22,6 +22,8 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactorimport com.carrotsearch.hppc.LongHashSet;import com.carrotsearch.hppc.LongSet;import io.opentelemetry.api.common.Attributes;+import io.opentelemetry.api.trace.Span;+import io.opentelemetry.api.trace.StatusCode;import java.io.Closeable;import java.io.IOException;import java.io.UncheckedIOException;@@ -93,6 +95,7 @@ import org.apache.solr.util.RefCounted;import org.apache.solr.util.TestInjection;import org.apache.solr.util.TimeOut;import org.apache.solr.util.plugin.PluginInfoInitialized;+import org.apache.solr.util.tracing.TraceUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -2129,7 +2132,16 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {// setting request info will help loggingSolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));- try {+ final Span span = TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay").startSpan();+ TraceUtils.ifNotNoop(+ span,+ s -> {+ TraceUtils.setDbInstance(s, req.getCore().getName());+ s.setAttribute("updatelog.replay.state", state.toString());+ s.setAttribute("updatelog.replay.logs", translogs.size());+ });+ try (var scope = span.makeCurrent()) {+ assert scope != null; // prevent javac warning about scope being unusedfor (; ; ) {TransactionLog translog = translogs.pollFirst();if (translog == null) break;@@ -2143,9 +2155,11 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {recoveryInfo.errors.incrementAndGet();log.error("Replay failed due to exception", e);}+ span.recordException(e);} catch (Exception e) {recoveryInfo.errors.incrementAndGet();log.error("Replay failed due to exception", e);+ span.recordException(e);} finally {// change the state while updates are still blocked to prevent racesstate = State.ACTIVE;@@ -2159,6 +2173,19 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {log.error("ERROR: didn't get to recover from tlog {}", translog);translog.decref();}++ TraceUtils.ifNotNoop(+ span,+ s -> {+ s.setAttribute("updatelog.replay.adds", recoveryInfo.adds);+ s.setAttribute("updatelog.replay.deletes", recoveryInfo.deletes);+ s.setAttribute("updatelog.replay.deleteByQuery", recoveryInfo.deleteByQuery);+ s.setAttribute("updatelog.replay.errors", recoveryInfo.errors.get());+ });+ if (recoveryInfo.failed || recoveryInfo.errors.get() > 0) {+ span.setStatus(StatusCode.ERROR);+ }+ span.end();}loglog.warn("Log replay finished. recoveryInfo={}", recoveryInfo);@@ -2169,7 +2196,18 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {}public void doReplay(TransactionLog translog) {- try {+ final Span span =+ TraceUtils.getGlobalTracer().spanBuilder("updatelog.replay.log").startSpan();+ TraceUtils.ifNotNoop(+ span,+ s -> {+ if (translog.tlog != null) {+ s.setAttribute("updatelog.replay.logFile", translog.tlog.getFileName().toString());+ }+ s.setAttribute("updatelog.replay.logSizeBytes", translog.getLogSize());+ });+ try (var scope = span.makeCurrent()) {+ assert scope != null; // prevent javac warning about scope being unusedloglog.warn("Starting log replay {} active={} starting pos={} inSortedOrder={}",translog,@@ -2346,10 +2384,12 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {} catch (ClassCastException cl) {recoveryInfo.errors.incrementAndGet();loglog.warn("REPLAY_ERR: Unexpected log entry or corrupt log. Entry={}", o, cl);+ span.recordException(cl);// would be caused by a corrupt transaction log} catch (Exception ex) {recoveryInfo.errors.incrementAndGet();loglog.warn("REPLAY_ERR: Exception replaying log", ex);+ span.recordException(ex);// something wrong with the request?}assert TestInjection.injectUpdateLogReplayRandomPause();@@ -2393,6 +2433,7 @@ public class UpdateLog implements PluginInfoInitialized, SolrMetricProducer {} finally {if (tlogReader != null) tlogReader.close();translog.decref();+ span.end();}}