Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: propagate request trace context to the kernel (#10166)

Myles Scolnick22ab341423bf
diff --git a/development_docs/traces.md b/development_docs/traces.md
index d59d6e4803..cc594cda0e 100644
--- a/development_docs/traces.md
+++ b/development_docs/traces.md
@@ -87,6 +87,14 @@ resulting spans are linked as children of the caller's trace. No extra
configuration is needed — propagation works automatically whenever tracing is
enabled.
+Propagation also reaches the kernel, which runs in a separate process. The
+request headers travel with each control command, and `handle_message`
+re-attaches the extracted trace context before dispatching, so kernel spans
+(cell execution, dataset previews, function calls, etc.) become children of the
+originating request's trace. This lets a single distributed trace span an
+upstream app (e.g., a FastAPI gateway instrumented with Logfire), the marimo
+server, and the kernel — even when marimo is mounted as an ASGI sub-app.
+
## Profiling the kernel
You can generate profiling statistics of the kernel in edit mode using the
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index b03a535213..a271799c12 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -156,7 +156,7 @@ from marimo._sql.engines.types import (
from marimo._sql.get_engines import (
get_engines_from_variables,
)
-from marimo._tracer import kernel_tracer
+from marimo._tracer import attach_trace_context, kernel_tracer
from marimo._types.ids import CellId_t, UIElementId, VariableName
from marimo._types.lifespan import Lifespan
from marimo._utils.lifespans import Lifespans
@@ -2338,7 +2338,12 @@ class Kernel:
acquiring an RLock costs ~100ns so the overhead is negligible.
"""
LOGGER.debug("Acquiring globals lock to handle request %s", request)
- with self.lock_globals():
+ # Link kernel spans to the trace of the originating HTTP request (if
+ # the request carried W3C trace headers), so distributed traces span
+ # the server and the kernel.
+ http_request = getattr(request, "request", None)
+ headers = getattr(http_request, "headers", None)
+ with self.lock_globals(), attach_trace_context(headers):
LOGGER.debug("Handling control request: %s", request)
await self.router.dispatch(request)
LOGGER.debug("Handled control request: %s", request)
diff --git a/marimo/_tracer.py b/marimo/_tracer.py
index 7f813be96a..a518a60daa 100644
--- a/marimo/_tracer.py
+++ b/marimo/_tracer.py
@@ -14,7 +14,7 @@ from marimo._utils.platform import is_pyodide
LOGGER = _loggers.marimo_logger()
if TYPE_CHECKING:
- from collections.abc import Sequence
+ from collections.abc import Iterator, Mapping, Sequence
from pathlib import Path
from opentelemetry import trace
@@ -285,6 +285,43 @@ def create_tracer(trace_name: str) -> trace.Tracer:
return cast(Any, MockTracer()) # type: ignore[no-any-return]
+@contextmanager
+def attach_trace_context(
+ headers: Mapping[str, str] | None,
+) -> Iterator[None]:
+ """Attach the W3C trace context from incoming request headers.
+
+ Extracts a `traceparent`/`tracestate` (or any configured propagator's)
+ context from `headers` and installs it as the current context for the
+ duration of the block. This links kernel spans to the trace of the
+ originating HTTP request, so a distributed trace (e.g. from an upstream
+ FastAPI/Starlette app) can span both the marimo server and the kernel,
+ even though the kernel runs in a separate process.
+
+ No-op when there are no headers, tracing is disabled, or opentelemetry
+ is unavailable.
+ """
+ if not headers or is_pyodide() or GLOBAL_SETTINGS.TRACING is False:
+ yield
+ return
+
+ try:
+ from opentelemetry import context as otel_context
+ from opentelemetry.propagate import extract
+
+ token = otel_context.attach(extract(carrier=dict(headers)))
+ except Exception as e:
+ # A missing dependency or malformed header must never break dispatch.
+ LOGGER.debug("Failed to attach trace context", exc_info=e)
+ yield
+ return
+
+ try:
+ yield
+ finally:
+ otel_context.detach(token)
+
+
try:
_set_tracer_provider()
except Exception as e:
diff --git a/tests/_runtime/test_trace.py b/tests/_runtime/test_trace.py
index 9d586dca96..b6eb8145e9 100644
--- a/tests/_runtime/test_trace.py
+++ b/tests/_runtime/test_trace.py
@@ -8,9 +8,12 @@ import subprocess
import sys
import textwrap
from typing import TYPE_CHECKING
+from unittest.mock import patch
from marimo._runtime.commands import (
ExecuteCellCommand,
+ ExecuteCellsCommand,
+ HTTPRequest,
)
from marimo._runtime.runtime import Kernel
from marimo._utils import async_path
@@ -482,3 +485,53 @@ class TestEmbedTrace:
assert "ZeroDivisionError: division by zero" in result
assert (file_path + "", line 17") in result
assert "y / x" in result
+
+
+def _http_request_with_headers(headers: dict[str, str]) -> HTTPRequest:
+ return HTTPRequest(
+ url={"path": "/", "port": None, "scheme": "http"},
+ base_url={"path": "/", "port": None, "scheme": "http"},
+ headers=headers,
+ query_params={},
+ path_params={},
+ cookies={},
+ meta={},
+ user={},
+ )
+
+
+class TestTraceContextPropagation:
+ @staticmethod
+ async def test_handle_message_attaches_request_trace_context(
+ k: Kernel,
+ ) -> None:
+ """The kernel links spans to the trace carried by the request."""
+ headers = {
+ "traceparent": (
+ "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"
+ )
+ }
+ command = ExecuteCellsCommand(
+ cell_ids=["0"],
+ codes=["x = 1"],
+ request=_http_request_with_headers(headers),
+ )
+
+ with patch(
+ "marimo._runtime.runtime.attach_trace_context"
+ ) as mock_attach:
+ await k.handle_message(command)
+
+ mock_attach.assert_called_once_with(headers)
+
+ @staticmethod
+ async def test_handle_message_without_request(k: Kernel) -> None:
+ """Commands without a request pass no headers (no-op propagation)."""
+ command = ExecuteCellsCommand(cell_ids=["0"], codes=["x = 1"])
+
+ with patch(
+ "marimo._runtime.runtime.attach_trace_context"
+ ) as mock_attach:
+ await k.handle_message(command)
+
+ mock_attach.assert_called_once_with(None)
diff --git a/tests/test_tracer.py b/tests/test_tracer.py
index de3ef2ef82..759cc1c89f 100644
--- a/tests/test_tracer.py
+++ b/tests/test_tracer.py
@@ -454,3 +454,83 @@ class TestInstrumentAI:
side_effect=RuntimeError("boom"),
):
_instrument_ai(MagicMock())
+
+
[email protected]("opentelemetry")
+class TestAttachTraceContext:
+ """Tests for attach_trace_context() cross-process propagation."""
+
+ def setup_method(self) -> None:
+ _reset_otel()
+
+ def teardown_method(self) -> None:
+ _reset_otel()
+
+ def _traceparent_for(self, trace_id: int, span_id: int) -> str:
+ return f"00-{trace_id:032x}-{span_id:016x}-01"
+
+ def test_links_span_to_incoming_traceparent(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ from opentelemetry import trace
+ from opentelemetry.sdk.trace import TracerProvider
+
+ from marimo._config.settings import GLOBAL_SETTINGS
+ from marimo._tracer import attach_trace_context
+
+ monkeypatch.setattr(GLOBAL_SETTINGS, "TRACING", True)
+ trace.set_tracer_provider(TracerProvider())
+ tracer = trace.get_tracer("test")
+
+ trace_id = 0x0AF7651916CD43DD8448EB211C80319C
+ span_id = 0xB7AD6B7169203331
+ headers = {"traceparent": self._traceparent_for(trace_id, span_id)}
+
+ with attach_trace_context(headers):
+ with tracer.start_as_current_span("child") as span:
+ ctx = span.get_span_context()
+ # Child span inherits the incoming trace id and is parented to
+ # the incoming span.
+ assert ctx.trace_id == trace_id
+ assert span.parent is not None
+ assert span.parent.span_id == span_id
+
+ def test_noop_when_no_headers(self) -> None:
+ from marimo._tracer import attach_trace_context
+
+ # Should not raise and should be a pure no-op.
+ with attach_trace_context(None):
+ pass
+ with attach_trace_context({}):
+ pass
+
+ def test_noop_when_tracing_disabled(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ from opentelemetry import trace
+ from opentelemetry.sdk.trace import TracerProvider
+
+ from marimo._config.settings import GLOBAL_SETTINGS
+ from marimo._tracer import attach_trace_context
+
+ monkeypatch.setattr(GLOBAL_SETTINGS, "TRACING", False)
+ trace.set_tracer_provider(TracerProvider())
+ tracer = trace.get_tracer("test")
+
+ headers = {
+ "traceparent": self._traceparent_for(
+ 0x0AF7651916CD43DD8448EB211C80319C, 0xB7AD6B7169203331
+ )
+ }
+
+ with attach_trace_context(headers):
+ with tracer.start_as_current_span("child") as span:
+ # No context attached; the span starts a fresh, parentless
+ # trace.
+ assert (
+ span.get_span_context().trace_id
+ != 0x0AF7651916CD43DD8448EB211C80319C
+ )
+ assert span.parent is None