A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: propagate request trace context to the kernel (#10166)
Myles Scolnick22ab341423bf
diff --git a/development_docs/traces.md b/development_docs/traces.mdindex d59d6e4803..cc594cda0e 100644--- a/development_docs/traces.md+++ b/development_docs/traces.md@@ -87,6 +87,14 @@ resulting spans are linked as children of the caller's trace. No extraconfiguration is needed — propagation works automatically whenever tracing isenabled.+Propagation also reaches the kernel, which runs in a separate process. The+request headers travel with each control command, and `handle_message`+re-attaches the extracted trace context before dispatching, so kernel spans+(cell execution, dataset previews, function calls, etc.) become children of the+originating request's trace. This lets a single distributed trace span an+upstream app (e.g., a FastAPI gateway instrumented with Logfire), the marimo+server, and the kernel — even when marimo is mounted as an ASGI sub-app.+## Profiling the kernelYou can generate profiling statistics of the kernel in edit mode using thediff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex b03a535213..a271799c12 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -156,7 +156,7 @@ from marimo._sql.engines.types import (from marimo._sql.get_engines import (get_engines_from_variables,)-from marimo._tracer import kernel_tracer+from marimo._tracer import attach_trace_context, kernel_tracerfrom marimo._types.ids import CellId_t, UIElementId, VariableNamefrom marimo._types.lifespan import Lifespanfrom marimo._utils.lifespans import Lifespans@@ -2338,7 +2338,12 @@ class Kernel:acquiring an RLock costs ~100ns so the overhead is negligible."""LOGGER.debug("Acquiring globals lock to handle request %s", request)- with self.lock_globals():+ # Link kernel spans to the trace of the originating HTTP request (if+ # the request carried W3C trace headers), so distributed traces span+ # the server and the kernel.+ http_request = getattr(request, "request", None)+ headers = getattr(http_request, "headers", None)+ with self.lock_globals(), attach_trace_context(headers):LOGGER.debug("Handling control request: %s", request)await self.router.dispatch(request)LOGGER.debug("Handled control request: %s", request)diff --git a/marimo/_tracer.py b/marimo/_tracer.pyindex 7f813be96a..a518a60daa 100644--- a/marimo/_tracer.py+++ b/marimo/_tracer.py@@ -14,7 +14,7 @@ from marimo._utils.platform import is_pyodideLOGGER = _loggers.marimo_logger()if TYPE_CHECKING:- from collections.abc import Sequence+ from collections.abc import Iterator, Mapping, Sequencefrom pathlib import Pathfrom opentelemetry import trace@@ -285,6 +285,43 @@ def create_tracer(trace_name: str) -> trace.Tracer:return cast(Any, MockTracer()) # type: ignore[no-any-return]+@contextmanager+def attach_trace_context(+ headers: Mapping[str, str] | None,+) -> Iterator[None]:+ """Attach the W3C trace context from incoming request headers.++ Extracts a `traceparent`/`tracestate` (or any configured propagator's)+ context from `headers` and installs it as the current context for the+ duration of the block. This links kernel spans to the trace of the+ originating HTTP request, so a distributed trace (e.g. from an upstream+ FastAPI/Starlette app) can span both the marimo server and the kernel,+ even though the kernel runs in a separate process.++ No-op when there are no headers, tracing is disabled, or opentelemetry+ is unavailable.+ """+ if not headers or is_pyodide() or GLOBAL_SETTINGS.TRACING is False:+ yield+ return++ try:+ from opentelemetry import context as otel_context+ from opentelemetry.propagate import extract++ token = otel_context.attach(extract(carrier=dict(headers)))+ except Exception as e:+ # A missing dependency or malformed header must never break dispatch.+ LOGGER.debug("Failed to attach trace context", exc_info=e)+ yield+ return++ try:+ yield+ finally:+ otel_context.detach(token)++try:_set_tracer_provider()except Exception as e:diff --git a/tests/_runtime/test_trace.py b/tests/_runtime/test_trace.pyindex 9d586dca96..b6eb8145e9 100644--- a/tests/_runtime/test_trace.py+++ b/tests/_runtime/test_trace.py@@ -8,9 +8,12 @@ import subprocessimport sysimport textwrapfrom typing import TYPE_CHECKING+from unittest.mock import patchfrom marimo._runtime.commands import (ExecuteCellCommand,+ ExecuteCellsCommand,+ HTTPRequest,)from marimo._runtime.runtime import Kernelfrom marimo._utils import async_path@@ -482,3 +485,53 @@ class TestEmbedTrace:assert "ZeroDivisionError: division by zero" in resultassert (file_path + "", line 17") in resultassert "y / x" in result+++def _http_request_with_headers(headers: dict[str, str]) -> HTTPRequest:+ return HTTPRequest(+ url={"path": "/", "port": None, "scheme": "http"},+ base_url={"path": "/", "port": None, "scheme": "http"},+ headers=headers,+ query_params={},+ path_params={},+ cookies={},+ meta={},+ user={},+ )+++class TestTraceContextPropagation:+ @staticmethod+ async def test_handle_message_attaches_request_trace_context(+ k: Kernel,+ ) -> None:+ """The kernel links spans to the trace carried by the request."""+ headers = {+ "traceparent": (+ "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"+ )+ }+ command = ExecuteCellsCommand(+ cell_ids=["0"],+ codes=["x = 1"],+ request=_http_request_with_headers(headers),+ )++ with patch(+ "marimo._runtime.runtime.attach_trace_context"+ ) as mock_attach:+ await k.handle_message(command)++ mock_attach.assert_called_once_with(headers)++ @staticmethod+ async def test_handle_message_without_request(k: Kernel) -> None:+ """Commands without a request pass no headers (no-op propagation)."""+ command = ExecuteCellsCommand(cell_ids=["0"], codes=["x = 1"])++ with patch(+ "marimo._runtime.runtime.attach_trace_context"+ ) as mock_attach:+ await k.handle_message(command)++ mock_attach.assert_called_once_with(None)diff --git a/tests/test_tracer.py b/tests/test_tracer.pyindex de3ef2ef82..759cc1c89f 100644--- a/tests/test_tracer.py+++ b/tests/test_tracer.py@@ -454,3 +454,83 @@ class TestInstrumentAI:side_effect=RuntimeError("boom"),):_instrument_ai(MagicMock())++[email protected]("opentelemetry")+class TestAttachTraceContext:+ """Tests for attach_trace_context() cross-process propagation."""++ def setup_method(self) -> None:+ _reset_otel()++ def teardown_method(self) -> None:+ _reset_otel()++ def _traceparent_for(self, trace_id: int, span_id: int) -> str:+ return f"00-{trace_id:032x}-{span_id:016x}-01"++ def test_links_span_to_incoming_traceparent(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ from opentelemetry import trace+ from opentelemetry.sdk.trace import TracerProvider++ from marimo._config.settings import GLOBAL_SETTINGS+ from marimo._tracer import attach_trace_context++ monkeypatch.setattr(GLOBAL_SETTINGS, "TRACING", True)+ trace.set_tracer_provider(TracerProvider())+ tracer = trace.get_tracer("test")++ trace_id = 0x0AF7651916CD43DD8448EB211C80319C+ span_id = 0xB7AD6B7169203331+ headers = {"traceparent": self._traceparent_for(trace_id, span_id)}++ with attach_trace_context(headers):+ with tracer.start_as_current_span("child") as span:+ ctx = span.get_span_context()+ # Child span inherits the incoming trace id and is parented to+ # the incoming span.+ assert ctx.trace_id == trace_id+ assert span.parent is not None+ assert span.parent.span_id == span_id++ def test_noop_when_no_headers(self) -> None:+ from marimo._tracer import attach_trace_context++ # Should not raise and should be a pure no-op.+ with attach_trace_context(None):+ pass+ with attach_trace_context({}):+ pass++ def test_noop_when_tracing_disabled(+ self,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ from opentelemetry import trace+ from opentelemetry.sdk.trace import TracerProvider++ from marimo._config.settings import GLOBAL_SETTINGS+ from marimo._tracer import attach_trace_context++ monkeypatch.setattr(GLOBAL_SETTINGS, "TRACING", False)+ trace.set_tracer_provider(TracerProvider())+ tracer = trace.get_tracer("test")++ headers = {+ "traceparent": self._traceparent_for(+ 0x0AF7651916CD43DD8448EB211C80319C, 0xB7AD6B7169203331+ )+ }++ with attach_trace_context(headers):+ with tracer.start_as_current_span("child") as span:+ # No context attached; the span starts a fresh, parentless+ # trace.+ assert (+ span.get_span_context().trace_id+ != 0x0AF7651916CD43DD8448EB211C80319C+ )+ assert span.parent is None