Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: trace the full HTTP response lifecycle (#11050)

Shahmir Varqhaa292369b1233
diff --git a/marimo/_server/api/middleware.py b/marimo/_server/api/middleware.py
index e9659cc625..be420fdc91 100644
--- a/marimo/_server/api/middleware.py
+++ b/marimo/_server/api/middleware.py
@@ -25,7 +25,6 @@ from starlette.background import BackgroundTask
from starlette.middleware.base import (
BaseHTTPMiddleware,
DispatchFunction,
- RequestResponseEndpoint,
)
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
@@ -45,7 +44,7 @@ from marimo._utils.print import print_tabbed
if TYPE_CHECKING:
from starlette.datastructures import State
from starlette.requests import HTTPConnection
- from starlette.types import ASGIApp, Receive, Scope, Send
+ from starlette.types import ASGIApp, Message, Receive, Scope, Send
LOGGER = _loggers.marimo_logger()
@@ -173,11 +172,11 @@ class SkewProtectionMiddleware:
return await self.app(scope, receive, send)
-class OpenTelemetryMiddleware(BaseHTTPMiddleware):
- def __init__(
- self, app: ASGIApp, dispatch: DispatchFunction | None = None
- ) -> None:
- super().__init__(app, dispatch)
+class OpenTelemetryMiddleware:
+ """Keep the request span current until the response finishes streaming."""
+
+ def __init__(self, app: ASGIApp) -> None:
+ self.app = app
self._tracing_enabled = is_tracing_enabled()
if not self._tracing_enabled:
@@ -191,16 +190,19 @@ class OpenTelemetryMiddleware(BaseHTTPMiddleware):
self.Status = Status
self.StatusCode = StatusCode
- async def dispatch(
+ async def __call__(
self,
- request: Request,
- call_next: RequestResponseEndpoint,
- ) -> Response:
- if not self._tracing_enabled:
- return await call_next(request)
+ scope: Scope,
+ receive: Receive,
+ send: Send,
+ ) -> None:
+ if not self._tracing_enabled or scope["type"] != "http":
+ await self.app(scope, receive, send)
+ return
from opentelemetry.propagate import extract
+ request = Request(scope)
ctx = extract(carrier=request.headers)
with server_tracer.start_as_current_span(
@@ -212,14 +214,18 @@ class OpenTelemetryMiddleware(BaseHTTPMiddleware):
"http.target": request.url.path or "",
},
) as span:
+
+ async def send_with_status(message: Message) -> None:
+ if message["type"] == "http.response.start":
+ span.set_attribute("http.status_code", message["status"])
+ await send(message)
+
try:
- response = await call_next(request)
- span.set_attribute("http.status_code", response.status_code)
+ await self.app(scope, receive, send_with_status)
span.set_status(self.Status(self.StatusCode.OK))
except Exception as e:
span.set_status(self.Status(self.StatusCode.ERROR, str(e)))
raise
- return response
@dataclass
diff --git a/tests/_server/ai/test_tracing.py b/tests/_server/ai/test_tracing.py
index f0b8d5c2aa..f0003741fa 100644
--- a/tests/_server/ai/test_tracing.py
+++ b/tests/_server/ai/test_tracing.py
@@ -1,6 +1,7 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
+import asyncio
from typing import TYPE_CHECKING, cast
from unittest.mock import patch
@@ -16,9 +17,14 @@ from marimo._server.ai.tracing import (
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Sequence
- from opentelemetry.sdk.trace import ReadableSpan
+ from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
- from opentelemetry.trace import Tracer
+ from opentelemetry.trace import Span, Tracer
+ from starlette.requests import Request
+ from starlette.responses import StreamingResponse
+ from starlette.types import Message, Scope
+
+ from marimo._config.config import CopilotMode
class _CollectingExporter:
@@ -40,13 +46,16 @@ class _CollectingExporter:
return True
-def _setup_tracing() -> tuple[Tracer, _CollectingExporter]:
+def _setup_tracing(
+ provider: TracerProvider | None = None,
+) -> tuple[Tracer, _CollectingExporter]:
"""Build an isolated tracer backed by a collecting exporter."""
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
exporter = _CollectingExporter()
- provider = TracerProvider()
+ if provider is None:
+ provider = TracerProvider()
# `cast` through `object` since the duck-typed exporter can't subclass the
# `SpanExporter` ABC (opentelemetry is an optional dependency).
provider.add_span_processor(
@@ -249,3 +258,167 @@ class TestTraceCompletion:
span = exporter.spans[0]
assert span.status.status_code == StatusCode.ERROR
assert any(e.name == "exception" for e in span.events)
+
+
[email protected]("opentelemetry", "pydantic_ai")
[email protected]("mode", ["manual", "code_mode"])
[email protected]("spec_version", ["2.3", "2.4"])
+async def test_ai_stream_keeps_exportable_parent_tree(
+ mode: CopilotMode, spec_version: str
+) -> None:
+ from opentelemetry import trace
+ from opentelemetry.sdk.trace import TracerProvider
+ from pydantic_ai import Agent, DeferredToolRequests
+ from pydantic_ai.models.instrumented import InstrumentationSettings
+ from pydantic_ai.models.test import TestModel
+ from starlette.applications import Starlette
+ from starlette.routing import Route
+
+ from marimo._server.ai.config import AnyProviderConfig
+ from marimo._server.ai.providers import OpenAIProvider, StreamOptions
+ from marimo._server.api.middleware import OpenTelemetryMiddleware
+
+ tracer_provider = TracerProvider()
+ tracer, exporter = _setup_tracing(tracer_provider)
+ response_started = asyncio.Event()
+ tool_calls: list[str] = []
+
+ async def lookup() -> str:
+ assert request_span.is_recording()
+ assert trace.get_current_span().is_recording()
+ tool_calls.append("lookup")
+ return "found"
+
+ agent = Agent(
+ TestModel(custom_output_text="done"),
+ name="chat",
+ tools=[lookup],
+ output_type=[str, DeferredToolRequests],
+ )
+ agent.instrument = InstrumentationSettings(
+ tracer_provider=tracer_provider, version=5
+ )
+ provider = OpenAIProvider(
+ "test", AnyProviderConfig(api_key="test-key", base_url=None)
+ )
+ request_span: Span = trace.INVALID_SPAN
+ messages: list[Message] = []
+
+ async def endpoint(request: Request) -> StreamingResponse:
+ del request
+ nonlocal request_span
+ request_span = trace.get_current_span()
+ # Chat and code mode both use this production streaming path.
+ response = provider._vercel_streaming_response(
+ agent,
+ [
+ {
+ "id": "user-message",
+ "role": "user",
+ "parts": [{"type": "text", "text": "look it up"}],
+ }
+ ],
+ StreamOptions(
+ span_info=SpanInfo(
+ endpoint="chat", model="openai/test", mode=mode
+ )
+ ),
+ )
+ stream = response.body_iterator
+
+ async def body() -> AsyncIterator[str | bytes | memoryview]:
+ # BaseHTTPMiddleware ends the request span before forwarding
+ # response headers. Waiting here makes that failure deterministic.
+ await asyncio.wait_for(response_started.wait(), timeout=5)
+ assert request_span.is_recording()
+ assert trace.get_current_span() is request_span
+ async for chunk in stream:
+ yield chunk
+ assert request_span.is_recording()
+
+ response.body_iterator = body()
+ return response
+
+ async def receive() -> Message:
+ # ASGI 2.3 listens for disconnects until streaming cancels the listener.
+ await asyncio.wait_for(asyncio.Event().wait(), timeout=5)
+ return {"type": "http.disconnect"}
+
+ async def send(message: Message) -> None:
+ messages.append(message)
+ if message["type"] == "http.response.start":
+ response_started.set()
+ if message["type"] == "http.response.body":
+ assert request_span.is_recording()
+
+ scope: Scope = {
+ "type": "http",
+ "asgi": {"version": "3.0", "spec_version": spec_version},
+ "http_version": "1.1",
+ "method": "POST",
+ "scheme": "http",
+ "path": "/api/ai/chat",
+ "root_path": "",
+ "query_string": b"",
+ "headers": [],
+ }
+ app = Starlette(routes=[Route("/api/ai/chat", endpoint, methods=["POST"])])
+ app.add_middleware(OpenTelemetryMiddleware)
+ with (
+ patch(
+ "marimo._server.api.middleware.is_tracing_enabled",
+ return_value=True,
+ ),
+ patch("marimo._server.api.middleware.server_tracer", tracer),
+ patch("marimo._server.ai.tracing.server_tracer", tracer),
+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),
+ ):
+ await app(scope, receive, send)
+
+ assert tool_calls == ["lookup"]
+ assert messages[-1] == {
+ "type": "http.response.body",
+ "body": b"",
+ "more_body": False,
+ }
+ assert not request_span.is_recording()
+ assert trace.get_current_span() is trace.INVALID_SPAN
+ spans = exporter.spans
+ roots = [span for span in spans if span.parent is None]
+ assert [span.name for span in roots] == ["POST /api/ai/chat"]
+ root_context = roots[0].context
+ assert root_context is not None
+ assert spans[-1] is roots[0]
+ assert root_context == request_span.get_span_context()
+ stream_span = next(
+ span for span in spans if span.name == "marimo.ai.stream"
+ )
+ assert stream_span.parent == roots[0].context
+ assert _attributes(stream_span)["marimo.ai.mode"] == mode
+ operations = [
+ (span.attributes or {}).get("gen_ai.operation.name") for span in spans
+ ]
+ assert "chat" in operations
+ assert "execute_tool" in operations
+ by_id = {}
+ for span in spans:
+ assert span.context is not None
+ by_id[span.context.span_id] = span
+ for child in spans:
+ if child.parent is None:
+ continue
+ # Every exported child must have an exported parent with a lifetime
+ # enclosing its own, including Pydantic AI's agent/model/tool spans.
+ parent = by_id[child.parent.span_id]
+ assert child.context is not None
+ assert child.context.trace_id == root_context.trace_id
+ assert parent.start_time is not None
+ assert child.start_time is not None
+ assert child.end_time is not None
+ assert parent.end_time is not None
+ assert (
+ parent.start_time
+ <= child.start_time
+ <= child.end_time
+ <= parent.end_time
+ )
diff --git a/tests/_server/api/test_opentelemetry_middleware.py b/tests/_server/api/test_opentelemetry_middleware.py
index 5ebf18d41b..be8e7f4158 100644
--- a/tests/_server/api/test_opentelemetry_middleware.py
+++ b/tests/_server/api/test_opentelemetry_middleware.py
@@ -1,20 +1,24 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
+import asyncio
import sys
from typing import TYPE_CHECKING, Any
from unittest.mock import patch
import pytest
from starlette.applications import Starlette
-from starlette.responses import PlainTextResponse
+from starlette.background import BackgroundTask
+from starlette.responses import PlainTextResponse, StreamingResponse
from starlette.routing import Route
from starlette.testclient import TestClient
if TYPE_CHECKING:
from collections.abc import Sequence
+ from opentelemetry.trace import Span
from starlette.requests import Request
+ from starlette.types import Message, Receive, Scope, Send
def _reset_otel() -> None:
@@ -172,6 +176,180 @@ class TestOpenTelemetryMiddleware:
assert attrs["http.method"] == "GET"
assert attrs["http.target"] == "/"
+ @pytest.mark.parametrize("async_background", [False, True])
+ def test_request_span_outlives_response_background_task(
+ self, async_background: bool
+ ) -> None:
+ from opentelemetry import trace
+
+ from marimo._server.api.middleware import OpenTelemetryMiddleware
+
+ tracer, exporter = _setup_tracing()
+ request_span: Span = trace.INVALID_SPAN
+
+ def background_work() -> None:
+ assert request_span.is_recording()
+ assert trace.get_current_span() is request_span
+ assert exporter.spans == []
+ with tracer.start_as_current_span("background-task"):
+ request_span.set_attribute("background.completed", True)
+
+ async def async_background_work() -> None:
+ await asyncio.sleep(0)
+ background_work()
+
+ async def endpoint(request: Request) -> StreamingResponse:
+ del request
+ nonlocal request_span
+ request_span = trace.get_current_span()
+ background = BackgroundTask(background_work)
+ if async_background:
+ background = BackgroundTask(async_background_work)
+ return StreamingResponse(
+ iter([b"response"]), background=background
+ )
+
+ with (
+ patch(
+ "marimo._server.api.middleware.is_tracing_enabled",
+ return_value=True,
+ ),
+ patch("marimo._server.api.middleware.server_tracer", tracer),
+ ):
+ app = Starlette(routes=[Route("/", endpoint)])
+ app.add_middleware(OpenTelemetryMiddleware)
+ response = TestClient(app).get("/")
+
+ assert response.status_code == 200
+ assert response.content == b"response"
+ assert not request_span.is_recording()
+ assert [span.name for span in exporter.spans] == [
+ "background-task",
+ "GET /",
+ ]
+ child, parent = exporter.spans
+ assert child.parent == parent.context
+ assert dict(parent.attributes or {}) == {
+ "http.method": "GET",
+ "http.target": "/",
+ "http.status_code": 200,
+ "background.completed": True,
+ }
+ assert (
+ parent.start_time
+ <= child.start_time
+ <= child.end_time
+ <= parent.end_time
+ )
+
+ @pytest.mark.parametrize(
+ "error_type", [RuntimeError, asyncio.CancelledError]
+ )
+ async def test_span_ends_when_stream_is_interrupted(
+ self, error_type: type[BaseException]
+ ) -> None:
+ from opentelemetry import trace
+ from opentelemetry.trace import StatusCode
+
+ from marimo._server.api.middleware import OpenTelemetryMiddleware
+
+ tracer, exporter = _setup_tracing()
+ request_span: Span = trace.INVALID_SPAN
+ messages: list[Message] = []
+
+ async def app(scope: Scope, receive: Receive, send: Send) -> None:
+ del scope, receive
+ nonlocal request_span
+ request_span = trace.get_current_span()
+ await send(
+ {"type": "http.response.start", "status": 200, "headers": []}
+ )
+ await send(
+ {
+ "type": "http.response.body",
+ "body": b"partial",
+ "more_body": True,
+ }
+ )
+ assert request_span.is_recording()
+ raise error_type("stream interrupted")
+
+ async def receive() -> Message:
+ pytest.fail("The tracing middleware must not consume the request")
+
+ async def send(message: Message) -> None:
+ messages.append(message)
+
+ scope: Scope = {
+ "type": "http",
+ "method": "GET",
+ "scheme": "http",
+ "path": "/stream",
+ "query_string": b"",
+ "headers": [],
+ }
+ with (
+ patch(
+ "marimo._server.api.middleware.is_tracing_enabled",
+ return_value=True,
+ ),
+ patch("marimo._server.api.middleware.server_tracer", tracer),
+ pytest.raises(error_type, match="stream interrupted"),
+ ):
+ await OpenTelemetryMiddleware(app)(scope, receive, send)
+
+ assert len(messages) == 2
+ assert not request_span.is_recording()
+ assert trace.get_current_span() is trace.INVALID_SPAN
+ assert len(exporter.spans) == 1
+ span = exporter.spans[0]
+ assert span.context == request_span.get_span_context()
+ assert dict(span.attributes or {}) == {
+ "http.method": "GET",
+ "http.target": "/stream",
+ "http.status_code": 200,
+ }
+ if error_type is RuntimeError:
+ assert span.status.status_code == StatusCode.ERROR
+ assert any(event.name == "exception" for event in span.events)
+
+ @pytest.mark.parametrize("scope_type", ["websocket", "lifespan"])
+ async def test_non_http_scopes_pass_through(self, scope_type: str) -> None:
+ from marimo._server.api.middleware import OpenTelemetryMiddleware
+
+ tracer, exporter = _setup_tracing()
+ scope: Scope = {"type": scope_type}
+ calls: list[Scope] = []
+
+ async def app(
+ received_scope: Scope, receive: Receive, send: Send
+ ) -> None:
+ assert received_scope is scope
+ assert receive is expected_receive
+ assert send is expected_send
+ calls.append(received_scope)
+
+ async def expected_receive() -> Message:
+ pytest.fail("The tracing middleware must not consume messages")
+
+ async def expected_send(message: Message) -> None:
+ del message
+ pytest.fail("The tracing middleware must not send messages")
+
+ with (
+ patch(
+ "marimo._server.api.middleware.is_tracing_enabled",
+ return_value=True,
+ ),
+ patch("marimo._server.api.middleware.server_tracer", tracer),
+ ):
+ await OpenTelemetryMiddleware(app)(
+ scope, expected_receive, expected_send
+ )
+
+ assert calls == [scope]
+ assert exporter.spans == []
+
@pytest.mark.requires("opentelemetry")
class TestTracePropagationThroughRealApp: