A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add more trace attributes under an AI span (#9968)
Shahmir Varqhabefed39aa127
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.pyindex 049956ecd2..b55281182b 100644--- a/marimo/_server/ai/providers.py+++ b/marimo/_server/ai/providers.py@@ -34,6 +34,11 @@ from marimo._server.ai.constants import ANTHROPIC_DEFAULT_MAX_TOKENSfrom marimo._server.ai.ids import AiModelId, AiProviderIdfrom marimo._server.ai.tools.tool_manager import get_tool_managerfrom marimo._server.ai.tools.types import ToolDefinition+from marimo._server.ai.tracing import (+ SpanInfo,+ trace_completion,+ trace_stream,+)from marimo._server.models.completion import UIMessage as ServerUIMessagefrom marimo._utils.http import HTTPStatusfrom marimo._utils.typing import override@@ -72,18 +77,12 @@ LOGGER = _loggers.marimo_logger()@dataclassclass StreamOptions:+ span_info: SpanInfotext_only: bool = Falseformat_stream: bool = Falseaccept: str | None = None-@dataclass-class ActiveToolCall:- tool_call_id: str- tool_call_name: str- tool_call_args: str--ProviderT = TypeVar("ProviderT", bound="Provider", covariant=True)@@ -124,6 +123,8 @@ class PydanticProvider(ABC, Generic[ProviderT]):def create_agent(self,+ *,+ name: str,max_tokens: int | None,tools: list[ToolDefinition],system_prompt: str,@@ -135,6 +136,7 @@ class PydanticProvider(ABC, Generic[ProviderT]):toolset, output_type = self._get_toolsets_and_output_type(tools)return Agent(model,+ name=name,model_settings=self._build_agent_settings(model),toolsets=[toolset] if tools else None,instructions=system_prompt,@@ -172,15 +174,20 @@ class PydanticProvider(ABC, Generic[ProviderT]):system_prompt: str,max_tokens: int | None,additional_tools: list[ToolDefinition],- stream_options: StreamOptions | None = None,+ stream_options: StreamOptions,) -> StreamingResponse:"""Return a streaming response from the given messages. The response are AI SDK events."""from pydantic_ai.ui.vercel_ai import VercelAIAdapterfrom pydantic_ai.ui.vercel_ai.request_types import SubmitMessagetools = (self.config.tools or []) + additional_tools+ stream_options.span_info.tool_count = len(tools)+agent = self.create_agent(- max_tokens=max_tokens, tools=tools, system_prompt=system_prompt+ name=stream_options.span_info.endpoint,+ max_tokens=max_tokens,+ tools=tools,+ system_prompt=system_prompt,)run_input = SubmitMessage(@@ -189,9 +196,6 @@ class PydanticProvider(ABC, Generic[ProviderT]):messages=self.convert_messages(messages),)- # TODO: Text only and format stream are not supported yet- stream_options = stream_options or StreamOptions()-adapter = VercelAIAdapter(agent=agent,run_input=run_input,@@ -199,6 +203,7 @@ class PydanticProvider(ABC, Generic[ProviderT]):sdk_version=AI_SDK_VERSION,)event_stream = adapter.run_stream()+ event_stream = trace_stream(event_stream, stream_options.span_info)return adapter.streaming_response(event_stream)async def completion(@@ -207,20 +212,28 @@ class PydanticProvider(ABC, Generic[ProviderT]):system_prompt: str,max_tokens: int,additional_tools: list[ToolDefinition],+ span_info: SpanInfo,) -> str:"""Return a string response from the given messages."""from pydantic_ai.ui.vercel_ai import VercelAIAdaptertools = (self.config.tools or []) + additional_tools+ span_info.tool_count = len(tools)+agent = self.create_agent(- max_tokens=max_tokens, tools=tools, system_prompt=system_prompt- )- result = await agent.run(- user_prompt=None,- message_history=VercelAIAdapter.load_messages(messages),+ name=span_info.endpoint,+ max_tokens=max_tokens,+ tools=tools,+ system_prompt=system_prompt,)+ with trace_completion(span_info):+ result = await agent.run(+ user_prompt=None,+ message_history=VercelAIAdapter.load_messages(messages),+ )+return str(result.output)async def stream_completion_harness(@@ -231,7 +244,7 @@ class PydanticProvider(ABC, Generic[ProviderT]):session: Session,request: Request,max_tokens: int | None,- stream_options: StreamOptions | None,+ stream_options: StreamOptions,) -> StreamingResponse:"""Experimental method to return code-mode streaming responses"""from pydantic_ai import Agent@@ -255,8 +268,8 @@ class PydanticProvider(ABC, Generic[ProviderT]):trigger="submit-message",messages=self.convert_messages(messages),)+ stream_options.span_info.tool_count = 1- stream_options = stream_options or StreamOptions()adapter = VercelAIAdapter(agent=agent,run_input=run_input,@@ -264,6 +277,7 @@ class PydanticProvider(ABC, Generic[ProviderT]):sdk_version=AI_SDK_VERSION,)event_stream = adapter.run_stream()+ event_stream = trace_stream(event_stream, stream_options.span_info)return adapter.streaming_response(event_stream)def _get_toolsets_and_output_type(@@ -782,6 +796,8 @@ class CustomProvider(OpenAIClientMixin, PydanticProvider["Provider"]):@overridedef create_agent(self,+ *,+ name: str,max_tokens: int | None,tools: list[ToolDefinition],system_prompt: str,@@ -816,6 +832,7 @@ class CustomProvider(OpenAIClientMixin, PydanticProvider["Provider"]):toolset, output_type = self._get_toolsets_and_output_type(tools)return Agent(model,+ name=name,model_settings=agent_settings,toolsets=[toolset] if tools else None,instructions=system_prompt,diff --git a/marimo/_server/ai/tracing.py b/marimo/_server/ai/tracing.pynew file mode 100644index 0000000000..601023c458--- /dev/null+++ b/marimo/_server/ai/tracing.py@@ -0,0 +1,97 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from contextlib import contextmanager+from dataclasses import dataclass+from typing import TYPE_CHECKING, Literal++from marimo._config.config import CopilotMode+from marimo._config.settings import GLOBAL_SETTINGS+from marimo._server.ai.ids import AiModelId+from marimo._tracer import server_tracer++if TYPE_CHECKING:+ from collections.abc import AsyncIterator, Generator++ from opentelemetry.trace import Span+ from opentelemetry.util.types import AttributeValue+ from pydantic_ai.ui.vercel_ai.response_types import BaseChunk+++@dataclass+class SpanInfo:+ endpoint: Literal["completion", "chat", "inline_completion", "invoke_tool"]+ model: str # qualified "<provider>/<model>"+ mode: CopilotMode | None = None+ language: str | None = None+ session_id: str | None = None+ tool_count: int | None = None+++def build_attributes(span_info: SpanInfo) -> dict[str, AttributeValue]:+ model_id = AiModelId.from_model(span_info.model)+ attributes: dict[str, AttributeValue] = {+ "marimo.ai.endpoint": span_info.endpoint,+ "marimo.ai.provider": model_id.provider,+ "marimo.ai.model": model_id.model,+ }+ if span_info.mode:+ attributes["marimo.ai.mode"] = span_info.mode+ if span_info.language:+ attributes["marimo.ai.language"] = span_info.language+ if span_info.session_id:+ attributes["marimo.ai.session_id"] = span_info.session_id+ if span_info.tool_count is not None:+ attributes["marimo.ai.tool_count"] = span_info.tool_count+ return attributes+++def _record_error(span: Span, e: Exception) -> None:+ from opentelemetry.trace.status import Status, StatusCode++ span.set_status(Status(StatusCode.ERROR, str(e)))+ span.record_exception(e)+++async def trace_stream(+ stream: AsyncIterator[BaseChunk], span_info: SpanInfo | None+) -> AsyncIterator[BaseChunk]:+ """Wrap a streaming response in a `marimo.ai.stream` span.++ The span stays open for the lifetime of the stream so that pydantic-ai's+ `gen_ai.*` spans nest underneath it and the span duration reflects the full+ streaming latency.+ """+ if not GLOBAL_SETTINGS.TRACING or span_info is None:+ async for event in stream:+ yield event+ return++ with server_tracer.start_as_current_span(+ "marimo.ai.stream", attributes=build_attributes(span_info)+ ) as span:+ try:+ async for event in stream:+ yield event+ except Exception as e:+ _record_error(span, e)+ raise+++@contextmanager+def trace_completion(+ span_info: SpanInfo | None,+) -> Generator[None, None, None]:+ """Wrap a non-streaming completion in a `marimo.ai.completion` span."""+ if not GLOBAL_SETTINGS.TRACING or span_info is None:+ yield+ return++ with server_tracer.start_as_current_span(+ "marimo.ai.completion", attributes=build_attributes(span_info)+ ) as span:+ try:+ yield+ except Exception as e:+ _record_error(span, e)+ raisediff --git a/marimo/_server/api/endpoints/ai.py b/marimo/_server/api/endpoints/ai.pyindex 300f1872b0..f93ba02c7b 100644--- a/marimo/_server/api/endpoints/ai.py+++ b/marimo/_server/api/endpoints/ai.py@@ -30,8 +30,12 @@ from marimo._server.ai.prompts import (get_inline_system_prompt,get_refactor_or_insert_notebook_cell_system_prompt,)-from marimo._server.ai.providers import StreamOptions, get_completion_provider+from marimo._server.ai.providers import (+ StreamOptions,+ get_completion_provider,+)from marimo._server.ai.tools.tool_manager import get_tool_manager+from marimo._server.ai.tracing import SpanInfofrom marimo._server.api.deps import AppStatefrom marimo._server.api.utils import parse_requestfrom marimo._server.models.completion import (@@ -127,6 +131,7 @@ async def ai_completion("""app_state = AppState(request)app_state.require_current_session()+ session_id = app_state.require_current_session_id()config = app_state.app_config_manager.get_config(hide_secrets=False)body = await parse_request(request, cls=AiCompletionRequest, allow_unknown_keys=True@@ -158,6 +163,14 @@ async def ai_completion(system_prompt=system_prompt,max_tokens=get_max_tokens(config),additional_tools=[],+ stream_options=StreamOptions(+ span_info=SpanInfo(+ endpoint="completion",+ model=model,+ language=body.language,+ session_id=session_id,+ )+ ),)@@ -212,7 +225,15 @@ async def ai_chat(additional_tools = body.tools or []stream_options = StreamOptions(- format_stream=True, text_only=False, accept=accept+ format_stream=True,+ text_only=False,+ accept=accept,+ span_info=SpanInfo(+ endpoint="chat",+ model=model,+ mode=mode,+ session_id=session_id,+ ),)if mode == "code_mode":@@ -264,6 +285,7 @@ async def ai_inline_completion("""app_state = AppState(request)app_state.require_current_session()+ session_id = app_state.require_current_session_id()config = app_state.app_config_manager.get_config(hide_secrets=False)body = await parse_request(request, cls=AiInlineCompletionRequest, allow_unknown_keys=True@@ -291,6 +313,12 @@ async def ai_inline_completion(system_prompt=system_prompt,max_tokens=INLINE_COMPLETION_MAX_TOKENS,additional_tools=[],+ span_info=SpanInfo(+ endpoint="inline_completion",+ model=model,+ language=body.language,+ session_id=session_id,+ ),)except Exception as e:LOGGER.error("Error in AI inline completion: %s", str(e))diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.pyindex 5ca88d3067..4a096f2127 100644--- a/tests/_server/ai/test_providers.py+++ b/tests/_server/ai/test_providers.py@@ -16,10 +16,12 @@ from marimo._server.ai.providers import (CustomProvider,GoogleProvider,OpenAIProvider,+ StreamOptions,_infer_provider_name_from_base_url,_normalize_base_url,get_completion_provider,)+from marimo._server.ai.tracing import SpanInfo@pytest.mark.parametrize(@@ -444,6 +446,7 @@ async def test_completion_does_not_pass_redundant_instructions() -> None:system_prompt="Test prompt",max_tokens=100,additional_tools=[],+ span_info=SpanInfo(endpoint="completion", model="openai/gpt-4"),)mock_request.assert_called_once()@@ -515,7 +518,7 @@ def test_custom_provider_agent_passes_explicit_max_tokens() -> None:with patch("marimo._server.ai.providers.get_tool_manager") as mock_get_tm:mock_get_tm.return_value = MagicMock()agent = provider.create_agent(- max_tokens=12345, tools=[], system_prompt="x"+ name="test", max_tokens=12345, tools=[], system_prompt="x")assert dict(agent.model_settings or {}).get("max_tokens") == 12345@@ -528,7 +531,7 @@ def test_custom_provider_agent_omits_max_tokens_when_none() -> None:with patch("marimo._server.ai.providers.get_tool_manager") as mock_get_tm:mock_get_tm.return_value = MagicMock()agent = provider.create_agent(- max_tokens=None, tools=[], system_prompt="x"+ name="test", max_tokens=None, tools=[], system_prompt="x")assert "max_tokens" not in dict(agent.model_settings or {})@@ -682,6 +685,9 @@ async def test_stream_completion_harness_wires_execute_code_toolset() -> None:streaming_response = MagicMock(name="streaming_response")adapter: MagicMock = MagicMock(name="adapter")adapter.streaming_response = MagicMock(return_value=streaming_response)+ stream_options = StreamOptions(+ span_info=SpanInfo(endpoint="chat", model="openai/gpt-4"),+ )with (patch.object(provider, "create_model", return_value=MagicMock()),@@ -703,10 +709,11 @@ async def test_stream_completion_harness_wires_execute_code_toolset() -> None:session=session,request=request,max_tokens=1234,- stream_options=None,+ stream_options=stream_options,)assert result is streaming_response+ assert stream_options.span_info.tool_count == 1# The toolset is bound to the caller's session and request.mock_build_toolset.assert_called_once_with(session, request)diff --git a/tests/_server/ai/test_tracing.py b/tests/_server/ai/test_tracing.pynew file mode 100644index 0000000000..f0b8d5c2aa--- /dev/null+++ b/tests/_server/ai/test_tracing.py@@ -0,0 +1,251 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++from typing import TYPE_CHECKING, cast+from unittest.mock import patch++import pytest++from marimo._server.ai.tracing import (+ SpanInfo,+ build_attributes,+ trace_completion,+ trace_stream,+)++if TYPE_CHECKING:+ from collections.abc import AsyncIterator, Sequence++ from opentelemetry.sdk.trace import ReadableSpan+ from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult+ from opentelemetry.trace import Tracer+++class _CollectingExporter:+ """Minimal span exporter that collects spans in a list."""++ def __init__(self) -> None:+ self.spans: list[ReadableSpan] = []++ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:+ from opentelemetry.sdk.trace.export import SpanExportResult++ self.spans.extend(spans)+ return SpanExportResult.SUCCESS++ def shutdown(self) -> None:+ pass++ def force_flush(self, _timeout_millis: int = 0) -> bool:+ return True+++def _setup_tracing() -> tuple[Tracer, _CollectingExporter]:+ """Build an isolated tracer backed by a collecting exporter."""+ from opentelemetry.sdk.trace import TracerProvider+ from opentelemetry.sdk.trace.export import SimpleSpanProcessor++ exporter = _CollectingExporter()+ provider = TracerProvider()+ # `cast` through `object` since the duck-typed exporter can't subclass the+ # `SpanExporter` ABC (opentelemetry is an optional dependency).+ provider.add_span_processor(+ SimpleSpanProcessor(cast("SpanExporter", cast(object, exporter)))+ )+ # Use the provider directly rather than the global tracer provider so each+ # test is isolated and we don't mutate global OTel state.+ return provider.get_tracer("marimo.server"), exporter+++def _attributes(span: ReadableSpan) -> dict[str, object]:+ return dict(span.attributes or {})+++async def _gen(*items: str) -> AsyncIterator[str]:+ for item in items:+ yield item+++async def _failing_gen(*items: str) -> AsyncIterator[str]:+ for item in items:+ yield item+ raise RuntimeError("boom")+++class TestBuildAttributes:+ def test_qualified_model_with_mode(self) -> None:+ attrs = build_attributes(+ SpanInfo(endpoint="chat", model="openai/gpt-4o", mode="manual")+ )+ assert attrs == {+ "marimo.ai.endpoint": "chat",+ "marimo.ai.provider": "openai",+ "marimo.ai.model": "gpt-4o",+ "marimo.ai.mode": "manual",+ }++ def test_without_mode_omits_mode_key(self) -> None:+ attrs = build_attributes(+ SpanInfo(endpoint="inline_completion", model="anthropic/claude-3")+ )+ assert attrs == {+ "marimo.ai.endpoint": "inline_completion",+ "marimo.ai.provider": "anthropic",+ "marimo.ai.model": "claude-3",+ }++ def test_includes_optional_fields_when_set(self) -> None:+ attrs = build_attributes(+ SpanInfo(+ endpoint="inline_completion",+ model="openai/gpt-4o",+ language="python",+ session_id="session-123",+ tool_count=3,+ )+ )+ assert attrs == {+ "marimo.ai.endpoint": "inline_completion",+ "marimo.ai.provider": "openai",+ "marimo.ai.model": "gpt-4o",+ "marimo.ai.language": "python",+ "marimo.ai.session_id": "session-123",+ "marimo.ai.tool_count": 3,+ }++ def test_tool_count_zero_is_recorded(self) -> None:+ attrs = build_attributes(+ SpanInfo(endpoint="chat", model="openai/gpt-4o", tool_count=0)+ )+ assert attrs["marimo.ai.tool_count"] == 0++[email protected]("opentelemetry")+class TestTraceStream:+ async def test_passthrough_when_tracing_disabled(self) -> None:+ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(endpoint="chat", model="openai/gpt-4o")++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", False),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ events = [e async for e in trace_stream(_gen("a", "b"), span_info)]++ assert events == ["a", "b"]+ assert exporter.spans == []++ async def test_passthrough_when_span_info_none(self) -> None:+ tracer, exporter = _setup_tracing()++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ events = [e async for e in trace_stream(_gen("a", "b"), None)]++ assert events == ["a", "b"]+ assert exporter.spans == []++ async def test_creates_span_with_attributes(self) -> None:+ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(+ endpoint="chat", model="openai/gpt-4o", mode="manual"+ )++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ events = [e async for e in trace_stream(_gen("a", "b"), span_info)]++ assert events == ["a", "b"]+ assert len(exporter.spans) == 1+ span = exporter.spans[0]+ assert span.name == "marimo.ai.stream"+ assert _attributes(span) == {+ "marimo.ai.endpoint": "chat",+ "marimo.ai.provider": "openai",+ "marimo.ai.model": "gpt-4o",+ "marimo.ai.mode": "manual",+ }++ async def test_records_error_and_reraises(self) -> None:+ from opentelemetry.trace import StatusCode++ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(endpoint="chat", model="openai/gpt-4o")++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ with pytest.raises(RuntimeError, match="boom"):+ async for _ in trace_stream(_failing_gen("a"), span_info):+ pass++ assert len(exporter.spans) == 1+ span = exporter.spans[0]+ assert span.status.status_code == StatusCode.ERROR+ assert any(e.name == "exception" for e in span.events)++[email protected]("opentelemetry")+class TestTraceCompletion:+ def test_noop_when_tracing_disabled(self) -> None:+ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(+ endpoint="inline_completion", model="openai/gpt-4o"+ )++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", False),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ with trace_completion(span_info):+ pass++ assert exporter.spans == []++ def test_creates_span_with_attributes(self) -> None:+ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(+ endpoint="inline_completion", model="openai/gpt-4o"+ )++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ with trace_completion(span_info):+ pass++ assert len(exporter.spans) == 1+ span = exporter.spans[0]+ assert span.name == "marimo.ai.completion"+ assert _attributes(span) == {+ "marimo.ai.endpoint": "inline_completion",+ "marimo.ai.provider": "openai",+ "marimo.ai.model": "gpt-4o",+ }++ def test_records_error_and_reraises(self) -> None:+ from opentelemetry.trace import StatusCode++ tracer, exporter = _setup_tracing()+ span_info = SpanInfo(+ endpoint="inline_completion", model="openai/gpt-4o"+ )++ with (+ patch("marimo._config.settings.GLOBAL_SETTINGS.TRACING", True),+ patch("marimo._server.ai.tracing.server_tracer", tracer),+ ):+ with pytest.raises(RuntimeError, match="kaboom"):+ with trace_completion(span_info):+ raise RuntimeError("kaboom")++ assert len(exporter.spans) == 1+ span = exporter.spans[0]+ assert span.status.status_code == StatusCode.ERROR+ assert any(e.name == "exception" for e in span.events)