A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(ai): return mimetype in executionResult, and pydantic parse (#11107)
Shahmir Varqhaefca6f35d382
diff --git a/marimo/_ai/_tools/types.py b/marimo/_ai/_tools/types.pyindex 1ecb10e1f0..9c97d89abd 100644--- a/marimo/_ai/_tools/types.py+++ b/marimo/_ai/_tools/types.py@@ -4,6 +4,7 @@ from __future__ import annotationsfrom dataclasses import dataclass, fieldfrom typing import Any, Literal+from marimo._messaging.mimetypes import KnownMimeTypefrom marimo._types.ids import CellId_t, SessionId# helper classes@@ -76,3 +77,4 @@ class CodeExecutionResult:stderr: list[str] = field(default_factory=list)errors: list[str] = field(default_factory=list)error: str | None = None+ output_mimetype: KnownMimeType | None = Nonediff --git a/marimo/_code_mode/_context.py b/marimo/_code_mode/_context.pyindex f297ac7909..d4635342ce 100644--- a/marimo/_code_mode/_context.py+++ b/marimo/_code_mode/_context.py@@ -1470,6 +1470,8 @@ class AsyncCodeModeContext:from marimo._code_mode.screenshot import (ScreenshotError,+ _ScreenshotBytes,+ _ScreenshotDataUrl,_ScreenshotSession,_to_data_url,)@@ -1522,8 +1524,8 @@ class AsyncCodeModeContext:Path(save_to).write_bytes(image) # noqa: ASYNC240if as_data_url:- return _to_data_url(image)- return image+ return _ScreenshotDataUrl(_to_data_url(image))+ return _ScreenshotBytes(image)async def close_screenshot_session(self) -> None:"""Close the Playwright browser opened by :meth:`screenshot`.diff --git a/marimo/_code_mode/screenshot.py b/marimo/_code_mode/screenshot.pyindex 9343369a7a..2e17c2b776 100644--- a/marimo/_code_mode/screenshot.py+++ b/marimo/_code_mode/screenshot.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport asyncioimport base64import time+from dataclasses import dataclassfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit@@ -20,6 +21,7 @@ from marimo._export._pdf_raster import ()if TYPE_CHECKING:+ from marimo._messaging.mimetypes import KnownMimeTypefrom marimo._types.ids import CellId_tLOGGER = _loggers.marimo_logger()@@ -63,6 +65,27 @@ def _to_data_url(image: bytes) -> str:return f"data:image/png;base64,{encoded}"+@dataclass(frozen=True)+class _ScreenshotOutput:+ """Preserve PNG MIME information when formatting screenshot values."""++ data_url: str++ def _mime_(self) -> tuple[KnownMimeType, str]:+ return ("image/png", self.data_url)+++# The display protocol takes precedence over registered bytes/str formatters.+class _ScreenshotBytes(bytes):+ def _display_(self) -> _ScreenshotOutput:+ return _ScreenshotOutput(_to_data_url(self))+++class _ScreenshotDataUrl(str):+ def _display_(self) -> _ScreenshotOutput:+ return _ScreenshotOutput(self)++def _require_playwright() -> Any:"""Import `async_playwright`, raising :class:`ScreenshotError` if missing."""from marimo._dependencies.dependencies import DependencyManagerdiff --git a/marimo/_server/ai/tools/code_mode.py b/marimo/_server/ai/tools/code_mode.pyindex 36a3f0c566..9152eed23f 100644--- a/marimo/_server/ai/tools/code_mode.py+++ b/marimo/_server/ai/tools/code_mode.py@@ -1,7 +1,10 @@# Copyright 2026 Marimo. All rights reserved.from __future__ import annotations-from typing import TYPE_CHECKING+import base64+import binascii+from dataclasses import replace+from typing import TYPE_CHECKING, Any, TypeAliasfrom marimo._ai._tools.types import CodeExecutionResultfrom marimo._server.ai.skills.utils import load_reference@@ -10,12 +13,16 @@ from marimo._server.api.utils import get_code_mode_credentialsfrom marimo._server.scratchpad import run_scratchpad_codeif TYPE_CHECKING:- from pydantic_ai import FunctionToolset+ from pydantic_ai import FunctionToolset, ToolReturnfrom pydantic_ai.capabilities import Capabilityfrom starlette.requests import Requestfrom marimo._session.session import Session+ ExecuteCodeResult: TypeAlias = CodeExecutionResult | ToolReturn+else:+ ExecuteCodeResult: TypeAlias = Any+def build_execute_code_toolset(session: Session,@@ -33,7 +40,7 @@ def build_execute_code_toolset(toolset: FunctionToolset = FunctionToolset()- async def execute_code(code: str) -> CodeExecutionResult:+ async def execute_code(code: str) -> ExecuteCodeResult:"""Run Python inside the running notebook's kernel scratchpad.Use this for all notebook mutations via `marimo._code_mode`.@@ -41,7 +48,7 @@ def build_execute_code_toolset(server_url, auth_token = get_code_mode_credentials(AppState(request), request)- return await run_scratchpad_code(+ result = await run_scratchpad_code(session,request,code=code,@@ -49,6 +56,26 @@ def build_execute_code_toolset(auth_token=auth_token,)+ if result.output_mimetype != "image/png" or result.output is None:+ return result++ from pydantic_ai import BinaryContent, ToolReturn++ prefix = "data:image/png;base64,"+ if not result.output.startswith(prefix):+ return result+ try:+ image = base64.b64decode(+ result.output[len(prefix) :], validate=True+ )+ except (binascii.Error, ValueError):+ # Other PNG formatters may use representations we cannot decode.+ return result+ return ToolReturn(+ return_value=replace(result, output=None),+ content=[BinaryContent(data=image, media_type="image/png")],+ )+toolset.add_function(execute_code,name="execute_code",diff --git a/marimo/_server/scratchpad.py b/marimo/_server/scratchpad.pyindex 620d5d7814..7b312f91b3 100644--- a/marimo/_server/scratchpad.py+++ b/marimo/_server/scratchpad.py@@ -366,6 +366,11 @@ def extract_result(return CodeExecutionResult(success=len(errors) == 0,output=output_data,+ output_mimetype=(+ cell_notif.output.mimetype+ if cell_notif.output is not None+ else None+ ),stdout=stdout,stderr=stderr,errors=errors,diff --git a/tests/_code_mode/test_screenshot.py b/tests/_code_mode/test_screenshot.pyindex 32a9ce3c11..fa2c7a5288 100644--- a/tests/_code_mode/test_screenshot.py+++ b/tests/_code_mode/test_screenshot.py@@ -36,6 +36,7 @@ from marimo._messaging.notebook.outputs import (CellOutputs,notebook_outputs_context,)+from marimo._output.formatting import try_formatfrom marimo._runtime.commands import HTTPRequestfrom marimo._types.ids import CellId_t@@ -305,10 +306,18 @@ async def test_screenshot_passes_credentials_and_reuses_browser():session.return_value.capture.return_value = b"png"ctx = AsyncCodeModeContext(k)- assert await ctx.screenshot(doc_cell.id) == b"png"- assert await ctx.screenshot(doc_cell.id, as_data_url=True) == (- _to_data_url(b"png")- )+ image = await ctx.screenshot(doc_cell.id)+ data_url = await ctx.screenshot(doc_cell.id, as_data_url=True)+ assert isinstance(image, bytes)+ assert image == b"png"+ assert isinstance(data_url, str)+ assert data_url == _to_data_url(b"png")+ for value in (image, data_url):+ formatted = try_format(value)+ assert (formatted.mimetype, formatted.data) == (+ "image/png",+ _to_data_url(b"png"),+ )session.assert_called_once_with("http://localhost:1234/base",screenshot_auth_token="secret",diff --git a/tests/_code_mode/test_screenshot_integration.py b/tests/_code_mode/test_screenshot_integration.pyindex 64b746d562..d13ed00cad 100644--- a/tests/_code_mode/test_screenshot_integration.py+++ b/tests/_code_mode/test_screenshot_integration.py@@ -7,6 +7,8 @@ The Playwright CI job supplies built assets and matching Chromium binaries.from __future__ import annotationsimport asyncio+import base64+import hashlibimport jsonimport osimport signal@@ -26,7 +28,11 @@ if TYPE_CHECKING:async def _execute(- client: httpx.AsyncClient, session_id: str, code: str+ client: httpx.AsyncClient,+ session_id: str,+ code: str,+ *,+ verify_png: bool = False,) -> str:response = await client.post("/api/kernel/execute",@@ -50,6 +56,17 @@ async def _execute(stderr += data["data"]elif event == "done":assert data["success"], stderr+ if verify_png:+ assert data["output"]["mimetype"] == "image/png"+ image_url = data["output"]["data"]+ assert image_url.startswith("data:image/png;base64,")+ png = base64.b64decode(+ image_url.split(",", 1)[1], validate=True+ )+ assert png.startswith(b"\x89PNG\r\n\x1a\n")+ assert (+ "PNG_SHA=" + hashlib.sha256(png).hexdigest() in stdout+ )completed = Trueassert completed, response.textreturn stdout.strip()@@ -215,9 +232,14 @@ retry_image = await session.capture(cell_id)assert retry_image.startswith(b"\\x89PNG\\r\\n\\x1a\\n")assert {marker!r} in await session._page.locator("#output-" + cell_id).inner_text()await session.close()+import hashlib+print("PNG_SHA=" + hashlib.sha256(image).hexdigest())print("CELL_ID=" + cell_id)+image"""- stdout = await _execute(client, session_id, code)+ stdout = await _execute(+ client, session_id, code, verify_png=True+ )cell_id = next(line.removeprefix("CELL_ID=")for line in stdout.splitlines()diff --git a/tests/_server/ai/tools/test_code_mode.py b/tests/_server/ai/tools/test_code_mode.pyindex 82b68c6adf..fe9108a169 100644--- a/tests/_server/ai/tools/test_code_mode.py+++ b/tests/_server/ai/tools/test_code_mode.py@@ -86,3 +86,47 @@ def test_references_capability_exposes_deferred_reference_bundles() -> None:"Improving, optimizing" in by_id["notebook-improvements"].description)assert "Custom widgets" in by_id["rich-representations"].description++[email protected]("pydantic_ai")+ "output",+ [+ "cG5n",+ "{'image/png': 'cG5n'}",+ "data:image/png;base64,cG5n\n",+ "data:image/png;base64,invalid!",+ "data:image/png;base64,cG5",+ "data:image/png;base64,é",+ ],+)+async def test_execute_code_preserves_unsupported_png_output(+ output: str,+) -> None:+ from marimo._ai._tools.types import CodeExecutionResult+ from marimo._server.ai.tools.code_mode import build_execute_code_toolset++ original = CodeExecutionResult(+ success=True,+ output=output,+ output_mimetype="image/png",+ stdout=["capture finished"],+ stderr=["warning"],+ )+ with (+ patch(+ "marimo._server.ai.tools.code_mode.get_code_mode_credentials",+ return_value=("http://localhost:2718", None),+ ),+ patch("marimo._server.ai.tools.code_mode.AppState"),+ patch(+ "marimo._server.ai.tools.code_mode.run_scratchpad_code",+ new=AsyncMock(return_value=original),+ ),+ ):+ toolset = build_execute_code_toolset(MagicMock(), MagicMock())+ execute_code = cast(+ Callable[[str], Awaitable[object]],+ toolset.tools["execute_code"].function,+ )+ assert await execute_code("capture()") is originaldiff --git a/tests/_server/ai/tools/test_screenshot_transport.py b/tests/_server/ai/tools/test_screenshot_transport.pynew file mode 100644index 0000000000..2942fac171--- /dev/null+++ b/tests/_server/ai/tools/test_screenshot_transport.py@@ -0,0 +1,106 @@+# Copyright 2026 Marimo. All rights reserved.+from __future__ import annotations++import base64+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from marimo._code_mode.screenshot import (+ _ScreenshotBytes,+ _ScreenshotDataUrl,+ _to_data_url,+)+from marimo._messaging.cell_output import CellChannel, CellOutput+from marimo._messaging.notification import CellNotification+from marimo._output.formatting import try_format+from marimo._runtime.scratch import SCRATCH_CELL_ID+from marimo._server.scratchpad import extract_result++[email protected]("pydantic_ai")[email protected]("as_data_url", [False, True])+async def test_screenshot_reaches_model_as_image(as_data_url: bool) -> None:+ from pydantic_ai import Agent, BinaryContent+ from pydantic_ai.messages import (+ ModelMessage,+ ModelRequest,+ ModelResponse,+ TextPart,+ ToolCallPart,+ ToolReturnPart,+ UserPromptPart,+ )+ from pydantic_ai.models.function import AgentInfo, FunctionModel++ from marimo._server.ai.tools.code_mode import build_execute_code_toolset++ png = b"\x89PNG\r\n\x1a\ntransport-test"+ value = (+ _ScreenshotDataUrl(_to_data_url(png))+ if as_data_url+ else _ScreenshotBytes(png)+ )+ formatted = try_format(value)+ assert formatted.mimetype == "image/png"+ assert formatted.data == _to_data_url(png)+ assert base64.b64decode(formatted.data.split(",", 1)[1]) == png+ session = MagicMock()+ session.session_view.cell_notifications = {+ SCRATCH_CELL_ID: CellNotification(+ cell_id=SCRATCH_CELL_ID,+ output=CellOutput(+ channel=CellChannel.OUTPUT,+ mimetype=formatted.mimetype,+ data=formatted.data,+ ),+ console=None,+ status="idle",+ )+ }+ result = extract_result(session)+ calls = 0++ def model(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse:+ nonlocal calls+ calls += 1+ if calls == 1:+ return ModelResponse(+ parts=[ToolCallPart("execute_code", {"code": "capture"})]+ )+ images = []+ for message in messages:+ if not isinstance(message, ModelRequest):+ continue+ for part in message.parts:+ if isinstance(part, UserPromptPart) and isinstance(+ part.content, list+ ):+ images.extend(+ item+ for item in part.content+ if isinstance(item, BinaryContent)+ )+ if isinstance(part, ToolReturnPart):+ assert "data:image" not in part.model_response_str()+ assert images == [BinaryContent(data=png, media_type="image/png")]+ return ModelResponse(parts=[TextPart("Image received")])++ with (+ patch(+ "marimo._server.ai.tools.code_mode.get_code_mode_credentials",+ return_value=("http://localhost", None),+ ),+ patch("marimo._server.ai.tools.code_mode.AppState"),+ patch(+ "marimo._server.ai.tools.code_mode.run_scratchpad_code",+ new=AsyncMock(return_value=result),+ ),+ ):+ agent = Agent(+ FunctionModel(model),+ toolsets=[build_execute_code_toolset(session, MagicMock())],+ )+ response = await agent.run("Capture the cell")+ assert response.output == "Image received"+ assert calls == 2