Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(ai): return mimetype in executionResult, and pydantic parse (#11107)

Shahmir Varqhaefca6f35d382
diff --git a/marimo/_ai/_tools/types.py b/marimo/_ai/_tools/types.py
index 1ecb10e1f0..9c97d89abd 100644
--- a/marimo/_ai/_tools/types.py
+++ b/marimo/_ai/_tools/types.py
@@ -4,6 +4,7 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any, Literal
+from marimo._messaging.mimetypes import KnownMimeType
from marimo._types.ids import CellId_t, SessionId
# helper classes
@@ -76,3 +77,4 @@ class CodeExecutionResult:
stderr: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list)
error: str | None = None
+ output_mimetype: KnownMimeType | None = None
diff --git a/marimo/_code_mode/_context.py b/marimo/_code_mode/_context.py
index f297ac7909..d4635342ce 100644
--- a/marimo/_code_mode/_context.py
+++ b/marimo/_code_mode/_context.py
@@ -1470,6 +1470,8 @@ class AsyncCodeModeContext:
from marimo._code_mode.screenshot import (
ScreenshotError,
+ _ScreenshotBytes,
+ _ScreenshotDataUrl,
_ScreenshotSession,
_to_data_url,
)
@@ -1522,8 +1524,8 @@ class AsyncCodeModeContext:
Path(save_to).write_bytes(image) # noqa: ASYNC240
if as_data_url:
- return _to_data_url(image)
- return image
+ return _ScreenshotDataUrl(_to_data_url(image))
+ return _ScreenshotBytes(image)
async def close_screenshot_session(self) -> None:
"""Close the Playwright browser opened by :meth:`screenshot`.
diff --git a/marimo/_code_mode/screenshot.py b/marimo/_code_mode/screenshot.py
index 9343369a7a..2e17c2b776 100644
--- a/marimo/_code_mode/screenshot.py
+++ b/marimo/_code_mode/screenshot.py
@@ -10,6 +10,7 @@ from __future__ import annotations
import asyncio
import base64
import time
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
@@ -20,6 +21,7 @@ from marimo._export._pdf_raster import (
)
if TYPE_CHECKING:
+ from marimo._messaging.mimetypes import KnownMimeType
from marimo._types.ids import CellId_t
LOGGER = _loggers.marimo_logger()
@@ -63,6 +65,27 @@ def _to_data_url(image: bytes) -> str:
return f"data:image/png;base64,{encoded}"
+@dataclass(frozen=True)
+class _ScreenshotOutput:
+ """Preserve PNG MIME information when formatting screenshot values."""
+
+ data_url: str
+
+ def _mime_(self) -> tuple[KnownMimeType, str]:
+ return ("image/png", self.data_url)
+
+
+# The display protocol takes precedence over registered bytes/str formatters.
+class _ScreenshotBytes(bytes):
+ def _display_(self) -> _ScreenshotOutput:
+ return _ScreenshotOutput(_to_data_url(self))
+
+
+class _ScreenshotDataUrl(str):
+ def _display_(self) -> _ScreenshotOutput:
+ return _ScreenshotOutput(self)
+
+
def _require_playwright() -> Any:
"""Import `async_playwright`, raising :class:`ScreenshotError` if missing."""
from marimo._dependencies.dependencies import DependencyManager
diff --git a/marimo/_server/ai/tools/code_mode.py b/marimo/_server/ai/tools/code_mode.py
index 36a3f0c566..9152eed23f 100644
--- a/marimo/_server/ai/tools/code_mode.py
+++ b/marimo/_server/ai/tools/code_mode.py
@@ -1,7 +1,10 @@
# Copyright 2026 Marimo. All rights reserved.
from __future__ import annotations
-from typing import TYPE_CHECKING
+import base64
+import binascii
+from dataclasses import replace
+from typing import TYPE_CHECKING, Any, TypeAlias
from marimo._ai._tools.types import CodeExecutionResult
from marimo._server.ai.skills.utils import load_reference
@@ -10,12 +13,16 @@ from marimo._server.api.utils import get_code_mode_credentials
from marimo._server.scratchpad import run_scratchpad_code
if TYPE_CHECKING:
- from pydantic_ai import FunctionToolset
+ from pydantic_ai import FunctionToolset, ToolReturn
from pydantic_ai.capabilities import Capability
from starlette.requests import Request
from marimo._session.session import Session
+ ExecuteCodeResult: TypeAlias = CodeExecutionResult | ToolReturn
+else:
+ ExecuteCodeResult: TypeAlias = Any
+
def build_execute_code_toolset(
session: Session,
@@ -33,7 +40,7 @@ def build_execute_code_toolset(
toolset: FunctionToolset = FunctionToolset()
- async def execute_code(code: str) -> CodeExecutionResult:
+ async def execute_code(code: str) -> ExecuteCodeResult:
"""Run Python inside the running notebook's kernel scratchpad.
Use this for all notebook mutations via `marimo._code_mode`.
@@ -41,7 +48,7 @@ def build_execute_code_toolset(
server_url, auth_token = get_code_mode_credentials(
AppState(request), request
)
- return await run_scratchpad_code(
+ result = await run_scratchpad_code(
session,
request,
code=code,
@@ -49,6 +56,26 @@ def build_execute_code_toolset(
auth_token=auth_token,
)
+ if result.output_mimetype != "image/png" or result.output is None:
+ return result
+
+ from pydantic_ai import BinaryContent, ToolReturn
+
+ prefix = "data:image/png;base64,"
+ if not result.output.startswith(prefix):
+ return result
+ try:
+ image = base64.b64decode(
+ result.output[len(prefix) :], validate=True
+ )
+ except (binascii.Error, ValueError):
+ # Other PNG formatters may use representations we cannot decode.
+ return result
+ return ToolReturn(
+ return_value=replace(result, output=None),
+ content=[BinaryContent(data=image, media_type="image/png")],
+ )
+
toolset.add_function(
execute_code,
name="execute_code",
diff --git a/marimo/_server/scratchpad.py b/marimo/_server/scratchpad.py
index 620d5d7814..7b312f91b3 100644
--- a/marimo/_server/scratchpad.py
+++ b/marimo/_server/scratchpad.py
@@ -366,6 +366,11 @@ def extract_result(
return CodeExecutionResult(
success=len(errors) == 0,
output=output_data,
+ output_mimetype=(
+ cell_notif.output.mimetype
+ if cell_notif.output is not None
+ else None
+ ),
stdout=stdout,
stderr=stderr,
errors=errors,
diff --git a/tests/_code_mode/test_screenshot.py b/tests/_code_mode/test_screenshot.py
index 32a9ce3c11..fa2c7a5288 100644
--- a/tests/_code_mode/test_screenshot.py
+++ b/tests/_code_mode/test_screenshot.py
@@ -36,6 +36,7 @@ from marimo._messaging.notebook.outputs import (
CellOutputs,
notebook_outputs_context,
)
+from marimo._output.formatting import try_format
from marimo._runtime.commands import HTTPRequest
from marimo._types.ids import CellId_t
@@ -305,10 +306,18 @@ async def test_screenshot_passes_credentials_and_reuses_browser(
):
session.return_value.capture.return_value = b"png"
ctx = AsyncCodeModeContext(k)
- assert await ctx.screenshot(doc_cell.id) == b"png"
- assert await ctx.screenshot(doc_cell.id, as_data_url=True) == (
- _to_data_url(b"png")
- )
+ image = await ctx.screenshot(doc_cell.id)
+ data_url = await ctx.screenshot(doc_cell.id, as_data_url=True)
+ assert isinstance(image, bytes)
+ assert image == b"png"
+ assert isinstance(data_url, str)
+ assert data_url == _to_data_url(b"png")
+ for value in (image, data_url):
+ formatted = try_format(value)
+ assert (formatted.mimetype, formatted.data) == (
+ "image/png",
+ _to_data_url(b"png"),
+ )
session.assert_called_once_with(
"http://localhost:1234/base",
screenshot_auth_token="secret",
diff --git a/tests/_code_mode/test_screenshot_integration.py b/tests/_code_mode/test_screenshot_integration.py
index 64b746d562..d13ed00cad 100644
--- a/tests/_code_mode/test_screenshot_integration.py
+++ b/tests/_code_mode/test_screenshot_integration.py
@@ -7,6 +7,8 @@ The Playwright CI job supplies built assets and matching Chromium binaries.
from __future__ import annotations
import asyncio
+import base64
+import hashlib
import json
import os
import signal
@@ -26,7 +28,11 @@ if TYPE_CHECKING:
async def _execute(
- client: httpx.AsyncClient, session_id: str, code: str
+ client: httpx.AsyncClient,
+ session_id: str,
+ code: str,
+ *,
+ verify_png: bool = False,
) -> str:
response = await client.post(
"/api/kernel/execute",
@@ -50,6 +56,17 @@ async def _execute(
stderr += data["data"]
elif event == "done":
assert data["success"], stderr
+ if verify_png:
+ assert data["output"]["mimetype"] == "image/png"
+ image_url = data["output"]["data"]
+ assert image_url.startswith("data:image/png;base64,")
+ png = base64.b64decode(
+ image_url.split(",", 1)[1], validate=True
+ )
+ assert png.startswith(b"\x89PNG\r\n\x1a\n")
+ assert (
+ "PNG_SHA=" + hashlib.sha256(png).hexdigest() in stdout
+ )
completed = True
assert completed, response.text
return stdout.strip()
@@ -215,9 +232,14 @@ retry_image = await session.capture(cell_id)
assert retry_image.startswith(b"\\x89PNG\\r\\n\\x1a\\n")
assert {marker!r} in await session._page.locator("#output-" + cell_id).inner_text()
await session.close()
+import hashlib
+print("PNG_SHA=" + hashlib.sha256(image).hexdigest())
print("CELL_ID=" + cell_id)
+image
"""
- stdout = await _execute(client, session_id, code)
+ stdout = await _execute(
+ client, session_id, code, verify_png=True
+ )
cell_id = next(
line.removeprefix("CELL_ID=")
for line in stdout.splitlines()
diff --git a/tests/_server/ai/tools/test_code_mode.py b/tests/_server/ai/tools/test_code_mode.py
index 82b68c6adf..fe9108a169 100644
--- a/tests/_server/ai/tools/test_code_mode.py
+++ b/tests/_server/ai/tools/test_code_mode.py
@@ -86,3 +86,47 @@ def test_references_capability_exposes_deferred_reference_bundles() -> None:
"Improving, optimizing" in by_id["notebook-improvements"].description
)
assert "Custom widgets" in by_id["rich-representations"].description
+
+
[email protected]("pydantic_ai")
+ "output",
+ [
+ "cG5n",
+ "{'image/png': 'cG5n'}",
+ "data:image/png;base64,cG5n\n",
+ "data:image/png;base64,invalid!",
+ "data:image/png;base64,cG5",
+ "data:image/png;base64,é",
+ ],
+)
+async def test_execute_code_preserves_unsupported_png_output(
+ output: str,
+) -> None:
+ from marimo._ai._tools.types import CodeExecutionResult
+ from marimo._server.ai.tools.code_mode import build_execute_code_toolset
+
+ original = CodeExecutionResult(
+ success=True,
+ output=output,
+ output_mimetype="image/png",
+ stdout=["capture finished"],
+ stderr=["warning"],
+ )
+ with (
+ patch(
+ "marimo._server.ai.tools.code_mode.get_code_mode_credentials",
+ return_value=("http://localhost:2718", None),
+ ),
+ patch("marimo._server.ai.tools.code_mode.AppState"),
+ patch(
+ "marimo._server.ai.tools.code_mode.run_scratchpad_code",
+ new=AsyncMock(return_value=original),
+ ),
+ ):
+ toolset = build_execute_code_toolset(MagicMock(), MagicMock())
+ execute_code = cast(
+ Callable[[str], Awaitable[object]],
+ toolset.tools["execute_code"].function,
+ )
+ assert await execute_code("capture()") is original
diff --git a/tests/_server/ai/tools/test_screenshot_transport.py b/tests/_server/ai/tools/test_screenshot_transport.py
new file mode 100644
index 0000000000..2942fac171
--- /dev/null
+++ b/tests/_server/ai/tools/test_screenshot_transport.py
@@ -0,0 +1,106 @@
+# Copyright 2026 Marimo. All rights reserved.
+from __future__ import annotations
+
+import base64
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from marimo._code_mode.screenshot import (
+ _ScreenshotBytes,
+ _ScreenshotDataUrl,
+ _to_data_url,
+)
+from marimo._messaging.cell_output import CellChannel, CellOutput
+from marimo._messaging.notification import CellNotification
+from marimo._output.formatting import try_format
+from marimo._runtime.scratch import SCRATCH_CELL_ID
+from marimo._server.scratchpad import extract_result
+
+
[email protected]("pydantic_ai")
[email protected]("as_data_url", [False, True])
+async def test_screenshot_reaches_model_as_image(as_data_url: bool) -> None:
+ from pydantic_ai import Agent, BinaryContent
+ from pydantic_ai.messages import (
+ ModelMessage,
+ ModelRequest,
+ ModelResponse,
+ TextPart,
+ ToolCallPart,
+ ToolReturnPart,
+ UserPromptPart,
+ )
+ from pydantic_ai.models.function import AgentInfo, FunctionModel
+
+ from marimo._server.ai.tools.code_mode import build_execute_code_toolset
+
+ png = b"\x89PNG\r\n\x1a\ntransport-test"
+ value = (
+ _ScreenshotDataUrl(_to_data_url(png))
+ if as_data_url
+ else _ScreenshotBytes(png)
+ )
+ formatted = try_format(value)
+ assert formatted.mimetype == "image/png"
+ assert formatted.data == _to_data_url(png)
+ assert base64.b64decode(formatted.data.split(",", 1)[1]) == png
+ session = MagicMock()
+ session.session_view.cell_notifications = {
+ SCRATCH_CELL_ID: CellNotification(
+ cell_id=SCRATCH_CELL_ID,
+ output=CellOutput(
+ channel=CellChannel.OUTPUT,
+ mimetype=formatted.mimetype,
+ data=formatted.data,
+ ),
+ console=None,
+ status="idle",
+ )
+ }
+ result = extract_result(session)
+ calls = 0
+
+ def model(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ return ModelResponse(
+ parts=[ToolCallPart("execute_code", {"code": "capture"})]
+ )
+ images = []
+ for message in messages:
+ if not isinstance(message, ModelRequest):
+ continue
+ for part in message.parts:
+ if isinstance(part, UserPromptPart) and isinstance(
+ part.content, list
+ ):
+ images.extend(
+ item
+ for item in part.content
+ if isinstance(item, BinaryContent)
+ )
+ if isinstance(part, ToolReturnPart):
+ assert "data:image" not in part.model_response_str()
+ assert images == [BinaryContent(data=png, media_type="image/png")]
+ return ModelResponse(parts=[TextPart("Image received")])
+
+ with (
+ patch(
+ "marimo._server.ai.tools.code_mode.get_code_mode_credentials",
+ return_value=("http://localhost", None),
+ ),
+ patch("marimo._server.ai.tools.code_mode.AppState"),
+ patch(
+ "marimo._server.ai.tools.code_mode.run_scratchpad_code",
+ new=AsyncMock(return_value=result),
+ ),
+ ):
+ agent = Agent(
+ FunctionModel(model),
+ toolsets=[build_execute_code_toolset(session, MagicMock())],
+ )
+ response = await agent.run("Capture the cell")
+ assert response.output == "Image received"
+ assert calls == 2