A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
ci(ai): catch chat protocol drift between frontend and backend (#11154)
Shahmir Varqhaf51aaf413926
diff --git a/.github/workflows/test_ai_sdk_canary.yaml b/.github/workflows/test_ai_sdk_canary.yamlnew file mode 100644index 0000000000..d6c7c9bd88--- /dev/null+++ b/.github/workflows/test_ai_sdk_canary.yaml@@ -0,0 +1,44 @@+name: AI SDK Upstream Canary++on:+ schedule:+ - cron: '37 1 * * 1'+ workflow_dispatch:++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++permissions:+ contents: read++env:+ PYDANTIC_AI_NO_BANNER: '1'++jobs:+ upstream:+ name: Chat protocol / newest upstream pydantic-ai+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ - uses: ./.github/actions/install+ - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7+ with:+ python-version: '3.13'+ enable-cache: true+ - name: Check newest upstream release+ id: protocol+ shell: bash+ run: |+ requirement=$(uv run --no-project --python 3.13 --with packaging python scripts/check_ai_sdk_compatibility.py --requirement upstream)+ # Bypass the repository's release-age policy and refresh the package index+ # so cached environments do not hide a newly published release.+ uv run --no-config --no-project --python 3.13 --refresh-package pydantic-ai-slim --refresh-package pydantic-graph --with-editable . --with "$requirement" python scripts/check_ai_sdk_compatibility.py+ - name: Report dependency rollout requirement+ if: failure() && steps.protocol.outcome == 'failure'+ shell: bash+ run: |+ message="When updating pydantic-ai to resolve this compatibility failure, Molab dependencies must also be upgraded."+ echo "::warning::$message"+ printf '%s\n' "$message" >> "$GITHUB_STEP_SUMMARY"diff --git a/.github/workflows/test_ai_sdk_compatibility.yaml b/.github/workflows/test_ai_sdk_compatibility.yamlnew file mode 100644index 0000000000..28392b467b--- /dev/null+++ b/.github/workflows/test_ai_sdk_compatibility.yaml@@ -0,0 +1,98 @@+name: AI SDK Compatibility++on:+ push:+ branches: [main]+ pull_request:++concurrency:+ group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++permissions:+ contents: read+ pull-requests: read++env:+ PYDANTIC_AI_NO_BANNER: '1'++jobs:+ changes:+ runs-on: ubuntu-latest+ outputs:+ chat: ${{ steps.filter.outputs.chat }}+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3+ id: filter+ with:+ filters: |+ chat:+ - 'frontend/package.json'+ - 'frontend/scripts/check-ai-sdk-compatibility.ts'+ - 'frontend/src/components/chat/**'+ - 'frontend/src/components/editor/ai/**'+ - 'frontend/src/core/ai/**'+ - 'frontend/src/plugins/impl/chat/**'+ - 'marimo/_ai/**'+ - 'marimo/_plugins/ui/_impl/chat/**'+ - 'marimo/_server/ai/**'+ - 'marimo/_server/api/endpoints/ai.py'+ - 'marimo/_server/models/completion.py'+ - 'package.json'+ - 'pnpm-lock.yaml'+ - 'pnpm-workspace.yaml'+ - 'pyproject.toml'+ - 'scripts/check_ai_sdk_compatibility.py'+ - '.github/actions/install/**'+ - '.github/workflows/test_ai_sdk_compatibility.yaml'+ - '.github/workflows/test_ai_sdk_canary.yaml'++ compatibility:+ needs: changes+ if: needs.changes.outputs.chat == 'true'+ name: Chat protocol / ${{ matrix.resolution }} pydantic-ai+ runs-on: ubuntu-latest+ timeout-minutes: 10+ strategy:+ fail-fast: false+ matrix:+ resolution: [minimum, latest]+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ - uses: ./.github/actions/install+ - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7+ with:+ python-version: '3.13'+ enable-cache: true+ - name: Check stream and second-turn history+ id: protocol+ shell: bash+ run: |+ requirement=$(uv run --no-project --python 3.13 --with packaging python scripts/check_ai_sdk_compatibility.py --requirement "${{ matrix.resolution }}")+ uv run --no-project --python 3.13 --with-editable . --with "$requirement" python scripts/check_ai_sdk_compatibility.py+ - name: Report dependency rollout requirement+ if: failure() && steps.protocol.outcome == 'failure'+ shell: bash+ run: |+ message="When updating pydantic-ai to resolve this compatibility failure, Molab dependencies must also be upgraded."+ echo "::warning::$message"+ printf '%s\n' "$message" >> "$GITHUB_STEP_SUMMARY"++ compatibility_check:+ name: AI SDK compatibility+ needs: [changes, compatibility]+ if: always()+ runs-on: ubuntu-latest+ steps:+ - name: Require compatibility checks for relevant changes+ env:+ CHECK_PASSED: ${{ needs.changes.result == 'success' && (needs.changes.outputs.chat != 'true' || needs.compatibility.result == 'success') }}+ run: test "$CHECK_PASSED" = "true"+ - name: Report dependency rollout requirement+ if: failure() && needs.compatibility.result == 'failure'+ shell: bash+ run: |+ message="When updating pydantic-ai to resolve this compatibility failure, Molab dependencies must also be upgraded."+ echo "::warning::$message"+ printf '%s\n' "$message" >> "$GITHUB_STEP_SUMMARY"diff --git a/frontend/scripts/check-ai-sdk-compatibility.ts b/frontend/scripts/check-ai-sdk-compatibility.tsnew file mode 100644index 0000000000..6a090c95da--- /dev/null+++ b/frontend/scripts/check-ai-sdk-compatibility.ts@@ -0,0 +1,194 @@+/* Copyright 2026 Marimo. All rights reserved. */++import assert from "node:assert/strict";+import { readFile, writeFile } from "node:fs/promises";+import { Chat } from "@ai-sdk/react";+import {+ DefaultChatTransport,+ isStaticToolUIPart,+ safeValidateUIMessages,+ type UIMessage,+} from "ai";+import { z } from "zod";++// New SDK part types must get coverage or an explicit unsupported designation.+type FixedPartType = Exclude<+ UIMessage["parts"][number]["type"],+ `tool-${string}` | `data-${string}`+>;+const partCoverage = {+ text: "history",+ reasoning: "history",+ file: "history",+ "source-url": "history",+ "source-document": "history",+ "step-start": "history",+ "dynamic-tool": "history",+ "reasoning-file": "unsupported",+ custom: "unsupported",+} satisfies Record<FixedPartType, "history" | "unsupported">;++const [streamPath, historyPath, scenario] = process.argv.slice(2);+assert(+ streamPath && historyPath && scenario,+ "Expected stream, history, and scenario arguments",+);++for (const name of ["ai", "@ai-sdk/react"]) {+ const packageInfo: unknown = JSON.parse(+ await readFile(+ new URL(`../node_modules/${name}/package.json`, import.meta.url),+ "utf8",+ ),+ );+ assert(+ typeof packageInfo === "object" &&+ packageInfo !== null &&+ "version" in packageInfo &&+ typeof packageInfo.version === "string",+ );+ process.stdout.write(`${name}=${packageInfo.version}\n`);+}+async function checkStream(streamPath: string, historyPath: string) {+ const stream = await readFile(streamPath, "utf8");+ const requests: string[] = [];+ const chat = new Chat({+ transport: new DefaultChatTransport({+ api: "https://chat.test/",+ fetch: async (_url, options) => {+ assert.equal(typeof options?.body, "string");+ requests.push(String(options?.body));+ return new Response(stream, {+ headers: {+ "content-type": "text/event-stream",+ "x-vercel-ai-ui-message-stream": "v1",+ },+ });+ },+ }),+ });++ await chat.sendMessage({ text: "Double two" });+ assert.equal(chat.status, "ready", chat.error?.message);+ const assistant = chat.messages.find(+ (message) => message.role === "assistant",+ );+ assert(assistant, "The SDK did not create an assistant message");+ assert(+ assistant.parts.some(+ (part) => part.type === "reasoning" && part.text.length > 0,+ ),+ );++ if (scenario === "approval" || scenario === "denied") {+ const tool = assistant.parts+ .filter(isStaticToolUIPart)+ .find((part) => part.type === "tool-double");+ assert(tool && tool.state === "approval-requested" && tool.approval);+ await chat.addToolApprovalResponse({+ id: tool.approval.id,+ approved: scenario === "approval",+ reason: scenario === "denied" ? "User declined" : undefined,+ });+ const responded = chat.messages+ .flatMap((message) => message.parts)+ .filter(isStaticToolUIPart)+ .find((part) => part.type === "tool-double");+ assert(responded && responded.state === "approval-responded");+ assert.equal(responded.approval.approved, scenario === "approval");+ } else {+ assert(+ assistant.parts.some(+ (part) => part.type === "text" && part.text.length > 0,+ ),+ );+ assert(+ assistant.parts.some(+ (part) =>+ part.type === "tool-double" &&+ (scenario === "error"+ ? part.state === "output-error" &&+ part.errorText.includes("Cannot double")+ : part.state === "output-available" && part.output === 4),+ ),+ );+ }++ const validation = await safeValidateUIMessages({ messages: chat.messages });+ assert(+ validation.success,+ validation.success ? undefined : validation.error.message,+ );+ await chat.sendMessage({ text: "Continue" });+ assert.equal(chat.status, "ready", chat.error?.message);+ assert.equal(requests.length, 2, "Expected exactly two chat requests");+ await writeFile(historyPath, requests[1] ?? "");+}++async function checkHistory(streamPath: string, historyPath: string) {+ const messagesSchema = z.object({ messages: z.array(z.unknown()) });+ const fixtureSchema = messagesSchema.extend({ name: z.string() });+ const cases = z+ .array(fixtureSchema)+ .parse(JSON.parse(await readFile(streamPath, "utf8")));+ const coveredTypes = new Set(+ cases.flatMap((fixture) =>+ fixture.messages.flatMap((message) => {+ const parsed = z+ .object({ parts: z.array(z.object({ type: z.string() })) })+ .parse(message);+ return parsed.parts.map((part) => part.type);+ }),+ ),+ );+ for (const [type, coverage] of Object.entries(partCoverage)) {+ if (coverage === "history") {+ assert(coveredTypes.has(type), `Missing history coverage for ${type}`);+ }+ }+ const requests: z.infer<typeof fixtureSchema>[] = [];+ for (const fixture of cases) {+ const validation = await safeValidateUIMessages({+ messages: fixture.messages,+ });+ assert(+ validation.success,+ `${fixture.name}: ${validation.success ? "" : validation.error.message}`,+ );+ let submitted = false;+ const chat = new Chat({+ messages: validation.data,+ transport: new DefaultChatTransport({+ api: "https://chat.test/",+ fetch: async (_url, options) => {+ assert.equal(typeof options?.body, "string");+ const request = messagesSchema.parse(+ JSON.parse(String(options?.body)),+ );+ requests.push({ name: fixture.name, messages: request.messages });+ submitted = true;+ return new Response(+ 'data: {"type":"start","messageId":"reply"}\n\ndata: {"type":"finish"}\n\ndata: [DONE]\n\n',+ {+ headers: {+ "content-type": "text/event-stream",+ "x-vercel-ai-ui-message-stream": "v1",+ },+ },+ );+ },+ }),+ });+ await chat.sendMessage({ text: "Continue" });+ assert.equal(+ chat.status,+ "ready",+ `${fixture.name}: ${chat.error?.message}`,+ );+ assert(submitted, `${fixture.name}: no request sent`);+ }+ await writeFile(historyPath, JSON.stringify(requests));+}++const check = scenario === "history" ? checkHistory : checkStream;+await check(streamPath, historyPath);diff --git a/frontend/tsconfig.json b/frontend/tsconfig.jsonindex 5c9da8a177..3b821e3ad4 100644--- a/frontend/tsconfig.json+++ b/frontend/tsconfig.json@@ -36,5 +36,5 @@"incremental": true,"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo"},- "include": ["src", "playwright.config.ts", "e2e-tests"]+ "include": ["src", "scripts", "playwright.config.ts", "e2e-tests"]}diff --git a/scripts/check_ai_sdk_compatibility.py b/scripts/check_ai_sdk_compatibility.pynew file mode 100644index 0000000000..b8d921b183--- /dev/null+++ b/scripts/check_ai_sdk_compatibility.py@@ -0,0 +1,389 @@+# Copyright 2026 Marimo. All rights reserved.+"""Exercise the pydantic-ai → frontend SDK → marimo chat history boundary.++Run with Python 3.12+ and installed frontend dependencies. No model API or+browser is used. `--requirement minimum|latest|upstream` prints the dependency+constraint for CI. Minimum and latest follow the recommended extra; upstream+keeps its provider extras but tests the newest release without version bounds.+"""++from __future__ import annotations++import argparse+import asyncio+import json+import subprocess+import sys+import tempfile+from importlib.metadata import version+from pathlib import Path+from typing import TYPE_CHECKING, Literal++import tomllib++if TYPE_CHECKING:+ from collections.abc import AsyncIterator+ from typing import Any++ROOT = Path(__file__).resolve().parent.parent+++def dependency_requirement(+ resolution: Literal["minimum", "latest", "upstream"],+) -> str:+ from packaging.requirements import Requirement++ project = tomllib.loads((ROOT / "pyproject.toml").read_text())+ requirements = project["project"]["optional-dependencies"]["recommended"]+ requirement = next(+ Requirement(value)+ for value in requirements+ if Requirement(value).name == "pydantic-ai-slim"+ )+ extras = ",".join(sorted(requirement.extras))+ if resolution == "upstream":+ return f"{requirement.name}[{extras}]"+ if resolution == "latest":+ return str(requirement)+ minimum = next(+ spec.version for spec in requirement.specifier if spec.operator == ">="+ )+ return f"{requirement.name}[{extras}]=={minimum}"+++Scenario = Literal["completed", "approval", "denied", "error"]+++async def check_round_trip(scenario: Scenario) -> None:+ from pydantic import ValidationError+ from pydantic_ai import Agent, DeferredToolRequests, ToolFailed+ from pydantic_ai.messages import ModelMessage, ToolReturnPart+ from pydantic_ai.models.function import (+ AgentInfo,+ DeltaThinkingCalls,+ DeltaThinkingPart,+ DeltaToolCall,+ DeltaToolCalls,+ FunctionModel,+ )+ from pydantic_ai.ui.vercel_ai import VercelAIAdapter+ from pydantic_ai.ui.vercel_ai.request_types import SubmitMessage++ from marimo._ai._pydantic_ai_utils import convert_to_pydantic_messages+ from marimo._plugins.ui._impl.chat.chat import AI_SDK_VERSION++ async def respond(+ messages: list[ModelMessage], _info: AgentInfo+ ) -> AsyncIterator[str | DeltaThinkingCalls | DeltaToolCalls]:+ if any(+ isinstance(part, ToolReturnPart) for part in messages[-1].parts+ ):+ yield "The answer is four."+ else:+ yield {0: DeltaThinkingPart(content="I will double two.")}+ yield {+ 1: DeltaToolCall(+ name="double",+ json_args='{"value": 2}',+ tool_call_id="double-call",+ )+ }++ agent = Agent(+ FunctionModel(stream_function=respond),+ output_type=[str, DeferredToolRequests],+ )++ @agent.tool_plain(requires_approval=scenario in ("approval", "denied"))+ def double(value: int) -> int:+ if scenario == "error":+ raise ToolFailed("Cannot double this value")+ return value * 2++ adapter = VercelAIAdapter(+ agent=agent,+ run_input=SubmitMessage(+ id="compatibility-chat",+ trigger="submit-message",+ messages=convert_to_pydantic_messages(+ [+ {+ "id": "user-1",+ "role": "user",+ "parts": [{"type": "text", "text": "Double two"}],+ }+ ]+ ),+ ),+ sdk_version=AI_SDK_VERSION,+ )+ stream = "".join(+ [chunk async for chunk in adapter.encode_stream(adapter.run_stream())]+ )+ request = json.loads(await run_frontend_check(stream, scenario))++ try:+ messages = convert_to_pydantic_messages(request["messages"])+ model_messages = VercelAIAdapter.load_messages(messages)+ except ValidationError as exc:+ raise RuntimeError(+ f"Chat protocol incompatibility ({scenario}): "+ f"pydantic-ai-slim={version('pydantic-ai-slim')} rejected "+ f"the frontend SDK's second-turn history:\n{exc}"+ ) from exc+ assert model_messages, "The adapter discarded the chat history"+ assert any(message.role == "assistant" for message in messages)+ sys.stdout.write(+ f"{scenario}: stream and second-turn history are compatible\n"+ )+++async def run_frontend_check(payload: str, scenario: str) -> str:+ with tempfile.TemporaryDirectory(+ prefix="marimo-chat-compatibility-"+ ) as tmp:+ input_path = Path(tmp) / "input.txt"+ output_path = Path(tmp) / "history.json"+ input_path.write_text(payload)+ await asyncio.to_thread(+ subprocess.run,+ [+ "node",+ "--experimental-strip-types",+ str(ROOT / "frontend/scripts/check-ai-sdk-compatibility.ts"),+ str(input_path),+ str(output_path),+ scenario,+ ],+ check=True,+ timeout=30,+ )+ return output_path.read_text()+++async def check_history_shapes() -> None:+ from pydantic_ai.messages import ToolCallPart, ToolReturnPart+ from pydantic_ai.ui.vercel_ai import VercelAIAdapter++ from marimo._ai._pydantic_ai_utils import convert_to_pydantic_messages++ # Keep wire fixtures independent of backend models so schema gaps stay visible.+ metadata = {"test": {"key": "value"}}+ cases = []+ states: tuple[tuple[str, dict[str, Any]], ...] = (+ ("input-streaming", {}),+ ("input-available", {}),+ ("output-available", {"output": 4}),+ ("output-error", {"errorText": "Tool failed"}),+ ("approval-requested", {"approval": {"id": "approve"}}),+ (+ "approval-responded",+ {"approval": {"id": "approve", "approved": True}},+ ),+ (+ "approval-responded",+ {+ "approval": {+ "id": "approve",+ "approved": False,+ "reason": "Declined",+ }+ },+ ),+ (+ "output-denied",+ {+ "approval": {+ "id": "approve",+ "approved": False,+ "reason": "Declined",+ }+ },+ ),+ )+ for dynamic in (False, True):+ for index, (state, fields) in enumerate(states):+ name = f"{'dynamic' if dynamic else 'static'}-{state}-{index}"+ part = {+ "type": "dynamic-tool" if dynamic else "tool-double",+ "toolCallId": name,+ "state": state,+ "input": {"value": 2},+ "title": "Double",+ "providerExecuted": False,+ "callProviderMetadata": metadata,+ **fields,+ }+ if dynamic:+ part["toolName"] = "double"+ cases.append(+ {+ "name": name,+ "messages": [+ {+ "id": "user",+ "role": "user",+ "parts": [{"type": "text", "text": "Double two"}],+ },+ {+ "id": "assistant",+ "role": "assistant",+ "parts": [part],+ },+ ],+ }+ )+ cases.append(+ {+ "name": "content-and-context",+ "messages": [+ {+ "id": "user",+ "role": "user",+ "metadata": {"request": "fixture"},+ "parts": [+ {+ "type": "text",+ "text": "Inspect this file",+ "state": "done",+ "providerMetadata": metadata,+ },+ {+ "type": "file",+ "mediaType": "image/png",+ "filename": "pixel.png",+ "url": "data:image/png;base64,iVBORw0KGgo=",+ "providerMetadata": metadata,+ },+ {+ "type": "data-marimo-context",+ "data": {"plainText": "Cell context"},+ },+ ],+ },+ {+ "id": "assistant",+ "role": "assistant",+ "parts": [+ {"type": "step-start"},+ {+ "type": "file",+ "mediaType": "text/plain",+ "filename": "answer.txt",+ "url": "data:text/plain;base64,RG9uZQ==",+ "providerMetadata": metadata,+ },+ {+ "type": "reasoning",+ "id": "reasoning",+ "text": "Inspecting",+ "state": "done",+ "providerMetadata": metadata,+ },+ {+ "type": "source-url",+ "sourceId": "url",+ "url": "https://example.com/",+ "title": "Reference",+ "providerMetadata": metadata,+ },+ {+ "type": "source-document",+ "sourceId": "doc",+ "mediaType": "text/plain",+ "title": "Document",+ "filename": "doc.txt",+ "providerMetadata": metadata,+ },+ {+ "type": "text",+ "text": "Done",+ "state": "done",+ "providerMetadata": metadata,+ },+ ],+ },+ ],+ }+ )+ histories = json.loads(+ await run_frontend_check(json.dumps(cases), "history")+ )+ assert len(histories) == len(cases), "The SDK omitted a history fixture"+ for history in histories:+ name = history["name"]+ try:+ messages = convert_to_pydantic_messages(history["messages"])+ loaded = VercelAIAdapter.load_messages(messages)+ assert loaded, f"{name}: the adapter discarded history"+ if name != "content-and-context":+ calls = [+ part+ for message in loaded+ for part in message.parts+ if isinstance(part, ToolCallPart)+ ]+ assert len(calls) == 1, f"{name}: tool call was lost"+ assert calls[0].tool_name == "double"+ assert calls[0].args_as_dict() == {"value": 2}+ assert calls[0].tool_call_id == name+ if "approval-" not in name:+ results = [+ part+ for message in loaded+ for part in message.parts+ if isinstance(part, ToolReturnPart)+ ]+ assert len(results) == 1, f"{name}: tool result was lost"+ assert results[0].tool_call_id == name+ if "output-available" in name:+ assert results[0].content == 4+ elif "output-error" in name:+ assert results[0].content == "Tool failed"+ elif "output-denied" in name:+ assert results[0].outcome == "denied"+ else:+ assert "interrupted" in str(results[0].content).lower()+ else:+ assert messages[0].metadata == {"request": "fixture"}+ assert any(+ "Cell context" in getattr(part, "text", "")+ for part in messages[0].parts+ )+ except Exception as exc:+ raise RuntimeError(+ f"History fixture {name} failed with pydantic-ai-slim={version('pydantic-ai-slim')}"+ ) from exc+ sys.stdout.write(f"{name}: frontend history is compatible\n")+++async def check_compatibility() -> None:+ sys.stdout.write(+ f"Checking pydantic-ai-slim={version('pydantic-ai-slim')}\n"+ )+ sys.stdout.flush()+ scenarios: tuple[Scenario, ...] = (+ "completed",+ "approval",+ "denied",+ "error",+ )+ for scenario in scenarios:+ await check_round_trip(scenario)+ await check_history_shapes()+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument(+ "--requirement", choices=("minimum", "latest", "upstream")+ )+ args = parser.parse_args()+ if args.requirement:+ sys.stdout.write(f"{dependency_requirement(args.requirement)}\n")+ else:+ asyncio.run(check_compatibility())+++if __name__ == "__main__":+ main()