A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
support custom models returning vercel chunks (#7797)
Shahmir Varqha64d35f12d261
diff --git a/examples/ai/chat/pydantic-ai-chat.py b/examples/ai/chat/pydantic-ai-chat.pyindex e5d6f1b520..5c41f6b5e8 100644--- a/examples/ai/chat/pydantic-ai-chat.py+++ b/examples/ai/chat/pydantic-ai-chat.py@@ -78,7 +78,7 @@ def _(models):return input_key, model_name, os_key[email protected](hide_code=True)def get_model(model_name: str, thinking: bool, api_key: str) -> tuple[Model, ModelSettings]:@@ -168,18 +168,17 @@ def _(@app.cell-def _(agent, model_name):+def _(agent):chatbot = mo.ui.chat(mo.ai.llm.pydantic_ai(agent),prompts=["Write the fibonacci function in Python","Who is Ada Lovelace?","What is marimo?",- "I need dogs",+ "I need dogs (render as markdown)",],allow_attachments=True,show_configuration_controls=True,- config={"top_p": None} if model_name.startswith("claude") else None,)chatbotreturn (chatbot,)@@ -191,5 +190,90 @@ def _(chatbot):return+def _():+ mo.md("""+ ## Custom Model Sample+ """)+ return+++def _():+ import uuid+ import pydantic_ai.ui.vercel_ai.response_types as vercel+++ async def custom_model(messages, config):+ # Generate unique IDs for message parts+ reasoning_id = f"reasoning_{uuid.uuid4().hex}"+ text_id = f"text_{uuid.uuid4().hex}"+ tool_id = f"tool_{uuid.uuid4().hex}"++ # --- Stream reasoning/thinking ---+ yield vercel.StartStepChunk()+ yield vercel.ReasoningStartChunk(id=reasoning_id)+ yield vercel.ReasoningDeltaChunk(+ id=reasoning_id,+ delta="The user is asking about Van Gogh. I should fetch information about his famous works.",+ )+ yield vercel.ReasoningEndChunk(id=reasoning_id)++ # --- Stream tool call to fetch artwork information ---+ yield vercel.ToolInputAvailableChunk(+ tool_call_id=tool_id,+ tool_name="search_artwork",+ input={"artist": "Vincent van Gogh", "limit": 1},+ )+ yield vercel.ToolInputStartChunk(+ tool_call_id=tool_id, tool_name="search_artwork"+ )+ yield vercel.ToolInputDeltaChunk(+ tool_call_id=tool_id,+ input_text_delta='{"artist": "Vincent van Gogh", "limit": 1}',+ )++ # --- Tool output (simulated artwork search result) ---+ yield vercel.ToolOutputAvailableChunk(+ tool_call_id=tool_id,+ output={+ "title": "The Starry Night",+ "year": 1889,+ "museum": "Museum of Modern Art",+ },+ )++ # --- Stream text response ---+ yield vercel.TextStartChunk(id=text_id)+ yield vercel.TextDeltaChunk(+ id=text_id,+ delta="One of Vincent van Gogh's most iconic works is 'The Starry Night', painted in 1889. Here's the painting:\n\n",+ )++ # --- Embed the artwork image ---+ yield vercel.FileChunk(+ url="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",+ media_type="image/jpeg",+ )+ yield vercel.TextDeltaChunk(+ id=text_id,+ delta="\nThis masterpiece is now housed at the Museum of Modern Art in New York and remains one of the most recognizable paintings in the world.",+ )+ yield vercel.TextEndChunk(id=text_id)+ yield vercel.FinishStepChunk()+ yield vercel.FinishChunk()+++ custom_chat = mo.ui.chat(custom_model)+ custom_chat+ return (custom_chat,)+++def _(custom_chat):+ custom_chat.value+ return++if __name__ == "__main__":app.run()diff --git a/frontend/src/plugins/impl/chat/ChatPlugin.tsx b/frontend/src/plugins/impl/chat/ChatPlugin.tsxindex 7454bb28c6..2972d80398 100644--- a/frontend/src/plugins/impl/chat/ChatPlugin.tsx+++ b/frontend/src/plugins/impl/chat/ChatPlugin.tsx@@ -15,7 +15,7 @@ export type PluginFunctions = {get_chat_history: (req: {}) => Promise<{ messages: UIMessage[] }>;delete_chat_history: (req: {}) => Promise<null>;delete_chat_message: (req: { index: number }) => Promise<null>;- send_prompt: (req: SendMessageRequest) => Promise<string | null>;+ send_prompt: (req: SendMessageRequest) => Promise<unknown>;};const messageSchema = z.array(@@ -47,7 +47,6 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(maxHeight: z.number().optional(),config: configSchema,allowAttachments: z.union([z.boolean(), z.string().array()]),- frontendManaged: z.boolean(),}),).withFunctions<PluginFunctions>({@@ -67,7 +66,7 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(config: configSchema,}),)- .output(z.union([z.string(), z.null()])),+ .output(z.unknown()),}).renderer((props) => (<TooltipProvider>@@ -77,7 +76,6 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(showConfigurationControls={props.data.showConfigurationControls}maxHeight={props.data.maxHeight}allowAttachments={props.data.allowAttachments}- frontendManaged={props.data.frontendManaged}config={props.data.config}get_chat_history={props.functions.get_chat_history}delete_chat_history={props.functions.delete_chat_history}diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsxindex e68cf7d8a4..4032fb154e 100644--- a/frontend/src/plugins/impl/chat/chat-ui.tsx+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx@@ -66,7 +66,6 @@ interface Props extends PluginFunctions {showConfigurationControls: boolean;maxHeight: number | undefined;allowAttachments: boolean | string[];- frontendManaged: boolean;value: UIMessage[];setValue: (messages: UIMessage[]) => void;host: HTMLElement;@@ -166,113 +165,42 @@ export const Chatbot: React.FC<Props> = (props) => {};});- if (props.frontendManaged) {- const stream = new ReadableStream<UIMessageChunk>({- start(controller) {- frontendStreamControllerRef.current = controller;+ const stream = new ReadableStream<UIMessageChunk>({+ start(controller) {+ frontendStreamControllerRef.current = controller;- const abortHandler = () => {- try {- controller.close();- } catch (error) {- Logger.debug("Controller may already be closed", { error });- }- frontendStreamControllerRef.current = null;- };- signal?.addEventListener("abort", abortHandler);-- return () => {- signal?.removeEventListener("abort", abortHandler);- };- },- cancel() {- frontendStreamControllerRef.current = null;- },- });-- // Start the prompt, chunks will be sent via events- props- .send_prompt({- messages: messages,- config: chatConfig,- })- .catch((error: Error) => {- frontendStreamControllerRef.current?.error(error);- frontendStreamControllerRef.current = null;- });-- return createUIMessageStreamResponse({ stream });- }-- if (signal?.aborted) {- return new Response("Aborted", { status: 499 });- }-- // Create a placeholder message for streaming (backend-managed)- const messageId = Date.now().toString();-- setMessages((prev) => [- ...prev,- {- id: messageId,- role: "assistant",- parts: [{ type: "text", text: "" }],- },- ]);-- // Create an abort-aware promise for the send_prompt call- const sendPromptPromise = props.send_prompt({- messages: messages,- config: chatConfig,- });-- // Race the send_prompt with an abort signal- const response = await new Promise<string | null>(- (resolve, reject) => {- // Listen for abortconst abortHandler = () => {- reject(new DOMException("Aborted", "AbortError"));+ try {+ controller.close();+ } catch (error) {+ Logger.debug("Controller may already be closed", { error });+ }+ frontendStreamControllerRef.current = null;};signal?.addEventListener("abort", abortHandler);- sendPromptPromise- .then(resolve)- .catch(reject)- .finally(() => {- signal?.removeEventListener("abort", abortHandler);- });+ return () => {+ signal?.removeEventListener("abort", abortHandler);+ };},- );+ cancel() {+ frontendStreamControllerRef.current = null;+ },+ });- if (response === null) {- Logger.error("Non-frontend-managed response is null", {- response,+ // Start the prompt, chunks will be sent via events+ void props+ .send_prompt({+ messages: messages,+ config: chatConfig,+ })+ .catch((error: Error) => {+ frontendStreamControllerRef.current?.error(error);+ frontendStreamControllerRef.current = null;});- return new Response("Internal server error", { status: 500 });- }- // If streaming didn't happen (non-generator response), update the message- // Check if streaming state is still set (meaning no chunks were received)- if (- streamingStateRef.current.backendMessageId === null &&- streamingStateRef.current.frontendMessageIndex === null- ) {- setMessages((prev) => {- const updated = [...prev];- const index = updated.findIndex((m) => m.id === messageId);- if (index !== -1) {- updated[index] = {- ...updated[index],- parts: [{ type: "text", text: response }],- };- }- return updated;- });- }-- return new Response(response);- // eslint-disable-next-line @typescript-eslint/no-explicit-any- } catch (error: any) {+ return createUIMessageStreamResponse({ stream });+ } catch (error: unknown) {// Clear streaming state on errorstreamingStateRef.current = {backendMessageId: null,@@ -280,13 +208,13 @@ export const Chatbot: React.FC<Props> = (props) => {};// Handle abort gracefully without showing an error- if (error.name === "AbortError") {+ if (error instanceof Error && error.name === "AbortError") {return new Response("Aborted", { status: 499 });}// HACK: strip the error message to clean up the response- const strippedError = error.message- .split("failed with exception ")+ const strippedError = (error as Error).message+ ?.split("failed with exception ").pop();return new Response(strippedError, { status: 400 });}@@ -307,11 +235,7 @@ export const Chatbot: React.FC<Props> = (props) => {frontendMessageIndex: null,};- // For frontend-managed streaming, we set the value directly from the frontend.- // Because useChat creates the proper message structure for us.- if (props.frontendManaged) {- props.setValue(message.messages);- }+ props.setValue(message.messages);},onError: (error) => {Logger.error("An error occurred:", error);@@ -338,90 +262,28 @@ export const Chatbot: React.FC<Props> = (props) => {return;}- if (props.frontendManaged) {- // Push to the stream for useChat to process- const controller = frontendStreamControllerRef.current;- if (!controller) {- return;- }-- const frontendMessage = message as {- type: string;- message_id: string;- content?: UIMessageChunk;- is_final?: boolean;- };-- if (frontendMessage.content) {- controller.enqueue(frontendMessage.content);- }- if (frontendMessage.is_final) {- controller.close();- frontendStreamControllerRef.current = null;- }+ // Push to the stream for useChat to process+ const controller = frontendStreamControllerRef.current;+ if (!controller) {return;}- // Handle regular text streaming chunks- const chunkMessage = message as {+ const frontendMessage = message as {type: string;message_id: string;- content: string;- is_final: boolean;+ content?: UIMessageChunk;+ is_final?: boolean;};- // Initialize streaming state on first chunk if not already set- if (streamingStateRef.current.backendMessageId === null) {- // Find the last assistant message (which should be the placeholder we created)- setMessages((prev) => {- const updated = [...prev];- // Find the last assistant message- for (let i = updated.length - 1; i >= 0; i--) {- if (updated[i].role === "assistant") {- streamingStateRef.current = {- backendMessageId: chunkMessage.message_id,- frontendMessageIndex: i,- };- break;- }- }- return updated;- });+ if (frontendMessage.content) {+ controller.enqueue(frontendMessage.content);}-- // Only process chunks for the current streaming message- const frontendIndex = streamingStateRef.current.frontendMessageIndex;- if (- streamingStateRef.current.backendMessageId ===- chunkMessage.message_id &&- frontendIndex !== null- ) {- setMessages((prev) => {- const updated = [...prev];- const index = frontendIndex;-- // Update the message at the tracked index- if (index < updated.length) {- const messageToUpdate = updated[index];- if (messageToUpdate.role === "assistant") {- updated[index] = {- ...messageToUpdate,- parts: [{ type: "text", text: chunkMessage.content }],- };- }- }-- return updated;- });-- // Clear streaming state when final chunk arrives- if (chunkMessage.is_final) {- streamingStateRef.current = {- backendMessageId: null,- frontendMessageIndex: null,- };- }+ if (frontendMessage.is_final) {+ controller.close();+ frontendStreamControllerRef.current = null;}++ return;},);@@ -434,10 +296,7 @@ export const Chatbot: React.FC<Props> = (props) => {props.delete_chat_message({ index });setMessages(newMessages);- // Since we manage the state in the frontend, we need to update the value.- if (props.frontendManaged) {- props.setValue(newMessages);- }+ props.setValue(newMessages);}};@@ -526,7 +385,7 @@ export const Chatbot: React.FC<Props> = (props) => {return (<div- key={message.id}+ key={`${message.id}-${index}`}className={cn("flex flex-col group gap-2",message.role === "user" ? "items-end" : "items-start",diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.pyindex 51a10abbad..abe6a0799c 100644--- a/marimo/_ai/_types.py+++ b/marimo/_ai/_types.py@@ -18,6 +18,7 @@ import msgspecfrom marimo import _loggersfrom marimo._dependencies.dependencies import DependencyManager+from marimo._plugins.utils import remove_none_valuesfrom marimo._utils.parse_dataclass import parse_rawLOGGER = _loggers.marimo_logger()@@ -270,12 +271,15 @@ class ChatMessage(msgspec.Struct):if isinstance(part, part_validator_class):validated_parts.append(part)elif isinstance(part, dict):+ sanitized_part = remove_none_values(part)# Try pydantic validation for dict -> class conversiontry:from pydantic import TypeAdapteradapter = TypeAdapter(part_validator_class)- validated_parts.append(adapter.validate_python(part))+ validated_parts.append(+ adapter.validate_python(sanitized_part)+ )except ImportError:LOGGER.debug("Pydantic not installed, skipping dict validation"diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.pyindex e1ed64dece..31f8042e7f 100644--- a/marimo/_ai/llm/_impl.py+++ b/marimo/_ai/llm/_impl.py@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, castfrom marimo import _loggersfrom marimo._ai._pydantic_ai_utils import generate_id+from marimo._plugins.utils import remove_none_valuesif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generator@@ -746,9 +747,11 @@ class pydantic_ai(ChatModel):parts = cast(list[UIMessagePart],[- dataclasses.asdict(part)- if dataclasses.is_dataclass(part)- else part+ self._remove_none_values(+ dataclasses.asdict(part)+ if dataclasses.is_dataclass(part)+ else part+ )for part in message.parts],)@@ -776,6 +779,11 @@ class pydantic_ai(ChatModel):)return ui_messages+ def _remove_none_values(self, obj: dict[str, Any]) -> dict[str, Any]:+ if isinstance(obj, dict) and hasattr(obj, "items"):+ return remove_none_values(obj)+ return obj+def _serialize_vercel_ai_chunk(self, chunk: BaseChunk) -> dict[str, Any] | None:diff --git a/marimo/_plugins/ui/_impl/chat/chat.py b/marimo/_plugins/ui/_impl/chat/chat.pyindex 0d484fa473..f3b6538966 100644--- a/marimo/_plugins/ui/_impl/chat/chat.py+++ b/marimo/_plugins/ui/_impl/chat/chat.py@@ -6,22 +6,24 @@ import uuidfrom dataclasses import dataclassfrom typing import Any, Callable, Final, Optional, Union, cast+from marimo import _loggersfrom marimo._ai._types import (ChatMessage,ChatModelConfig,ChatModelConfigDict,)-from marimo._ai.llm._impl import pydantic_ai+from marimo._dependencies.dependencies import DependencyManagerfrom marimo._output.formatting import as_htmlfrom marimo._output.rich_help import mddocfrom marimo._plugins.core.web_component import JSONTypefrom marimo._plugins.ui._core.ui_element import UIElement-from marimo._plugins.ui._impl.chat.utils import from_chat_message_dictfrom marimo._runtime.commands import UpdateUIElementCommandfrom marimo._runtime.context import get_contextfrom marimo._runtime.context.types import ContextNotInitializedErrorfrom marimo._runtime.functions import EmptyArgs, Function+LOGGER = _loggers.marimo_logger()+DEFAULT_CONFIG = ChatModelConfigDict(max_tokens=4096,temperature=0.5,@@ -31,6 +33,8 @@ DEFAULT_CONFIG = ChatModelConfigDict(presence_penalty=0,)+DONE_CHUNK: Final[str] = "[DONE]"+@dataclassclass SendMessageRequest:@@ -125,6 +129,33 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):)```+ Custom model with Vercel AI SDK streaming (reasoning, tool calls):+ ```python+ import pydantic_ai.ui.vercel_ai.response_types as vercel+++ async def custom_model(messages, config):+ # Stream reasoning/thinking+ yield vercel.ReasoningStartChunk(id="reasoning-1")+ yield vercel.ReasoningDeltaChunk(+ id="reasoning-1", delta="Let me think..."+ )+ yield vercel.ReasoningEndChunk(id="reasoning-1")++ # Stream text response (can also use plain dicts)+ yield {"type": "text-start", "id": "text-1"}+ yield vercel.TextDeltaChunk(+ id="text-1", delta="Here is my answer."+ )+ yield vercel.TextEndChunk(id="text-1")++ yield vercel.FinishChunk(finish_reason="stop")+++ chat = mo.ui.chat(custom_model)+ ```+ Refer to examples/ai/chat/pydantic-ai-chat.py for a complete example.+Attributes:value (List[ChatMessage]): The current chat history, a list of ChatMessage objects.@@ -166,10 +197,6 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):) -> None:self._model = modelself._chat_history: list[ChatMessage] = []- self._frontend_managed = False-- if isinstance(model, pydantic_ai):- self._frontend_managed = Trueif config is None:config = DEFAULT_CONFIG@@ -188,7 +215,6 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):"config": cast(JSONType, config or {}),"allow-attachments": allow_attachments,"max-height": max_height,- "frontend-managed": self._frontend_managed,},functions=(Function(@@ -246,7 +272,7 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):buffers=None,)- async def _handle_streaming_response(self, response: Any) -> str | None:+ async def _handle_streaming_response(self, response: Any) -> None:"""Handle streaming from both sync and async generators.Generators should yield delta chunks (new content only), which this@@ -256,47 +282,31 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):so we don't need to return anything."""message_id = str(uuid.uuid4())- accumulated_text = ""- if self._frontend_managed:- async for delta in response:- self._send_chat_message(- message_id=message_id, content=delta, is_final=False- )- # Send final message to indicate streaming is complete+ def send_chunk(chunk: dict[str, Any]) -> None:self._send_chat_message(- message_id=message_id, content=None, is_final=True+ message_id=message_id,+ content=chunk,+ is_final=False,)- return None- # Use async for if it's an async generator, otherwise regular for+ serializer = ChunkSerializer(on_send_chunk=send_chunk)+if inspect.isasyncgen(response):async for delta in response:- # Accumulate each delta chunk- delta_str = str(delta)- accumulated_text += delta_str- self._send_chat_message(- message_id=message_id,- content=accumulated_text,- is_final=False,- )+ serializer.handle_chunk(delta)else:for delta in response:- # Accumulate each delta chunk- delta_str = str(delta)- accumulated_text += delta_str- self._send_chat_message(- message_id=message_id,- content=accumulated_text,- is_final=False,- )+ serializer.handle_chunk(delta)++ serializer.on_end()# Send final message to indicate streaming is complete- if accumulated_text:- self._send_chat_message(- message_id=message_id, content=accumulated_text, is_final=True- )- return accumulated_text+ self._send_chat_message(+ message_id=message_id,+ content=None,+ is_final=True,+ )def _update_chat_history(self, chat_history: list[ChatMessage]) -> None:self._chat_history = chat_history@@ -326,6 +336,8 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):async def _send_prompt(self, args: SendMessageRequest) -> str | None:messages = args.messages+ self._chat_history = messages+# If the model is a callable that takes a single argument,# call it with just the messages.response: object@@ -338,28 +350,31 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):else:response = self._model(messages, args.config)+ if inspect.isasyncgen(response) or inspect.isgenerator(response):+ # We support functions that stream the response with generators+ # (both sync and async)+ await self._handle_streaming_response(response)+ # For streaming, we don't have a final response string to add to history+ # The frontend will add the accumulated message+ return None+if inspect.isawaitable(response):response = await response- elif inspect.isasyncgen(response) or inspect.isgenerator(response):- # We support functions that stream the response with generators- # (both sync and async); each yielded value is the latest- # representation of the response, and the last value is the full value- response = await self._handle_streaming_response(response)- if self._frontend_managed:- # For frontend-managed streaming, the response is set on the frontend,- # so we don't need to return anything.- return None+ # Return the response as a string+ # If the response is a rich object, convert it to markdown+ response_str = (+ response if isinstance(response, str) else as_html(response).text+ )- response_message = ChatMessage(role="assistant", content=response)- chat_history = messages + [response_message]- self._update_chat_history(chat_history)+ # Add assistant response to chat history+ assistant_message = ChatMessage(role="assistant", content=response_str)+ self._chat_history.append(assistant_message)- # Return the response as HTML- # If the response is a string, convert it to markdown- if isinstance(response, str):- return response- return as_html(response).text+ # Update the chat history to trigger UI updates and on_message callback+ self._update_chat_history(self._chat_history)++ return response_strdef _convert_value(self, value: dict[str, Any]) -> list[ChatMessage]:if not isinstance(value, dict) or "messages" not in value:@@ -367,20 +382,71 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):messages = value["messages"]- if self._frontend_managed:- from pydantic_ai.ui.vercel_ai.request_types import UIMessagePart-- # The frontend sends messages as UIMessage dicts so we use pydantic-ai to cast them- # as Vercel messages- return [- ChatMessage.create(- role=msg.get("role", "user"),- message_id=msg.get("id"),- content=None,- parts=msg.get("parts", []),- part_validator_class=UIMessagePart,+ part_validator_class = None+ if DependencyManager.pydantic_ai.imported():+ from pydantic_ai.ui.vercel_ai.request_types import (+ UIMessagePart,+ )++ # The frontend sends messages as ChatMessage parts so we use pydantic-ai to cast them+ # as Vercel UIMessagePart+ part_validator_class = UIMessagePart++ msg_to_content = {+ msg.get("id"): msg.get("content")+ for msg in messages+ if msg.get("content") is not None+ }++ return [+ ChatMessage.create(+ role=msg.get("role", "user"),+ message_id=msg.get("id"),+ content=msg_to_content.get(msg.get("id")),+ parts=msg.get("parts", []),+ part_validator_class=part_validator_class,+ )+ for msg in messages+ ]+++@dataclass+class ChunkSerializer:+ on_send_chunk: Callable[[dict[str, Any]], None]+ _text_id: str | None = None++ def handle_chunk(self, chunk: Any) -> None:+ """Handle a Vercel AI SDK chunk"""++ # Handle Pydantic AI's Vercel AI SDK chunks+ if DependencyManager.pydantic_ai.imported():+ from pydantic_ai.ui.vercel_ai.response_types import (+ BaseChunk,+ )++ if isinstance(chunk, BaseChunk):+ # by_alias=True: Use camelCase keys expected by Vercel AI SDK.+ # exclude_none=True: Remove null values which cause validation errors.+ self.on_send_chunk(+ chunk.model_dump(+ mode="json", by_alias=True, exclude_none=True+ ))- for msg in messages- ]+ return++ # Handle plain text chunks+ if isinstance(chunk, str):+ if self._text_id is None:+ self._text_id = f"text_{uuid.uuid4().hex}"+ self.on_send_chunk({"type": "text-start", "id": self._text_id})+ self.on_send_chunk(+ {"type": "text-delta", "id": self._text_id, "delta": chunk}+ )+ return++ # Otherwise, we return the chunk as is+ self.on_send_chunk(chunk)- return [from_chat_message_dict(msg) for msg in messages]+ def on_end(self) -> None:+ if self._text_id is not None:+ self.on_send_chunk({"type": "text-end", "id": self._text_id})diff --git a/marimo/_smoke_tests/ai/chat-variations.py b/marimo/_smoke_tests/ai/chat-variations.pynew file mode 100644index 0000000000..959e7e127a--- /dev/null+++ b/marimo/_smoke_tests/ai/chat-variations.py@@ -0,0 +1,173 @@+import marimo++__generated_with = "0.19.2"+app = marimo.App(width="medium", auto_download=["html"])+++def _():+ import uuid+ import time+ import asyncio+ import pydantic_ai.ui.vercel_ai.response_types as vercel+ return asyncio, time, uuid, vercel+++def _(+ async_no_streaming_text,+ async_streaming_chunks,+ async_streaming_text,+ mo,+ sync_no_streaming_object,+ sync_no_streaming_text,+ sync_streaming_text,+):+ model = mo.ui.dropdown(+ [+ async_streaming_chunks,+ async_streaming_text,+ async_no_streaming_text,+ sync_streaming_text,+ sync_no_streaming_text,+ sync_no_streaming_object,+ ],+ value=async_streaming_text,+ label="Select Model",+ )+ model+ return (model,)+++def _(mo, model):+ chat = mo.ui.chat(model.value)+ chat+ return (chat,)+++def _(chat):+ chat.value+ return+++def _(uuid, vercel):+ async def async_streaming_chunks(messages, config):+ # Generate unique IDs for message parts+ reasoning_id = f"reasoning_{uuid.uuid4().hex}"+ text_id = f"text_{uuid.uuid4().hex}"+ tool_id = f"tool_{uuid.uuid4().hex}"++ # --- Stream reasoning/thinking ---+ yield vercel.StartStepChunk()+ yield vercel.ReasoningStartChunk(id=reasoning_id)+ yield vercel.ReasoningDeltaChunk(+ id=reasoning_id,+ delta="The user is asking about Van Gogh. I should fetch information about his famous works.",+ )+ yield vercel.ReasoningEndChunk(id=reasoning_id)++ # --- Stream tool call to fetch artwork information ---+ yield vercel.ToolInputAvailableChunk(+ tool_call_id=tool_id,+ tool_name="search_artwork",+ input={"artist": "Vincent van Gogh", "limit": 1},+ )+ yield vercel.ToolInputStartChunk(+ tool_call_id=tool_id, tool_name="search_artwork"+ )+ yield vercel.ToolInputDeltaChunk(+ tool_call_id=tool_id,+ input_text_delta='{"artist": "Vincent van Gogh", "limit": 1}',+ )++ # --- Tool output (simulated artwork search result) ---+ yield vercel.ToolOutputAvailableChunk(+ tool_call_id=tool_id,+ output={+ "title": "The Starry Night",+ "year": 1889,+ "museum": "Museum of Modern Art",+ },+ )++ # --- Stream text response ---+ yield vercel.TextStartChunk(id=text_id)+ yield vercel.TextDeltaChunk(+ id=text_id,+ delta="One of Vincent van Gogh's most iconic works is 'The Starry Night', painted in 1889. Here's the painting:\n\n",+ )++ # --- Embed the artwork image ---+ yield vercel.FileChunk(+ url="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",+ media_type="image/jpeg",+ )+ yield vercel.TextDeltaChunk(+ id=text_id,+ delta="\nThis masterpiece is now housed at the Museum of Modern Art in New York and remains one of the most recognizable paintings in the world.",+ )+ yield vercel.TextEndChunk(id=text_id)+ yield vercel.FinishStepChunk()+ yield vercel.FinishChunk()+ return (async_streaming_chunks,)+++def _():+ import marimo as mo+ return (mo,)+++def _(asyncio):+ async def async_streaming_text(messages, config):+ await asyncio.sleep(1)+ yield "This is a simple text-only response."+ await asyncio.sleep(1)+ yield " It does not include reasoning or tool calls."+ await asyncio.sleep(1)+ yield " Have a nice day!"+ return (async_streaming_text,)+++def _(time):+ def sync_streaming_text(messages, config):+ time.sleep(1)+ yield "This"+ time.sleep(1)+ yield " is simple "+ time.sleep(1)+ yield " streaming."+ return (sync_streaming_text,)+++def _(asyncio):+ async def async_no_streaming_text(messages, config):+ await asyncio.sleep(1)+ return f"**echo**: _{messages[-1].content}_"+ return (async_no_streaming_text,)+++def _(time):+ def sync_no_streaming_text(messages, config):+ time.sleep(1)+ return f"**echo**: _{messages[-1].content}_"+ return (sync_no_streaming_text,)+++def _(mo, time):+ def sync_no_streaming_object(messages, config):+ time.sleep(1)+ return mo.ui.table([1, 2, 3, messages[-1].content])+ return (sync_no_streaming_object,)+++if __name__ == "__main__":+ app.run()diff --git a/tests/_plugins/ui/_impl/chat/test_chat.py b/tests/_plugins/ui/_impl/chat/test_chat.pyindex 510d57abdb..f7b07376ba 100644--- a/tests/_plugins/ui/_impl/chat/test_chat.py+++ b/tests/_plugins/ui/_impl/chat/test_chat.py@@ -15,6 +15,7 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._plugins import uifrom marimo._plugins.ui._impl.chat.chat import (DEFAULT_CONFIG,+ ChunkSerializer,DeleteChatMessageRequest,SendMessageRequest,)@@ -74,7 +75,13 @@ async def test_chat_send_prompt():messages: list[ChatMessage], config: ChatModelConfig) -> str:del config- return f"Response to: {messages[-1].content}"+ # Messages come in with parts from the frontend, but for testing+ # we can send them with content+ if messages[-1].content:+ content = messages[-1].content+ else:+ content = messages[-1].parts[0].text # type: ignore+ return f"Response to: {content}"chat = ui.chat(mock_model)request = SendMessageRequest(@@ -126,19 +133,26 @@ async def test_chat_send_prompt_async_generator():yield str(i)chat = ui.chat(mock_model)++ sent_messages: list[dict] = []++ def capture_send_message(message: dict, buffers): # noqa: ARG001+ sent_messages.append(message)++ chat._send_message = capture_send_message+request = SendMessageRequest(messages=[ChatMessage(role="user", content="Hello")],config=ChatModelConfig(),)response = await chat._send_prompt(request)- # All deltas are accumulated: "0" + "1" + "2" = "012"- assert response == "012"- assert len(chat._chat_history) == 2- assert chat._chat_history[0].role == "user"- assert chat._chat_history[0].content == "Hello"- assert chat._chat_history[1].role == "assistant"- assert chat._chat_history[1].content == "012"+ # Streaming returns None+ assert response is None+ # Check that chunks were sent+ assert len(sent_messages) > 0+ # Check final message is marked as final+ assert sent_messages[-1]["is_final"] is Trueasync def test_chat_streaming_sends_messages():@@ -155,9 +169,6 @@ async def test_chat_streaming_sends_messages():chat = ui.chat(mock_streaming_model)- # Mock _send_message to capture calls- original_send_message = chat._send_message-def capture_send_message(message: dict[str, object], buffers): # noqa: ARG001sent_messages.append(message)# Don't actually send to avoid needing kernel context@@ -171,12 +182,12 @@ async def test_chat_streaming_sends_messages():response = await chat._send_prompt(request)- # Verify final response (deltas accumulated)- assert response == "Hello world !"+ # Streaming returns None+ assert response is None# Verify streaming messages were sent- # Should have sent chunks for each yield, plus final message- assert len(sent_messages) >= 3+ # Should have text-start, text-deltas, text-end, plus final None message+ assert len(sent_messages) >= 7 # start + 5 deltas + end + final# Check that messages have streaming structurefor msg in sent_messages[:-1]: # All but last@@ -185,10 +196,10 @@ async def test_chat_streaming_sends_messages():assert "content" in msgassert not msg["is_final"]- # Last message should be final+ # Last message should be final with None contentassert sent_messages[-1]["type"] == "stream_chunk"assert sent_messages[-1]["is_final"]- assert sent_messages[-1]["content"] == "Hello world !"+ assert sent_messages[-1]["content"] is Noneasync def test_chat_sync_generator_streaming():@@ -216,11 +227,12 @@ async def test_chat_sync_generator_streaming():response = await chat._send_prompt(request)- # Verify final response (deltas accumulated)- assert response == "Hello world !"+ # Streaming returns None+ assert response is None# Verify streaming messages were sent- assert len(sent_messages) >= 3+ # Should have text-start, text-deltas, text-end, plus final None message+ assert len(sent_messages) >= 7# Check that messages have streaming structurefor msg in sent_messages[:-1]: # All but last@@ -229,14 +241,15 @@ async def test_chat_sync_generator_streaming():assert "content" in msgassert not msg["is_final"]- # Last message should be final+ # Last message should be final with None contentassert sent_messages[-1]["type"] == "stream_chunk"assert sent_messages[-1]["is_final"]- assert sent_messages[-1]["content"] == "Hello world !"+ assert sent_messages[-1]["content"] is Noneasync def test_chat_streaming_complete_response():- """Test that streaming returns complete response even with empty final chunks"""+ """Test that streaming sends all chunks correctly"""+ sent_messages = []def mock_streaming_model_with_empty_final(messages: list[ChatMessage], config: ChatModelConfig@@ -250,6 +263,11 @@ async def test_chat_streaming_complete_response():chat = ui.chat(mock_streaming_model_with_empty_final)+ def capture_send_message(message: dict, buffers): # noqa: ARG001+ sent_messages.append(message)++ chat._send_message = capture_send_message+request = SendMessageRequest(messages=[ChatMessage(role="user", content="Test")],config=ChatModelConfig(),@@ -257,9 +275,11 @@ async def test_chat_streaming_complete_response():response = await chat._send_prompt(request)- # Verify we got the complete final response (all deltas accumulated)- assert response == "Hello world!"- assert chat._chat_history[-1].content == "Hello world!"+ # Streaming returns None+ assert response is None+ # Verify all chunks were sent+ assert len(sent_messages) >= 4 # text-start + 3 deltas + text-end + final+ assert sent_messages[-1]["is_final"]def test_chat_get_history():@@ -332,6 +352,8 @@ def test_chat_delete_message():def test_chat_convert_value():+ """Test _convert_value with messages in parts format."""+def mock_model(messages: list[ChatMessage], config: ChatModelConfig) -> str:@@ -339,19 +361,25 @@ def test_chat_convert_value():return "Mock response"chat = ui.chat(mock_model)- value: dict[str, list[dict[str, str]]] = {+ value = {"messages": [- {"role": "user", "content": "Hello"},- {"role": "assistant", "content": "Hi there!"},+ {+ "role": "user",+ "parts": [{"type": "text", "text": "Hello"}],+ },+ {+ "role": "assistant",+ "parts": [{"type": "text", "text": "Hi there!"}],+ },]}converted: list[ChatMessage] = chat._convert_value(value)assert len(converted) == 2assert converted[0].role == "user"- assert converted[0].content == "Hello"+ assert len(converted[0].parts) == 1assert converted[1].role == "assistant"- assert converted[1].content == "Hi there!"+ assert len(converted[1].parts) == 1def test_chat_convert_value_invalid():@@ -368,7 +396,7 @@ def test_chat_convert_value_invalid():async def test_chat_with_on_message():- def mock_model(+ async def mock_model(messages: list[ChatMessage], config: ChatModelConfig) -> str:del messages, config@@ -432,7 +460,7 @@ async def test_chat_send_message_enqueues_ui_element_request(exec_req.get("""import marimo as mo- def f(messages, config):+ async def f(messages, config):return "response"chatbot = mo.ui.chat(f)@@ -579,12 +607,298 @@ def test_convert_value_with_list_input():chat._convert_value(cast(Any, [{"role": "user", "content": "Hello"}]))[email protected]("use_async", [True, False])+async def test_vercel_messages_streaming(use_async: bool):+ """Test custom model (sync/async) streaming with dict chunks."""+ sent_messages: list[dict] = []++ chunks = [+ {"type": "text-start", "id": "text-1"},+ {"type": "text-delta", "id": "text-1", "delta": "Hello"},+ {"type": "text-delta", "id": "text-1", "delta": " world"},+ {"type": "text-end", "id": "text-1"},+ ]++ async def async_model(+ messages: list[ChatMessage], config: ChatModelConfig+ ):+ del messages, config+ for chunk in chunks:+ yield chunk++ def sync_model(messages: list[ChatMessage], config: ChatModelConfig):+ del messages, config+ yield from chunks++ custom_model = async_model if use_async else sync_model+ chat = ui.chat(custom_model)++ def capture_send_message(message: dict, buffers): # noqa: ARG001+ sent_messages.append(message)++ chat._send_message = capture_send_message++ request = SendMessageRequest(+ messages=[ChatMessage(role="user", content="Hello")],+ config=ChatModelConfig(),+ )+ response = await chat._send_prompt(request)++ assert response is None++ # Extract message_id from first message for comparison+ message_id = sent_messages[0]["message_id"]++ assert sent_messages == [+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": chunks[0],+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": chunks[1],+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": chunks[2],+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": chunks[3],+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": None,+ "is_final": True,+ },+ ]+++def test_serialize_vercel_ai_chunk_dict():+ """Test ChunkSerializer with dict input."""+ sent_chunks: list[dict] = []++ def on_send_chunk(chunk: dict):+ sent_chunks.append(chunk)++ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)++ # Dict input should pass through unchanged+ chunk = {"type": "text-delta", "id": "text-1", "delta": "Hello"}+ serializer.handle_chunk(chunk)+ assert sent_chunks == [chunk]+++def test_serialize_plain_string():+ """Test ChunkSerializer with plain string input."""+ sent_chunks: list[dict] = []++ def on_send_chunk(chunk: dict):+ sent_chunks.append(chunk)++ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)++ # Plain strings should be wrapped in text-start/text-delta chunks+ serializer.handle_chunk("Hello")+ serializer.handle_chunk(" world")+ serializer.on_end()++ # Should have text-start, two text-deltas, and text-end+ assert len(sent_chunks) == 4+ assert sent_chunks[0]["type"] == "text-start"+ text_id = sent_chunks[0]["id"]+ assert sent_chunks[1] == {+ "type": "text-delta",+ "id": text_id,+ "delta": "Hello",+ }+ assert sent_chunks[2] == {+ "type": "text-delta",+ "id": text_id,+ "delta": " world",+ }+ assert sent_chunks[3] == {"type": "text-end", "id": text_id}+++def test_serialize_mixed_chunks():+ """Test ChunkSerializer with mixed dict and string chunks."""+ sent_chunks: list[dict] = []++ def on_send_chunk(chunk: dict):+ sent_chunks.append(chunk)++ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)++ # Send a dict chunk, then strings, then another dict+ serializer.handle_chunk({"type": "reasoning-start", "id": "r-1"})+ serializer.handle_chunk("text1")+ serializer.handle_chunk("text2")+ serializer.handle_chunk({"type": "reasoning-end", "id": "r-1"})+ serializer.on_end()++ # Dict chunks are sent as-is, strings are wrapped in text-start/delta/end+ # text-end is only sent when on_end() is called+ assert len(sent_chunks) == 6+ assert sent_chunks[0] == {"type": "reasoning-start", "id": "r-1"}+ assert sent_chunks[1]["type"] == "text-start"+ text_id = sent_chunks[1]["id"]+ assert sent_chunks[2] == {+ "type": "text-delta",+ "id": text_id,+ "delta": "text1",+ }+ assert sent_chunks[3] == {+ "type": "text-delta",+ "id": text_id,+ "delta": "text2",+ }+ # Dict chunk is sent as-is, even after strings+ assert sent_chunks[4] == {"type": "reasoning-end", "id": "r-1"}+ # text-end is sent when on_end() is called+ assert sent_chunks[5] == {"type": "text-end", "id": text_id}+++def test_serialize_only_dict_chunks():+ """Test ChunkSerializer with only dict chunks (no text)."""+ sent_chunks: list[dict] = []++ def on_send_chunk(chunk: dict):+ sent_chunks.append(chunk)++ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)++ # Send only dict chunks+ serializer.handle_chunk({"type": "reasoning-start", "id": "r-1"})+ serializer.handle_chunk({"type": "reasoning-end", "id": "r-1"})+ serializer.on_end()++ # Should have only the dict chunks, no text-end added+ assert sent_chunks == [+ {"type": "reasoning-start", "id": "r-1"},+ {"type": "reasoning-end", "id": "r-1"},+ ]+++ not DependencyManager.pydantic_ai.has(),+ reason="Pydantic AI is not installed",+)+def test_serialize_pydantic_chunk():+ """Test ChunkSerializer with pydantic BaseChunk input."""+ from pydantic_ai.ui.vercel_ai.response_types import TextDeltaChunk++ sent_chunks: list[dict] = []++ def on_send_chunk(chunk: dict):+ sent_chunks.append(chunk)++ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)++ # Pydantic BaseChunk should be serialized with model_dump+ chunk = TextDeltaChunk(id="text-1", delta="Hello")+ serializer.handle_chunk(chunk)++ # Should be a dict with camelCase keys (by_alias=True)+ assert sent_chunks == [+ {"type": "text-delta", "id": "text-1", "delta": "Hello"}+ ]+++ not DependencyManager.pydantic_ai.has(),+ reason="Pydantic AI is not installed",+)[email protected]("use_async", [True, False])+async def test_vercel_messages_with_pydantic_chunks(use_async: bool):+ """Test custom model (sync/async) with pydantic-ai response_types chunks."""+ from pydantic_ai.ui.vercel_ai.response_types import (+ TextDeltaChunk,+ TextEndChunk,+ TextStartChunk,+ )++ sent_messages: list[dict] = []++ chunks = [+ TextStartChunk(id="text-1"),+ TextDeltaChunk(id="text-1", delta="Hello"),+ TextEndChunk(id="text-1"),+ ]++ async def async_model(+ messages: list[ChatMessage], config: ChatModelConfig+ ):+ del messages, config+ for chunk in chunks:+ yield chunk++ def sync_model(messages: list[ChatMessage], config: ChatModelConfig):+ del messages, config+ yield from chunks++ custom_model = async_model if use_async else sync_model+ chat = ui.chat(custom_model)++ def capture_send_message(message: dict, buffers): # noqa: ARG001+ sent_messages.append(message)++ chat._send_message = capture_send_message++ request = SendMessageRequest(+ messages=[ChatMessage(role="user", content="Hello")],+ config=ChatModelConfig(),+ )+ response = await chat._send_prompt(request)+ assert response is None++ message_id = sent_messages[0]["message_id"]+ assert sent_messages == [+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": {"type": "text-start", "id": "text-1"},+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": {+ "type": "text-delta",+ "id": "text-1",+ "delta": "Hello",+ },+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": {"type": "text-end", "id": "text-1"},+ "is_final": False,+ },+ {+ "type": "stream_chunk",+ "message_id": message_id,+ "content": None,+ "is_final": True,+ },+ ]++@pytest.mark.skipif(not DependencyManager.pydantic_ai.has(),reason="Pydantic AI is not installed",)-def test_convert_value_frontend_managed():- """Test _convert_value in frontend-managed mode."""+def test_convert_value_with_parts():+ """Test _convert_value with message parts."""from pydantic_ai.ui.vercel_ai.request_types import TextUIPartdef mock_model(@@ -594,8 +908,6 @@ def test_convert_value_frontend_managed():return "Mock response"chat = ui.chat(mock_model)- # Force frontend-managed mode- chat._frontend_managed = Truevalue = {"messages": [@@ -629,12 +941,49 @@ def test_convert_value_frontend_managed():]+ not DependencyManager.pydantic_ai.has(),+ reason="Pydantic AI is not installed",+)+def test_convert_value_single_message_with_parts():+ """Test _convert_value with a single message containing parts."""+ from pydantic_ai.ui.vercel_ai.request_types import TextUIPart++ def mock_model(+ messages: list[ChatMessage], config: ChatModelConfig+ ) -> str:+ del messages, config+ return "Mock response"++ chat = ui.chat(mock_model)++ value = {+ "messages": [+ {+ "id": "msg-1",+ "role": "user",+ "parts": [{"type": "text", "text": "Hello"}],+ },+ ]+ }++ converted = chat._convert_value(value)+ assert converted == [+ ChatMessage(+ role="user",+ id="msg-1",+ parts=[TextUIPart(type="text", text="Hello")], # type: ignore+ content=None,+ )+ ]++@pytest.mark.skipif(not DependencyManager.pydantic_ai.has(),reason="We use Pydantic to check for vercel parts",)-def test_convert_value_frontend_managed_missing_fields():- """Test _convert_value in frontend-managed mode with missing fields."""+def test_convert_value_with_missing_parts():+ """Test _convert_value with message missing parts field."""def mock_model(messages: list[ChatMessage], config: ChatModelConfig@@ -643,7 +992,6 @@ def test_convert_value_frontend_managed_missing_fields():return "Mock response"chat = ui.chat(mock_model)- chat._frontend_managed = True# Message without partsvalue = {"messages": [{"role": "user", "id": ""}]}@@ -654,16 +1002,15 @@ def test_convert_value_frontend_managed_missing_fields():]-async def test_frontend_managed_streaming_sends_serialized_chunks():- """Test that frontend-managed streaming sends already-serialized chunks correctly."""+async def test_streaming_with_dict_chunks():+ """Test that streaming sends dict chunks correctly."""- # Chunks are already serialized dicts (from pydantic_ai._serialize_vercel_ai_chunk)+ # Chunks are dicts (like from pydantic_ai vercel response types)chunk1 = {"type": "text-delta", "textDelta": "Hello"}chunk2 = {"type": "text-delta", "textDelta": " world"}- # "done" chunks are filtered out by pydantic_ai._serialize_vercel_ai_chunk- async def mock_pydantic_ai_generator():- """Mock async generator yielding pre-serialized dict chunks."""+ async def mock_dict_generator():+ """Mock async generator yielding dict chunks."""yield chunk1yield chunk2@@ -674,7 +1021,6 @@ async def test_frontend_managed_streaming_sends_serialized_chunks():return "Mock response"chat = ui.chat(mock_model)- chat._frontend_managed = Truesent_messages: list[dict] = []@@ -683,11 +1029,9 @@ async def test_frontend_managed_streaming_sends_serialized_chunks():chat._send_message = capture_send_message- result = await chat._handle_streaming_response(- mock_pydantic_ai_generator()- )+ result = await chat._handle_streaming_response(mock_dict_generator())- # Result should be None for frontend-managed mode+ # Result should be Noneassert result is Noneassert sent_messages == [diff --git a/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py b/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.pyindex 2ee69a3a64..f1c530980c 100644--- a/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py+++ b/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py@@ -1,10 +1,16 @@# Copyright 2026 Marimo. All rights reserved.-"""Tests for delta-based streaming in chat UI."""+"""Tests for delta-based streaming in chat UI.++The streaming implementation uses the Vercel AI SDK protocol:+- text-start: Begins a text stream with an id+- text-delta: Contains incremental text content+- text-end: Marks the end of a text stream+"""from __future__ import annotationsimport asyncio-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyif TYPE_CHECKING:from collections.abc import AsyncGenerator, Generator@@ -13,11 +19,26 @@ from marimo._ai._types import ChatMessagefrom marimo._plugins.ui._impl.chat.chat import chat+def extract_deltas(sent_messages: list[dict[str, Any]]) -> list[str]:+ """Extract delta text from Vercel AI SDK format messages."""+ deltas = []+ for msg in sent_messages:+ content = msg.get("content")+ if isinstance(content, dict) and content.get("type") == "text-delta":+ deltas.append(content.get("delta", ""))+ return deltas+++def get_accumulated_text(sent_messages: list[dict[str, Any]]) -> str:+ """Get the accumulated text from all deltas."""+ return "".join(extract_deltas(sent_messages))++class TestDeltaStreaming:- """Test delta-based streaming mode."""+ """Test delta-based streaming mode using Vercel AI SDK protocol."""- async def test_async_delta_streaming_accumulates(self):- """Test that async generators yielding deltas are accumulated correctly."""+ async def test_async_delta_streaming_sends_vercel_chunks(self):+ """Test that async generators send Vercel AI SDK format chunks."""async def delta_generator() -> AsyncGenerator[str, None]:"""Yields individual delta chunks."""@@ -26,35 +47,40 @@ class TestDeltaStreaming:yield "world"yield "!"- # Create chat with the delta generatorchat_ui = chat(delta_generator)+ sent_messages: list[dict[str, Any]] = []- # Track messages sent- sent_messages: list[dict] = []-- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)chat_ui._send_message = capture_send # type: ignore- # Handle the streaming response+ # Handle the streaming response (returns None in new implementation)result = await chat_ui._handle_streaming_response(delta_generator())- # Verify final result is fully accumulated- assert result == "Hello world!"+ # New implementation returns None (frontend manages state)+ assert result is None++ # Verify Vercel AI SDK format: text-start, text-delta(s), text-end, final+ content_types = [+ msg["content"]["type"]+ for msg in sent_messages+ if isinstance(msg.get("content"), dict)+ ]+ assert content_types[0] == "text-start"+ assert all(t == "text-delta" for t in content_types[1:-1])+ assert content_types[-1] == "text-end"- # Verify incremental chunks were sent correctly- assert len(sent_messages) == 5 # 4 deltas + 1 final- assert sent_messages[0]["content"] == "Hello"- assert sent_messages[1]["content"] == "Hello "- assert sent_messages[2]["content"] == "Hello world"- assert sent_messages[3]["content"] == "Hello world!"- assert sent_messages[4]["content"] == "Hello world!"- assert sent_messages[4]["is_final"] is True+ # Verify accumulated text from deltas+ accumulated = get_accumulated_text(sent_messages)+ assert accumulated == "Hello world!"++ # Final message should have is_final=True+ assert sent_messages[-1]["is_final"] is True- async def test_sync_delta_streaming_accumulates(self):- """Test that sync generators yielding deltas are accumulated correctly."""+ async def test_sync_delta_streaming_sends_vercel_chunks(self):+ """Test that sync generators send Vercel AI SDK format chunks."""def delta_generator() -> Generator[str, None, None]:"""Yields individual delta chunks."""@@ -64,32 +90,20 @@ class TestDeltaStreaming:yield " "yield "Three"- # Create chat with the delta generatorchat_ui = chat(delta_generator)+ sent_messages: list[dict[str, Any]] = []- # Track messages sent- sent_messages: list[dict] = []-- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)chat_ui._send_message = capture_send # type: ignore- # Handle the streaming responseresult = await chat_ui._handle_streaming_response(delta_generator())- # Verify final result is fully accumulated- assert result == "One Two Three"-- # Verify incremental chunks were sent correctly- assert len(sent_messages) == 6 # 5 deltas + 1 final- assert sent_messages[0]["content"] == "One"- assert sent_messages[1]["content"] == "One "- assert sent_messages[2]["content"] == "One Two"- assert sent_messages[3]["content"] == "One Two "- assert sent_messages[4]["content"] == "One Two Three"- assert sent_messages[5]["is_final"] is True+ assert result is None+ accumulated = get_accumulated_text(sent_messages)+ assert accumulated == "One Two Three"async def test_empty_delta_handled(self):"""Test that empty deltas are handled gracefully."""@@ -103,9 +117,9 @@ class TestDeltaStreaming:yield "world"chat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -113,9 +127,11 @@ class TestDeltaStreaming:result = await chat_ui._handle_streaming_response(delta_generator())- assert result == "Hello world"- # Empty deltas still trigger message sends- assert len(sent_messages) == 6 # 5 deltas (including empty) + 1 final+ assert result is None+ # Empty deltas are still sent as text-delta chunks+ deltas = extract_deltas(sent_messages)+ assert deltas == ["Hello", "", " ", "", "world"]+ assert get_accumulated_text(sent_messages) == "Hello world"async def test_single_delta(self):"""Test streaming with a single delta."""@@ -124,9 +140,9 @@ class TestDeltaStreaming:yield "Single chunk"chat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -134,10 +150,12 @@ class TestDeltaStreaming:result = await chat_ui._handle_streaming_response(delta_generator())- assert result == "Single chunk"- assert len(sent_messages) == 2 # 1 delta + 1 final- assert sent_messages[0]["content"] == "Single chunk"- assert sent_messages[1]["is_final"] is True+ assert result is None+ assert get_accumulated_text(sent_messages) == "Single chunk"++ # Should have: text-start, text-delta, text-end, final+ assert len(sent_messages) == 4+ assert sent_messages[-1]["is_final"] is Trueasync def test_no_deltas(self):"""Test streaming with no deltas (empty generator)."""@@ -148,9 +166,9 @@ class TestDeltaStreaming:yield ""chat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -158,9 +176,10 @@ class TestDeltaStreaming:result = await chat_ui._handle_streaming_response(delta_generator())- assert result == ""- # No final message sent if no content- assert len(sent_messages) == 0+ assert result is None+ # Should only have the final message (no text-start/end since no text)+ assert len(sent_messages) == 1+ assert sent_messages[0]["is_final"] is Trueasync def test_unicode_deltas(self):"""Test that unicode characters in deltas are handled correctly."""@@ -173,9 +192,9 @@ class TestDeltaStreaming:yield "!"chat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -183,8 +202,8 @@ class TestDeltaStreaming:result = await chat_ui._handle_streaming_response(delta_generator())- assert result == "Hello 🌍 世界!"- assert sent_messages[-1]["content"] == "Hello 🌍 世界!"+ assert result is None+ assert get_accumulated_text(sent_messages) == "Hello 🌍 世界!"async def test_message_id_consistency(self):"""Test that all chunks in a stream share the same message_id."""@@ -195,9 +214,9 @@ class TestDeltaStreaming:yield "C"chat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -232,9 +251,9 @@ class TestDeltaStreaming:await asyncio.sleep(0.001) # Tiny delaychat_ui = chat(delta_generator)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -242,16 +261,24 @@ class TestDeltaStreaming:result = await chat_ui._handle_streaming_response(delta_generator())+ assert result is Noneexpected = "This is a long streaming response with many deltas."- assert result == expected- assert len(sent_messages) == 11 # 10 deltas + 1 final-- # Verify progressive accumulation- assert sent_messages[0]["content"] == "This"- assert sent_messages[1]["content"] == "This is"- assert sent_messages[2]["content"] == "This is a"- assert sent_messages[-1]["content"] == expected- assert sent_messages[-1]["is_final"] is True+ assert get_accumulated_text(sent_messages) == expected++ # Verify deltas are individual words+ deltas = extract_deltas(sent_messages)+ assert deltas == [+ "This",+ " is",+ " a",+ " long",+ " streaming",+ " response",+ " with",+ " many",+ " deltas",+ ".",+ ]class TestStreamingWithChatModels:@@ -262,7 +289,7 @@ class TestStreamingWithChatModels:async def custom_model(messages: list[ChatMessage],- config: dict, # noqa: ARG001+ config: dict[str, Any], # noqa: ARG001) -> AsyncGenerator[str, None]:"""Custom model that yields deltas."""user_msg = messages[-1].content@@ -272,9 +299,9 @@ class TestStreamingWithChatModels:await asyncio.sleep(0.001)chat_ui = chat(custom_model)- sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_send(msg: dict, **kwargs: object) -> None:+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unusedsent_messages.append(msg)@@ -286,61 +313,44 @@ class TestStreamingWithChatModels:result = await chat_ui._handle_streaming_response(generator)- assert result == "You said: Hello"- # Should have one chunk per character plus final- assert len(sent_messages) > 0+ assert result is None+ assert get_accumulated_text(sent_messages) == "You said: Hello"assert sent_messages[-1]["is_final"] is Trueclass TestStreamingEfficiency:- """Test that delta streaming is efficient compared to accumulated streaming."""+ """Test that delta streaming is efficient."""- async def test_delta_vs_accumulated_bandwidth(self):- """Demonstrate bandwidth savings of delta vs accumulated streaming."""+ async def test_delta_streaming_sends_individual_chunks(self):+ """Verify delta streaming sends individual chunks, not accumulated."""# Simulate 100-word responsewords = ["word" + str(i) for i in range(100)]- # Delta streaming: each word is sent onceasync def delta_stream() -> AsyncGenerator[str, None]:for word in words:yield word + " "chat_ui = chat(delta_stream)- delta_sent_messages: list[dict] = []+ sent_messages: list[dict[str, Any]] = []- def capture_delta(msg: dict, **kwargs: object) -> None:+ def capture_delta(msg: dict[str, Any], **kwargs: object) -> None:del kwargs # Unused- delta_sent_messages.append(msg)+ sent_messages.append(msg)chat_ui._send_message = capture_delta # type: ignoreawait chat_ui._handle_streaming_response(delta_stream())- # Calculate bytes sent with delta streaming- # Backend sends accumulated text, but receives deltas- delta_bytes_received = sum(len(word + " ") for word in words)-- # Each message sends the accumulated content- delta_bytes_sent = sum(- len(msg["content"]) for msg in delta_sent_messages- )-- # With old accumulated approach, model would yield:- # "word0 ", "word0 word1 ", "word0 word1 word2 ", etc.- accumulated_bytes_received = sum(- len(" ".join(words[: i + 1]) + " ") for i in range(len(words))- )-- # Delta mode receives much less data from the model- assert delta_bytes_received < accumulated_bytes_received-- # For 100 words, delta receives ~100 words worth of data- # while accumulated receives ~5000 words worth (1+2+3+...+100)- efficiency_ratio = accumulated_bytes_received / delta_bytes_received- assert efficiency_ratio > 40 # Should be ~50x more efficient-- print("\nEfficiency Test Results:")- print(f"Delta bytes received: {delta_bytes_received}")- print(f"Accumulated bytes received: {accumulated_bytes_received}")- print(f"Efficiency ratio: {efficiency_ratio:.1f}x")+ # Extract all deltas sent+ deltas = extract_deltas(sent_messages)++ # Each delta should be a single word (not accumulated)+ assert len(deltas) == 100+ assert deltas[0] == "word0 "+ assert deltas[1] == "word1 " # NOT "word0 word1 "+ assert deltas[99] == "word99 "++ # Total accumulated text should be all words+ accumulated = get_accumulated_text(sent_messages)+ assert accumulated == " ".join(words) + " "