Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

support custom models returning vercel chunks (#7797)

Shahmir Varqha64d35f12d261
diff --git a/examples/ai/chat/pydantic-ai-chat.py b/examples/ai/chat/pydantic-ai-chat.py
index e5d6f1b520..5c41f6b5e8 100644
--- a/examples/ai/chat/pydantic-ai-chat.py
+++ b/examples/ai/chat/pydantic-ai-chat.py
@@ -78,7 +78,7 @@ def _(models):
return input_key, model_name, os_key
[email protected](hide_code=True)
def get_model(
model_name: str, thinking: bool, api_key: str
) -> tuple[Model, ModelSettings]:
@@ -168,18 +168,17 @@ def _(
@app.cell
-def _(agent, model_name):
+def _(agent):
chatbot = mo.ui.chat(
mo.ai.llm.pydantic_ai(agent),
prompts=[
"Write the fibonacci function in Python",
"Who is Ada Lovelace?",
"What is marimo?",
- "I need dogs",
+ "I need dogs (render as markdown)",
],
allow_attachments=True,
show_configuration_controls=True,
- config={"top_p": None} if model_name.startswith("claude") else None,
)
chatbot
return (chatbot,)
@@ -191,5 +190,90 @@ def _(chatbot):
return
+def _():
+ mo.md("""
+ ## Custom Model Sample
+ """)
+ return
+
+
+def _():
+ import uuid
+ import pydantic_ai.ui.vercel_ai.response_types as vercel
+
+
+ async def custom_model(messages, config):
+ # Generate unique IDs for message parts
+ reasoning_id = f"reasoning_{uuid.uuid4().hex}"
+ text_id = f"text_{uuid.uuid4().hex}"
+ tool_id = f"tool_{uuid.uuid4().hex}"
+
+ # --- Stream reasoning/thinking ---
+ yield vercel.StartStepChunk()
+ yield vercel.ReasoningStartChunk(id=reasoning_id)
+ yield vercel.ReasoningDeltaChunk(
+ id=reasoning_id,
+ delta="The user is asking about Van Gogh. I should fetch information about his famous works.",
+ )
+ yield vercel.ReasoningEndChunk(id=reasoning_id)
+
+ # --- Stream tool call to fetch artwork information ---
+ yield vercel.ToolInputAvailableChunk(
+ tool_call_id=tool_id,
+ tool_name="search_artwork",
+ input={"artist": "Vincent van Gogh", "limit": 1},
+ )
+ yield vercel.ToolInputStartChunk(
+ tool_call_id=tool_id, tool_name="search_artwork"
+ )
+ yield vercel.ToolInputDeltaChunk(
+ tool_call_id=tool_id,
+ input_text_delta='{"artist": "Vincent van Gogh", "limit": 1}',
+ )
+
+ # --- Tool output (simulated artwork search result) ---
+ yield vercel.ToolOutputAvailableChunk(
+ tool_call_id=tool_id,
+ output={
+ "title": "The Starry Night",
+ "year": 1889,
+ "museum": "Museum of Modern Art",
+ },
+ )
+
+ # --- Stream text response ---
+ yield vercel.TextStartChunk(id=text_id)
+ yield vercel.TextDeltaChunk(
+ id=text_id,
+ delta="One of Vincent van Gogh's most iconic works is 'The Starry Night', painted in 1889. Here's the painting:\n\n",
+ )
+
+ # --- Embed the artwork image ---
+ yield vercel.FileChunk(
+ url="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",
+ media_type="image/jpeg",
+ )
+ yield vercel.TextDeltaChunk(
+ id=text_id,
+ delta="\nThis masterpiece is now housed at the Museum of Modern Art in New York and remains one of the most recognizable paintings in the world.",
+ )
+ yield vercel.TextEndChunk(id=text_id)
+ yield vercel.FinishStepChunk()
+ yield vercel.FinishChunk()
+
+
+ custom_chat = mo.ui.chat(custom_model)
+ custom_chat
+ return (custom_chat,)
+
+
+def _(custom_chat):
+ custom_chat.value
+ return
+
+
if __name__ == "__main__":
app.run()
diff --git a/frontend/src/plugins/impl/chat/ChatPlugin.tsx b/frontend/src/plugins/impl/chat/ChatPlugin.tsx
index 7454bb28c6..2972d80398 100644
--- a/frontend/src/plugins/impl/chat/ChatPlugin.tsx
+++ b/frontend/src/plugins/impl/chat/ChatPlugin.tsx
@@ -15,7 +15,7 @@ export type PluginFunctions = {
get_chat_history: (req: {}) => Promise<{ messages: UIMessage[] }>;
delete_chat_history: (req: {}) => Promise<null>;
delete_chat_message: (req: { index: number }) => Promise<null>;
- send_prompt: (req: SendMessageRequest) => Promise<string | null>;
+ send_prompt: (req: SendMessageRequest) => Promise<unknown>;
};
const messageSchema = z.array(
@@ -47,7 +47,6 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(
maxHeight: z.number().optional(),
config: configSchema,
allowAttachments: z.union([z.boolean(), z.string().array()]),
- frontendManaged: z.boolean(),
}),
)
.withFunctions<PluginFunctions>({
@@ -67,7 +66,7 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(
config: configSchema,
}),
)
- .output(z.union([z.string(), z.null()])),
+ .output(z.unknown()),
})
.renderer((props) => (
<TooltipProvider>
@@ -77,7 +76,6 @@ export const ChatPlugin = createPlugin<{ messages: UIMessage[] }>(
showConfigurationControls={props.data.showConfigurationControls}
maxHeight={props.data.maxHeight}
allowAttachments={props.data.allowAttachments}
- frontendManaged={props.data.frontendManaged}
config={props.data.config}
get_chat_history={props.functions.get_chat_history}
delete_chat_history={props.functions.delete_chat_history}
diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsx
index e68cf7d8a4..4032fb154e 100644
--- a/frontend/src/plugins/impl/chat/chat-ui.tsx
+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx
@@ -66,7 +66,6 @@ interface Props extends PluginFunctions {
showConfigurationControls: boolean;
maxHeight: number | undefined;
allowAttachments: boolean | string[];
- frontendManaged: boolean;
value: UIMessage[];
setValue: (messages: UIMessage[]) => void;
host: HTMLElement;
@@ -166,113 +165,42 @@ export const Chatbot: React.FC<Props> = (props) => {
};
});
- if (props.frontendManaged) {
- const stream = new ReadableStream<UIMessageChunk>({
- start(controller) {
- frontendStreamControllerRef.current = controller;
+ const stream = new ReadableStream<UIMessageChunk>({
+ start(controller) {
+ frontendStreamControllerRef.current = controller;
- const abortHandler = () => {
- try {
- controller.close();
- } catch (error) {
- Logger.debug("Controller may already be closed", { error });
- }
- frontendStreamControllerRef.current = null;
- };
- signal?.addEventListener("abort", abortHandler);
-
- return () => {
- signal?.removeEventListener("abort", abortHandler);
- };
- },
- cancel() {
- frontendStreamControllerRef.current = null;
- },
- });
-
- // Start the prompt, chunks will be sent via events
- props
- .send_prompt({
- messages: messages,
- config: chatConfig,
- })
- .catch((error: Error) => {
- frontendStreamControllerRef.current?.error(error);
- frontendStreamControllerRef.current = null;
- });
-
- return createUIMessageStreamResponse({ stream });
- }
-
- if (signal?.aborted) {
- return new Response("Aborted", { status: 499 });
- }
-
- // Create a placeholder message for streaming (backend-managed)
- const messageId = Date.now().toString();
-
- setMessages((prev) => [
- ...prev,
- {
- id: messageId,
- role: "assistant",
- parts: [{ type: "text", text: "" }],
- },
- ]);
-
- // Create an abort-aware promise for the send_prompt call
- const sendPromptPromise = props.send_prompt({
- messages: messages,
- config: chatConfig,
- });
-
- // Race the send_prompt with an abort signal
- const response = await new Promise<string | null>(
- (resolve, reject) => {
- // Listen for abort
const abortHandler = () => {
- reject(new DOMException("Aborted", "AbortError"));
+ try {
+ controller.close();
+ } catch (error) {
+ Logger.debug("Controller may already be closed", { error });
+ }
+ frontendStreamControllerRef.current = null;
};
signal?.addEventListener("abort", abortHandler);
- sendPromptPromise
- .then(resolve)
- .catch(reject)
- .finally(() => {
- signal?.removeEventListener("abort", abortHandler);
- });
+ return () => {
+ signal?.removeEventListener("abort", abortHandler);
+ };
},
- );
+ cancel() {
+ frontendStreamControllerRef.current = null;
+ },
+ });
- if (response === null) {
- Logger.error("Non-frontend-managed response is null", {
- response,
+ // Start the prompt, chunks will be sent via events
+ void props
+ .send_prompt({
+ messages: messages,
+ config: chatConfig,
+ })
+ .catch((error: Error) => {
+ frontendStreamControllerRef.current?.error(error);
+ frontendStreamControllerRef.current = null;
});
- return new Response("Internal server error", { status: 500 });
- }
- // If streaming didn't happen (non-generator response), update the message
- // Check if streaming state is still set (meaning no chunks were received)
- if (
- streamingStateRef.current.backendMessageId === null &&
- streamingStateRef.current.frontendMessageIndex === null
- ) {
- setMessages((prev) => {
- const updated = [...prev];
- const index = updated.findIndex((m) => m.id === messageId);
- if (index !== -1) {
- updated[index] = {
- ...updated[index],
- parts: [{ type: "text", text: response }],
- };
- }
- return updated;
- });
- }
-
- return new Response(response);
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- } catch (error: any) {
+ return createUIMessageStreamResponse({ stream });
+ } catch (error: unknown) {
// Clear streaming state on error
streamingStateRef.current = {
backendMessageId: null,
@@ -280,13 +208,13 @@ export const Chatbot: React.FC<Props> = (props) => {
};
// Handle abort gracefully without showing an error
- if (error.name === "AbortError") {
+ if (error instanceof Error && error.name === "AbortError") {
return new Response("Aborted", { status: 499 });
}
// HACK: strip the error message to clean up the response
- const strippedError = error.message
- .split("failed with exception ")
+ const strippedError = (error as Error).message
+ ?.split("failed with exception ")
.pop();
return new Response(strippedError, { status: 400 });
}
@@ -307,11 +235,7 @@ export const Chatbot: React.FC<Props> = (props) => {
frontendMessageIndex: null,
};
- // For frontend-managed streaming, we set the value directly from the frontend.
- // Because useChat creates the proper message structure for us.
- if (props.frontendManaged) {
- props.setValue(message.messages);
- }
+ props.setValue(message.messages);
},
onError: (error) => {
Logger.error("An error occurred:", error);
@@ -338,90 +262,28 @@ export const Chatbot: React.FC<Props> = (props) => {
return;
}
- if (props.frontendManaged) {
- // Push to the stream for useChat to process
- const controller = frontendStreamControllerRef.current;
- if (!controller) {
- return;
- }
-
- const frontendMessage = message as {
- type: string;
- message_id: string;
- content?: UIMessageChunk;
- is_final?: boolean;
- };
-
- if (frontendMessage.content) {
- controller.enqueue(frontendMessage.content);
- }
- if (frontendMessage.is_final) {
- controller.close();
- frontendStreamControllerRef.current = null;
- }
+ // Push to the stream for useChat to process
+ const controller = frontendStreamControllerRef.current;
+ if (!controller) {
return;
}
- // Handle regular text streaming chunks
- const chunkMessage = message as {
+ const frontendMessage = message as {
type: string;
message_id: string;
- content: string;
- is_final: boolean;
+ content?: UIMessageChunk;
+ is_final?: boolean;
};
- // Initialize streaming state on first chunk if not already set
- if (streamingStateRef.current.backendMessageId === null) {
- // Find the last assistant message (which should be the placeholder we created)
- setMessages((prev) => {
- const updated = [...prev];
- // Find the last assistant message
- for (let i = updated.length - 1; i >= 0; i--) {
- if (updated[i].role === "assistant") {
- streamingStateRef.current = {
- backendMessageId: chunkMessage.message_id,
- frontendMessageIndex: i,
- };
- break;
- }
- }
- return updated;
- });
+ if (frontendMessage.content) {
+ controller.enqueue(frontendMessage.content);
}
-
- // Only process chunks for the current streaming message
- const frontendIndex = streamingStateRef.current.frontendMessageIndex;
- if (
- streamingStateRef.current.backendMessageId ===
- chunkMessage.message_id &&
- frontendIndex !== null
- ) {
- setMessages((prev) => {
- const updated = [...prev];
- const index = frontendIndex;
-
- // Update the message at the tracked index
- if (index < updated.length) {
- const messageToUpdate = updated[index];
- if (messageToUpdate.role === "assistant") {
- updated[index] = {
- ...messageToUpdate,
- parts: [{ type: "text", text: chunkMessage.content }],
- };
- }
- }
-
- return updated;
- });
-
- // Clear streaming state when final chunk arrives
- if (chunkMessage.is_final) {
- streamingStateRef.current = {
- backendMessageId: null,
- frontendMessageIndex: null,
- };
- }
+ if (frontendMessage.is_final) {
+ controller.close();
+ frontendStreamControllerRef.current = null;
}
+
+ return;
},
);
@@ -434,10 +296,7 @@ export const Chatbot: React.FC<Props> = (props) => {
props.delete_chat_message({ index });
setMessages(newMessages);
- // Since we manage the state in the frontend, we need to update the value.
- if (props.frontendManaged) {
- props.setValue(newMessages);
- }
+ props.setValue(newMessages);
}
};
@@ -526,7 +385,7 @@ export const Chatbot: React.FC<Props> = (props) => {
return (
<div
- key={message.id}
+ key={`${message.id}-${index}`}
className={cn(
"flex flex-col group gap-2",
message.role === "user" ? "items-end" : "items-start",
diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.py
index 51a10abbad..abe6a0799c 100644
--- a/marimo/_ai/_types.py
+++ b/marimo/_ai/_types.py
@@ -18,6 +18,7 @@ import msgspec
from marimo import _loggers
from marimo._dependencies.dependencies import DependencyManager
+from marimo._plugins.utils import remove_none_values
from marimo._utils.parse_dataclass import parse_raw
LOGGER = _loggers.marimo_logger()
@@ -270,12 +271,15 @@ class ChatMessage(msgspec.Struct):
if isinstance(part, part_validator_class):
validated_parts.append(part)
elif isinstance(part, dict):
+ sanitized_part = remove_none_values(part)
# Try pydantic validation for dict -> class conversion
try:
from pydantic import TypeAdapter
adapter = TypeAdapter(part_validator_class)
- validated_parts.append(adapter.validate_python(part))
+ validated_parts.append(
+ adapter.validate_python(sanitized_part)
+ )
except ImportError:
LOGGER.debug(
"Pydantic not installed, skipping dict validation"
diff --git a/marimo/_ai/llm/_impl.py b/marimo/_ai/llm/_impl.py
index e1ed64dece..31f8042e7f 100644
--- a/marimo/_ai/llm/_impl.py
+++ b/marimo/_ai/llm/_impl.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, cast
from marimo import _loggers
from marimo._ai._pydantic_ai_utils import generate_id
+from marimo._plugins.utils import remove_none_values
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
@@ -746,9 +747,11 @@ class pydantic_ai(ChatModel):
parts = cast(
list[UIMessagePart],
[
- dataclasses.asdict(part)
- if dataclasses.is_dataclass(part)
- else part
+ self._remove_none_values(
+ dataclasses.asdict(part)
+ if dataclasses.is_dataclass(part)
+ else part
+ )
for part in message.parts
],
)
@@ -776,6 +779,11 @@ class pydantic_ai(ChatModel):
)
return ui_messages
+ def _remove_none_values(self, obj: dict[str, Any]) -> dict[str, Any]:
+ if isinstance(obj, dict) and hasattr(obj, "items"):
+ return remove_none_values(obj)
+ return obj
+
def _serialize_vercel_ai_chunk(
self, chunk: BaseChunk
) -> dict[str, Any] | None:
diff --git a/marimo/_plugins/ui/_impl/chat/chat.py b/marimo/_plugins/ui/_impl/chat/chat.py
index 0d484fa473..f3b6538966 100644
--- a/marimo/_plugins/ui/_impl/chat/chat.py
+++ b/marimo/_plugins/ui/_impl/chat/chat.py
@@ -6,22 +6,24 @@ import uuid
from dataclasses import dataclass
from typing import Any, Callable, Final, Optional, Union, cast
+from marimo import _loggers
from marimo._ai._types import (
ChatMessage,
ChatModelConfig,
ChatModelConfigDict,
)
-from marimo._ai.llm._impl import pydantic_ai
+from marimo._dependencies.dependencies import DependencyManager
from marimo._output.formatting import as_html
from marimo._output.rich_help import mddoc
from marimo._plugins.core.web_component import JSONType
from marimo._plugins.ui._core.ui_element import UIElement
-from marimo._plugins.ui._impl.chat.utils import from_chat_message_dict
from marimo._runtime.commands import UpdateUIElementCommand
from marimo._runtime.context import get_context
from marimo._runtime.context.types import ContextNotInitializedError
from marimo._runtime.functions import EmptyArgs, Function
+LOGGER = _loggers.marimo_logger()
+
DEFAULT_CONFIG = ChatModelConfigDict(
max_tokens=4096,
temperature=0.5,
@@ -31,6 +33,8 @@ DEFAULT_CONFIG = ChatModelConfigDict(
presence_penalty=0,
)
+DONE_CHUNK: Final[str] = "[DONE]"
+
@dataclass
class SendMessageRequest:
@@ -125,6 +129,33 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
)
```
+ Custom model with Vercel AI SDK streaming (reasoning, tool calls):
+ ```python
+ import pydantic_ai.ui.vercel_ai.response_types as vercel
+
+
+ async def custom_model(messages, config):
+ # Stream reasoning/thinking
+ yield vercel.ReasoningStartChunk(id="reasoning-1")
+ yield vercel.ReasoningDeltaChunk(
+ id="reasoning-1", delta="Let me think..."
+ )
+ yield vercel.ReasoningEndChunk(id="reasoning-1")
+
+ # Stream text response (can also use plain dicts)
+ yield {"type": "text-start", "id": "text-1"}
+ yield vercel.TextDeltaChunk(
+ id="text-1", delta="Here is my answer."
+ )
+ yield vercel.TextEndChunk(id="text-1")
+
+ yield vercel.FinishChunk(finish_reason="stop")
+
+
+ chat = mo.ui.chat(custom_model)
+ ```
+ Refer to examples/ai/chat/pydantic-ai-chat.py for a complete example.
+
Attributes:
value (List[ChatMessage]): The current chat history, a list of ChatMessage objects.
@@ -166,10 +197,6 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
) -> None:
self._model = model
self._chat_history: list[ChatMessage] = []
- self._frontend_managed = False
-
- if isinstance(model, pydantic_ai):
- self._frontend_managed = True
if config is None:
config = DEFAULT_CONFIG
@@ -188,7 +215,6 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
"config": cast(JSONType, config or {}),
"allow-attachments": allow_attachments,
"max-height": max_height,
- "frontend-managed": self._frontend_managed,
},
functions=(
Function(
@@ -246,7 +272,7 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
buffers=None,
)
- async def _handle_streaming_response(self, response: Any) -> str | None:
+ async def _handle_streaming_response(self, response: Any) -> None:
"""Handle streaming from both sync and async generators.
Generators should yield delta chunks (new content only), which this
@@ -256,47 +282,31 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
so we don't need to return anything.
"""
message_id = str(uuid.uuid4())
- accumulated_text = ""
- if self._frontend_managed:
- async for delta in response:
- self._send_chat_message(
- message_id=message_id, content=delta, is_final=False
- )
- # Send final message to indicate streaming is complete
+ def send_chunk(chunk: dict[str, Any]) -> None:
self._send_chat_message(
- message_id=message_id, content=None, is_final=True
+ message_id=message_id,
+ content=chunk,
+ is_final=False,
)
- return None
- # Use async for if it's an async generator, otherwise regular for
+ serializer = ChunkSerializer(on_send_chunk=send_chunk)
+
if inspect.isasyncgen(response):
async for delta in response:
- # Accumulate each delta chunk
- delta_str = str(delta)
- accumulated_text += delta_str
- self._send_chat_message(
- message_id=message_id,
- content=accumulated_text,
- is_final=False,
- )
+ serializer.handle_chunk(delta)
else:
for delta in response:
- # Accumulate each delta chunk
- delta_str = str(delta)
- accumulated_text += delta_str
- self._send_chat_message(
- message_id=message_id,
- content=accumulated_text,
- is_final=False,
- )
+ serializer.handle_chunk(delta)
+
+ serializer.on_end()
# Send final message to indicate streaming is complete
- if accumulated_text:
- self._send_chat_message(
- message_id=message_id, content=accumulated_text, is_final=True
- )
- return accumulated_text
+ self._send_chat_message(
+ message_id=message_id,
+ content=None,
+ is_final=True,
+ )
def _update_chat_history(self, chat_history: list[ChatMessage]) -> None:
self._chat_history = chat_history
@@ -326,6 +336,8 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
async def _send_prompt(self, args: SendMessageRequest) -> str | None:
messages = args.messages
+ self._chat_history = messages
+
# If the model is a callable that takes a single argument,
# call it with just the messages.
response: object
@@ -338,28 +350,31 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
else:
response = self._model(messages, args.config)
+ if inspect.isasyncgen(response) or inspect.isgenerator(response):
+ # We support functions that stream the response with generators
+ # (both sync and async)
+ await self._handle_streaming_response(response)
+ # For streaming, we don't have a final response string to add to history
+ # The frontend will add the accumulated message
+ return None
+
if inspect.isawaitable(response):
response = await response
- elif inspect.isasyncgen(response) or inspect.isgenerator(response):
- # We support functions that stream the response with generators
- # (both sync and async); each yielded value is the latest
- # representation of the response, and the last value is the full value
- response = await self._handle_streaming_response(response)
- if self._frontend_managed:
- # For frontend-managed streaming, the response is set on the frontend,
- # so we don't need to return anything.
- return None
+ # Return the response as a string
+ # If the response is a rich object, convert it to markdown
+ response_str = (
+ response if isinstance(response, str) else as_html(response).text
+ )
- response_message = ChatMessage(role="assistant", content=response)
- chat_history = messages + [response_message]
- self._update_chat_history(chat_history)
+ # Add assistant response to chat history
+ assistant_message = ChatMessage(role="assistant", content=response_str)
+ self._chat_history.append(assistant_message)
- # Return the response as HTML
- # If the response is a string, convert it to markdown
- if isinstance(response, str):
- return response
- return as_html(response).text
+ # Update the chat history to trigger UI updates and on_message callback
+ self._update_chat_history(self._chat_history)
+
+ return response_str
def _convert_value(self, value: dict[str, Any]) -> list[ChatMessage]:
if not isinstance(value, dict) or "messages" not in value:
@@ -367,20 +382,71 @@ class chat(UIElement[dict[str, Any], list[ChatMessage]]):
messages = value["messages"]
- if self._frontend_managed:
- from pydantic_ai.ui.vercel_ai.request_types import UIMessagePart
-
- # The frontend sends messages as UIMessage dicts so we use pydantic-ai to cast them
- # as Vercel messages
- return [
- ChatMessage.create(
- role=msg.get("role", "user"),
- message_id=msg.get("id"),
- content=None,
- parts=msg.get("parts", []),
- part_validator_class=UIMessagePart,
+ part_validator_class = None
+ if DependencyManager.pydantic_ai.imported():
+ from pydantic_ai.ui.vercel_ai.request_types import (
+ UIMessagePart,
+ )
+
+ # The frontend sends messages as ChatMessage parts so we use pydantic-ai to cast them
+ # as Vercel UIMessagePart
+ part_validator_class = UIMessagePart
+
+ msg_to_content = {
+ msg.get("id"): msg.get("content")
+ for msg in messages
+ if msg.get("content") is not None
+ }
+
+ return [
+ ChatMessage.create(
+ role=msg.get("role", "user"),
+ message_id=msg.get("id"),
+ content=msg_to_content.get(msg.get("id")),
+ parts=msg.get("parts", []),
+ part_validator_class=part_validator_class,
+ )
+ for msg in messages
+ ]
+
+
+@dataclass
+class ChunkSerializer:
+ on_send_chunk: Callable[[dict[str, Any]], None]
+ _text_id: str | None = None
+
+ def handle_chunk(self, chunk: Any) -> None:
+ """Handle a Vercel AI SDK chunk"""
+
+ # Handle Pydantic AI's Vercel AI SDK chunks
+ if DependencyManager.pydantic_ai.imported():
+ from pydantic_ai.ui.vercel_ai.response_types import (
+ BaseChunk,
+ )
+
+ if isinstance(chunk, BaseChunk):
+ # by_alias=True: Use camelCase keys expected by Vercel AI SDK.
+ # exclude_none=True: Remove null values which cause validation errors.
+ self.on_send_chunk(
+ chunk.model_dump(
+ mode="json", by_alias=True, exclude_none=True
+ )
)
- for msg in messages
- ]
+ return
+
+ # Handle plain text chunks
+ if isinstance(chunk, str):
+ if self._text_id is None:
+ self._text_id = f"text_{uuid.uuid4().hex}"
+ self.on_send_chunk({"type": "text-start", "id": self._text_id})
+ self.on_send_chunk(
+ {"type": "text-delta", "id": self._text_id, "delta": chunk}
+ )
+ return
+
+ # Otherwise, we return the chunk as is
+ self.on_send_chunk(chunk)
- return [from_chat_message_dict(msg) for msg in messages]
+ def on_end(self) -> None:
+ if self._text_id is not None:
+ self.on_send_chunk({"type": "text-end", "id": self._text_id})
diff --git a/marimo/_smoke_tests/ai/chat-variations.py b/marimo/_smoke_tests/ai/chat-variations.py
new file mode 100644
index 0000000000..959e7e127a
--- /dev/null
+++ b/marimo/_smoke_tests/ai/chat-variations.py
@@ -0,0 +1,173 @@
+import marimo
+
+__generated_with = "0.19.2"
+app = marimo.App(width="medium", auto_download=["html"])
+
+
+def _():
+ import uuid
+ import time
+ import asyncio
+ import pydantic_ai.ui.vercel_ai.response_types as vercel
+ return asyncio, time, uuid, vercel
+
+
+def _(
+ async_no_streaming_text,
+ async_streaming_chunks,
+ async_streaming_text,
+ mo,
+ sync_no_streaming_object,
+ sync_no_streaming_text,
+ sync_streaming_text,
+):
+ model = mo.ui.dropdown(
+ [
+ async_streaming_chunks,
+ async_streaming_text,
+ async_no_streaming_text,
+ sync_streaming_text,
+ sync_no_streaming_text,
+ sync_no_streaming_object,
+ ],
+ value=async_streaming_text,
+ label="Select Model",
+ )
+ model
+ return (model,)
+
+
+def _(mo, model):
+ chat = mo.ui.chat(model.value)
+ chat
+ return (chat,)
+
+
+def _(chat):
+ chat.value
+ return
+
+
+def _(uuid, vercel):
+ async def async_streaming_chunks(messages, config):
+ # Generate unique IDs for message parts
+ reasoning_id = f"reasoning_{uuid.uuid4().hex}"
+ text_id = f"text_{uuid.uuid4().hex}"
+ tool_id = f"tool_{uuid.uuid4().hex}"
+
+ # --- Stream reasoning/thinking ---
+ yield vercel.StartStepChunk()
+ yield vercel.ReasoningStartChunk(id=reasoning_id)
+ yield vercel.ReasoningDeltaChunk(
+ id=reasoning_id,
+ delta="The user is asking about Van Gogh. I should fetch information about his famous works.",
+ )
+ yield vercel.ReasoningEndChunk(id=reasoning_id)
+
+ # --- Stream tool call to fetch artwork information ---
+ yield vercel.ToolInputAvailableChunk(
+ tool_call_id=tool_id,
+ tool_name="search_artwork",
+ input={"artist": "Vincent van Gogh", "limit": 1},
+ )
+ yield vercel.ToolInputStartChunk(
+ tool_call_id=tool_id, tool_name="search_artwork"
+ )
+ yield vercel.ToolInputDeltaChunk(
+ tool_call_id=tool_id,
+ input_text_delta='{"artist": "Vincent van Gogh", "limit": 1}',
+ )
+
+ # --- Tool output (simulated artwork search result) ---
+ yield vercel.ToolOutputAvailableChunk(
+ tool_call_id=tool_id,
+ output={
+ "title": "The Starry Night",
+ "year": 1889,
+ "museum": "Museum of Modern Art",
+ },
+ )
+
+ # --- Stream text response ---
+ yield vercel.TextStartChunk(id=text_id)
+ yield vercel.TextDeltaChunk(
+ id=text_id,
+ delta="One of Vincent van Gogh's most iconic works is 'The Starry Night', painted in 1889. Here's the painting:\n\n",
+ )
+
+ # --- Embed the artwork image ---
+ yield vercel.FileChunk(
+ url="https://upload.wikimedia.org/wikipedia/commons/thumb/e/ea/Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg/1280px-Van_Gogh_-_Starry_Night_-_Google_Art_Project.jpg",
+ media_type="image/jpeg",
+ )
+ yield vercel.TextDeltaChunk(
+ id=text_id,
+ delta="\nThis masterpiece is now housed at the Museum of Modern Art in New York and remains one of the most recognizable paintings in the world.",
+ )
+ yield vercel.TextEndChunk(id=text_id)
+ yield vercel.FinishStepChunk()
+ yield vercel.FinishChunk()
+ return (async_streaming_chunks,)
+
+
+def _():
+ import marimo as mo
+ return (mo,)
+
+
+def _(asyncio):
+ async def async_streaming_text(messages, config):
+ await asyncio.sleep(1)
+ yield "This is a simple text-only response."
+ await asyncio.sleep(1)
+ yield " It does not include reasoning or tool calls."
+ await asyncio.sleep(1)
+ yield " Have a nice day!"
+ return (async_streaming_text,)
+
+
+def _(time):
+ def sync_streaming_text(messages, config):
+ time.sleep(1)
+ yield "This"
+ time.sleep(1)
+ yield " is simple "
+ time.sleep(1)
+ yield " streaming."
+ return (sync_streaming_text,)
+
+
+def _(asyncio):
+ async def async_no_streaming_text(messages, config):
+ await asyncio.sleep(1)
+ return f"**echo**: _{messages[-1].content}_"
+ return (async_no_streaming_text,)
+
+
+def _(time):
+ def sync_no_streaming_text(messages, config):
+ time.sleep(1)
+ return f"**echo**: _{messages[-1].content}_"
+ return (sync_no_streaming_text,)
+
+
+def _(mo, time):
+ def sync_no_streaming_object(messages, config):
+ time.sleep(1)
+ return mo.ui.table([1, 2, 3, messages[-1].content])
+ return (sync_no_streaming_object,)
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/tests/_plugins/ui/_impl/chat/test_chat.py b/tests/_plugins/ui/_impl/chat/test_chat.py
index 510d57abdb..f7b07376ba 100644
--- a/tests/_plugins/ui/_impl/chat/test_chat.py
+++ b/tests/_plugins/ui/_impl/chat/test_chat.py
@@ -15,6 +15,7 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._plugins import ui
from marimo._plugins.ui._impl.chat.chat import (
DEFAULT_CONFIG,
+ ChunkSerializer,
DeleteChatMessageRequest,
SendMessageRequest,
)
@@ -74,7 +75,13 @@ async def test_chat_send_prompt():
messages: list[ChatMessage], config: ChatModelConfig
) -> str:
del config
- return f"Response to: {messages[-1].content}"
+ # Messages come in with parts from the frontend, but for testing
+ # we can send them with content
+ if messages[-1].content:
+ content = messages[-1].content
+ else:
+ content = messages[-1].parts[0].text # type: ignore
+ return f"Response to: {content}"
chat = ui.chat(mock_model)
request = SendMessageRequest(
@@ -126,19 +133,26 @@ async def test_chat_send_prompt_async_generator():
yield str(i)
chat = ui.chat(mock_model)
+
+ sent_messages: list[dict] = []
+
+ def capture_send_message(message: dict, buffers): # noqa: ARG001
+ sent_messages.append(message)
+
+ chat._send_message = capture_send_message
+
request = SendMessageRequest(
messages=[ChatMessage(role="user", content="Hello")],
config=ChatModelConfig(),
)
response = await chat._send_prompt(request)
- # All deltas are accumulated: "0" + "1" + "2" = "012"
- assert response == "012"
- assert len(chat._chat_history) == 2
- assert chat._chat_history[0].role == "user"
- assert chat._chat_history[0].content == "Hello"
- assert chat._chat_history[1].role == "assistant"
- assert chat._chat_history[1].content == "012"
+ # Streaming returns None
+ assert response is None
+ # Check that chunks were sent
+ assert len(sent_messages) > 0
+ # Check final message is marked as final
+ assert sent_messages[-1]["is_final"] is True
async def test_chat_streaming_sends_messages():
@@ -155,9 +169,6 @@ async def test_chat_streaming_sends_messages():
chat = ui.chat(mock_streaming_model)
- # Mock _send_message to capture calls
- original_send_message = chat._send_message
-
def capture_send_message(message: dict[str, object], buffers): # noqa: ARG001
sent_messages.append(message)
# Don't actually send to avoid needing kernel context
@@ -171,12 +182,12 @@ async def test_chat_streaming_sends_messages():
response = await chat._send_prompt(request)
- # Verify final response (deltas accumulated)
- assert response == "Hello world !"
+ # Streaming returns None
+ assert response is None
# Verify streaming messages were sent
- # Should have sent chunks for each yield, plus final message
- assert len(sent_messages) >= 3
+ # Should have text-start, text-deltas, text-end, plus final None message
+ assert len(sent_messages) >= 7 # start + 5 deltas + end + final
# Check that messages have streaming structure
for msg in sent_messages[:-1]: # All but last
@@ -185,10 +196,10 @@ async def test_chat_streaming_sends_messages():
assert "content" in msg
assert not msg["is_final"]
- # Last message should be final
+ # Last message should be final with None content
assert sent_messages[-1]["type"] == "stream_chunk"
assert sent_messages[-1]["is_final"]
- assert sent_messages[-1]["content"] == "Hello world !"
+ assert sent_messages[-1]["content"] is None
async def test_chat_sync_generator_streaming():
@@ -216,11 +227,12 @@ async def test_chat_sync_generator_streaming():
response = await chat._send_prompt(request)
- # Verify final response (deltas accumulated)
- assert response == "Hello world !"
+ # Streaming returns None
+ assert response is None
# Verify streaming messages were sent
- assert len(sent_messages) >= 3
+ # Should have text-start, text-deltas, text-end, plus final None message
+ assert len(sent_messages) >= 7
# Check that messages have streaming structure
for msg in sent_messages[:-1]: # All but last
@@ -229,14 +241,15 @@ async def test_chat_sync_generator_streaming():
assert "content" in msg
assert not msg["is_final"]
- # Last message should be final
+ # Last message should be final with None content
assert sent_messages[-1]["type"] == "stream_chunk"
assert sent_messages[-1]["is_final"]
- assert sent_messages[-1]["content"] == "Hello world !"
+ assert sent_messages[-1]["content"] is None
async def test_chat_streaming_complete_response():
- """Test that streaming returns complete response even with empty final chunks"""
+ """Test that streaming sends all chunks correctly"""
+ sent_messages = []
def mock_streaming_model_with_empty_final(
messages: list[ChatMessage], config: ChatModelConfig
@@ -250,6 +263,11 @@ async def test_chat_streaming_complete_response():
chat = ui.chat(mock_streaming_model_with_empty_final)
+ def capture_send_message(message: dict, buffers): # noqa: ARG001
+ sent_messages.append(message)
+
+ chat._send_message = capture_send_message
+
request = SendMessageRequest(
messages=[ChatMessage(role="user", content="Test")],
config=ChatModelConfig(),
@@ -257,9 +275,11 @@ async def test_chat_streaming_complete_response():
response = await chat._send_prompt(request)
- # Verify we got the complete final response (all deltas accumulated)
- assert response == "Hello world!"
- assert chat._chat_history[-1].content == "Hello world!"
+ # Streaming returns None
+ assert response is None
+ # Verify all chunks were sent
+ assert len(sent_messages) >= 4 # text-start + 3 deltas + text-end + final
+ assert sent_messages[-1]["is_final"]
def test_chat_get_history():
@@ -332,6 +352,8 @@ def test_chat_delete_message():
def test_chat_convert_value():
+ """Test _convert_value with messages in parts format."""
+
def mock_model(
messages: list[ChatMessage], config: ChatModelConfig
) -> str:
@@ -339,19 +361,25 @@ def test_chat_convert_value():
return "Mock response"
chat = ui.chat(mock_model)
- value: dict[str, list[dict[str, str]]] = {
+ value = {
"messages": [
- {"role": "user", "content": "Hello"},
- {"role": "assistant", "content": "Hi there!"},
+ {
+ "role": "user",
+ "parts": [{"type": "text", "text": "Hello"}],
+ },
+ {
+ "role": "assistant",
+ "parts": [{"type": "text", "text": "Hi there!"}],
+ },
]
}
converted: list[ChatMessage] = chat._convert_value(value)
assert len(converted) == 2
assert converted[0].role == "user"
- assert converted[0].content == "Hello"
+ assert len(converted[0].parts) == 1
assert converted[1].role == "assistant"
- assert converted[1].content == "Hi there!"
+ assert len(converted[1].parts) == 1
def test_chat_convert_value_invalid():
@@ -368,7 +396,7 @@ def test_chat_convert_value_invalid():
async def test_chat_with_on_message():
- def mock_model(
+ async def mock_model(
messages: list[ChatMessage], config: ChatModelConfig
) -> str:
del messages, config
@@ -432,7 +460,7 @@ async def test_chat_send_message_enqueues_ui_element_request(
exec_req.get(
"""
import marimo as mo
- def f(messages, config):
+ async def f(messages, config):
return "response"
chatbot = mo.ui.chat(f)
@@ -579,12 +607,298 @@ def test_convert_value_with_list_input():
chat._convert_value(cast(Any, [{"role": "user", "content": "Hello"}]))
[email protected]("use_async", [True, False])
+async def test_vercel_messages_streaming(use_async: bool):
+ """Test custom model (sync/async) streaming with dict chunks."""
+ sent_messages: list[dict] = []
+
+ chunks = [
+ {"type": "text-start", "id": "text-1"},
+ {"type": "text-delta", "id": "text-1", "delta": "Hello"},
+ {"type": "text-delta", "id": "text-1", "delta": " world"},
+ {"type": "text-end", "id": "text-1"},
+ ]
+
+ async def async_model(
+ messages: list[ChatMessage], config: ChatModelConfig
+ ):
+ del messages, config
+ for chunk in chunks:
+ yield chunk
+
+ def sync_model(messages: list[ChatMessage], config: ChatModelConfig):
+ del messages, config
+ yield from chunks
+
+ custom_model = async_model if use_async else sync_model
+ chat = ui.chat(custom_model)
+
+ def capture_send_message(message: dict, buffers): # noqa: ARG001
+ sent_messages.append(message)
+
+ chat._send_message = capture_send_message
+
+ request = SendMessageRequest(
+ messages=[ChatMessage(role="user", content="Hello")],
+ config=ChatModelConfig(),
+ )
+ response = await chat._send_prompt(request)
+
+ assert response is None
+
+ # Extract message_id from first message for comparison
+ message_id = sent_messages[0]["message_id"]
+
+ assert sent_messages == [
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": chunks[0],
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": chunks[1],
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": chunks[2],
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": chunks[3],
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": None,
+ "is_final": True,
+ },
+ ]
+
+
+def test_serialize_vercel_ai_chunk_dict():
+ """Test ChunkSerializer with dict input."""
+ sent_chunks: list[dict] = []
+
+ def on_send_chunk(chunk: dict):
+ sent_chunks.append(chunk)
+
+ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)
+
+ # Dict input should pass through unchanged
+ chunk = {"type": "text-delta", "id": "text-1", "delta": "Hello"}
+ serializer.handle_chunk(chunk)
+ assert sent_chunks == [chunk]
+
+
+def test_serialize_plain_string():
+ """Test ChunkSerializer with plain string input."""
+ sent_chunks: list[dict] = []
+
+ def on_send_chunk(chunk: dict):
+ sent_chunks.append(chunk)
+
+ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)
+
+ # Plain strings should be wrapped in text-start/text-delta chunks
+ serializer.handle_chunk("Hello")
+ serializer.handle_chunk(" world")
+ serializer.on_end()
+
+ # Should have text-start, two text-deltas, and text-end
+ assert len(sent_chunks) == 4
+ assert sent_chunks[0]["type"] == "text-start"
+ text_id = sent_chunks[0]["id"]
+ assert sent_chunks[1] == {
+ "type": "text-delta",
+ "id": text_id,
+ "delta": "Hello",
+ }
+ assert sent_chunks[2] == {
+ "type": "text-delta",
+ "id": text_id,
+ "delta": " world",
+ }
+ assert sent_chunks[3] == {"type": "text-end", "id": text_id}
+
+
+def test_serialize_mixed_chunks():
+ """Test ChunkSerializer with mixed dict and string chunks."""
+ sent_chunks: list[dict] = []
+
+ def on_send_chunk(chunk: dict):
+ sent_chunks.append(chunk)
+
+ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)
+
+ # Send a dict chunk, then strings, then another dict
+ serializer.handle_chunk({"type": "reasoning-start", "id": "r-1"})
+ serializer.handle_chunk("text1")
+ serializer.handle_chunk("text2")
+ serializer.handle_chunk({"type": "reasoning-end", "id": "r-1"})
+ serializer.on_end()
+
+ # Dict chunks are sent as-is, strings are wrapped in text-start/delta/end
+ # text-end is only sent when on_end() is called
+ assert len(sent_chunks) == 6
+ assert sent_chunks[0] == {"type": "reasoning-start", "id": "r-1"}
+ assert sent_chunks[1]["type"] == "text-start"
+ text_id = sent_chunks[1]["id"]
+ assert sent_chunks[2] == {
+ "type": "text-delta",
+ "id": text_id,
+ "delta": "text1",
+ }
+ assert sent_chunks[3] == {
+ "type": "text-delta",
+ "id": text_id,
+ "delta": "text2",
+ }
+ # Dict chunk is sent as-is, even after strings
+ assert sent_chunks[4] == {"type": "reasoning-end", "id": "r-1"}
+ # text-end is sent when on_end() is called
+ assert sent_chunks[5] == {"type": "text-end", "id": text_id}
+
+
+def test_serialize_only_dict_chunks():
+ """Test ChunkSerializer with only dict chunks (no text)."""
+ sent_chunks: list[dict] = []
+
+ def on_send_chunk(chunk: dict):
+ sent_chunks.append(chunk)
+
+ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)
+
+ # Send only dict chunks
+ serializer.handle_chunk({"type": "reasoning-start", "id": "r-1"})
+ serializer.handle_chunk({"type": "reasoning-end", "id": "r-1"})
+ serializer.on_end()
+
+ # Should have only the dict chunks, no text-end added
+ assert sent_chunks == [
+ {"type": "reasoning-start", "id": "r-1"},
+ {"type": "reasoning-end", "id": "r-1"},
+ ]
+
+
+ not DependencyManager.pydantic_ai.has(),
+ reason="Pydantic AI is not installed",
+)
+def test_serialize_pydantic_chunk():
+ """Test ChunkSerializer with pydantic BaseChunk input."""
+ from pydantic_ai.ui.vercel_ai.response_types import TextDeltaChunk
+
+ sent_chunks: list[dict] = []
+
+ def on_send_chunk(chunk: dict):
+ sent_chunks.append(chunk)
+
+ serializer = ChunkSerializer(on_send_chunk=on_send_chunk)
+
+ # Pydantic BaseChunk should be serialized with model_dump
+ chunk = TextDeltaChunk(id="text-1", delta="Hello")
+ serializer.handle_chunk(chunk)
+
+ # Should be a dict with camelCase keys (by_alias=True)
+ assert sent_chunks == [
+ {"type": "text-delta", "id": "text-1", "delta": "Hello"}
+ ]
+
+
+ not DependencyManager.pydantic_ai.has(),
+ reason="Pydantic AI is not installed",
+)
[email protected]("use_async", [True, False])
+async def test_vercel_messages_with_pydantic_chunks(use_async: bool):
+ """Test custom model (sync/async) with pydantic-ai response_types chunks."""
+ from pydantic_ai.ui.vercel_ai.response_types import (
+ TextDeltaChunk,
+ TextEndChunk,
+ TextStartChunk,
+ )
+
+ sent_messages: list[dict] = []
+
+ chunks = [
+ TextStartChunk(id="text-1"),
+ TextDeltaChunk(id="text-1", delta="Hello"),
+ TextEndChunk(id="text-1"),
+ ]
+
+ async def async_model(
+ messages: list[ChatMessage], config: ChatModelConfig
+ ):
+ del messages, config
+ for chunk in chunks:
+ yield chunk
+
+ def sync_model(messages: list[ChatMessage], config: ChatModelConfig):
+ del messages, config
+ yield from chunks
+
+ custom_model = async_model if use_async else sync_model
+ chat = ui.chat(custom_model)
+
+ def capture_send_message(message: dict, buffers): # noqa: ARG001
+ sent_messages.append(message)
+
+ chat._send_message = capture_send_message
+
+ request = SendMessageRequest(
+ messages=[ChatMessage(role="user", content="Hello")],
+ config=ChatModelConfig(),
+ )
+ response = await chat._send_prompt(request)
+ assert response is None
+
+ message_id = sent_messages[0]["message_id"]
+ assert sent_messages == [
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": {"type": "text-start", "id": "text-1"},
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": {
+ "type": "text-delta",
+ "id": "text-1",
+ "delta": "Hello",
+ },
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": {"type": "text-end", "id": "text-1"},
+ "is_final": False,
+ },
+ {
+ "type": "stream_chunk",
+ "message_id": message_id,
+ "content": None,
+ "is_final": True,
+ },
+ ]
+
+
@pytest.mark.skipif(
not DependencyManager.pydantic_ai.has(),
reason="Pydantic AI is not installed",
)
-def test_convert_value_frontend_managed():
- """Test _convert_value in frontend-managed mode."""
+def test_convert_value_with_parts():
+ """Test _convert_value with message parts."""
from pydantic_ai.ui.vercel_ai.request_types import TextUIPart
def mock_model(
@@ -594,8 +908,6 @@ def test_convert_value_frontend_managed():
return "Mock response"
chat = ui.chat(mock_model)
- # Force frontend-managed mode
- chat._frontend_managed = True
value = {
"messages": [
@@ -629,12 +941,49 @@ def test_convert_value_frontend_managed():
]
+ not DependencyManager.pydantic_ai.has(),
+ reason="Pydantic AI is not installed",
+)
+def test_convert_value_single_message_with_parts():
+ """Test _convert_value with a single message containing parts."""
+ from pydantic_ai.ui.vercel_ai.request_types import TextUIPart
+
+ def mock_model(
+ messages: list[ChatMessage], config: ChatModelConfig
+ ) -> str:
+ del messages, config
+ return "Mock response"
+
+ chat = ui.chat(mock_model)
+
+ value = {
+ "messages": [
+ {
+ "id": "msg-1",
+ "role": "user",
+ "parts": [{"type": "text", "text": "Hello"}],
+ },
+ ]
+ }
+
+ converted = chat._convert_value(value)
+ assert converted == [
+ ChatMessage(
+ role="user",
+ id="msg-1",
+ parts=[TextUIPart(type="text", text="Hello")], # type: ignore
+ content=None,
+ )
+ ]
+
+
@pytest.mark.skipif(
not DependencyManager.pydantic_ai.has(),
reason="We use Pydantic to check for vercel parts",
)
-def test_convert_value_frontend_managed_missing_fields():
- """Test _convert_value in frontend-managed mode with missing fields."""
+def test_convert_value_with_missing_parts():
+ """Test _convert_value with message missing parts field."""
def mock_model(
messages: list[ChatMessage], config: ChatModelConfig
@@ -643,7 +992,6 @@ def test_convert_value_frontend_managed_missing_fields():
return "Mock response"
chat = ui.chat(mock_model)
- chat._frontend_managed = True
# Message without parts
value = {"messages": [{"role": "user", "id": ""}]}
@@ -654,16 +1002,15 @@ def test_convert_value_frontend_managed_missing_fields():
]
-async def test_frontend_managed_streaming_sends_serialized_chunks():
- """Test that frontend-managed streaming sends already-serialized chunks correctly."""
+async def test_streaming_with_dict_chunks():
+ """Test that streaming sends dict chunks correctly."""
- # Chunks are already serialized dicts (from pydantic_ai._serialize_vercel_ai_chunk)
+ # Chunks are dicts (like from pydantic_ai vercel response types)
chunk1 = {"type": "text-delta", "textDelta": "Hello"}
chunk2 = {"type": "text-delta", "textDelta": " world"}
- # "done" chunks are filtered out by pydantic_ai._serialize_vercel_ai_chunk
- async def mock_pydantic_ai_generator():
- """Mock async generator yielding pre-serialized dict chunks."""
+ async def mock_dict_generator():
+ """Mock async generator yielding dict chunks."""
yield chunk1
yield chunk2
@@ -674,7 +1021,6 @@ async def test_frontend_managed_streaming_sends_serialized_chunks():
return "Mock response"
chat = ui.chat(mock_model)
- chat._frontend_managed = True
sent_messages: list[dict] = []
@@ -683,11 +1029,9 @@ async def test_frontend_managed_streaming_sends_serialized_chunks():
chat._send_message = capture_send_message
- result = await chat._handle_streaming_response(
- mock_pydantic_ai_generator()
- )
+ result = await chat._handle_streaming_response(mock_dict_generator())
- # Result should be None for frontend-managed mode
+ # Result should be None
assert result is None
assert sent_messages == [
diff --git a/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py b/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py
index 2ee69a3a64..f1c530980c 100644
--- a/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py
+++ b/tests/_plugins/ui/_impl/chat/test_chat_delta_streaming.py
@@ -1,10 +1,16 @@
# Copyright 2026 Marimo. All rights reserved.
-"""Tests for delta-based streaming in chat UI."""
+"""Tests for delta-based streaming in chat UI.
+
+The streaming implementation uses the Vercel AI SDK protocol:
+- text-start: Begins a text stream with an id
+- text-delta: Contains incremental text content
+- text-end: Marks the end of a text stream
+"""
from __future__ import annotations
import asyncio
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator
@@ -13,11 +19,26 @@ from marimo._ai._types import ChatMessage
from marimo._plugins.ui._impl.chat.chat import chat
+def extract_deltas(sent_messages: list[dict[str, Any]]) -> list[str]:
+ """Extract delta text from Vercel AI SDK format messages."""
+ deltas = []
+ for msg in sent_messages:
+ content = msg.get("content")
+ if isinstance(content, dict) and content.get("type") == "text-delta":
+ deltas.append(content.get("delta", ""))
+ return deltas
+
+
+def get_accumulated_text(sent_messages: list[dict[str, Any]]) -> str:
+ """Get the accumulated text from all deltas."""
+ return "".join(extract_deltas(sent_messages))
+
+
class TestDeltaStreaming:
- """Test delta-based streaming mode."""
+ """Test delta-based streaming mode using Vercel AI SDK protocol."""
- async def test_async_delta_streaming_accumulates(self):
- """Test that async generators yielding deltas are accumulated correctly."""
+ async def test_async_delta_streaming_sends_vercel_chunks(self):
+ """Test that async generators send Vercel AI SDK format chunks."""
async def delta_generator() -> AsyncGenerator[str, None]:
"""Yields individual delta chunks."""
@@ -26,35 +47,40 @@ class TestDeltaStreaming:
yield "world"
yield "!"
- # Create chat with the delta generator
chat_ui = chat(delta_generator)
+ sent_messages: list[dict[str, Any]] = []
- # Track messages sent
- sent_messages: list[dict] = []
-
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
chat_ui._send_message = capture_send # type: ignore
- # Handle the streaming response
+ # Handle the streaming response (returns None in new implementation)
result = await chat_ui._handle_streaming_response(delta_generator())
- # Verify final result is fully accumulated
- assert result == "Hello world!"
+ # New implementation returns None (frontend manages state)
+ assert result is None
+
+ # Verify Vercel AI SDK format: text-start, text-delta(s), text-end, final
+ content_types = [
+ msg["content"]["type"]
+ for msg in sent_messages
+ if isinstance(msg.get("content"), dict)
+ ]
+ assert content_types[0] == "text-start"
+ assert all(t == "text-delta" for t in content_types[1:-1])
+ assert content_types[-1] == "text-end"
- # Verify incremental chunks were sent correctly
- assert len(sent_messages) == 5 # 4 deltas + 1 final
- assert sent_messages[0]["content"] == "Hello"
- assert sent_messages[1]["content"] == "Hello "
- assert sent_messages[2]["content"] == "Hello world"
- assert sent_messages[3]["content"] == "Hello world!"
- assert sent_messages[4]["content"] == "Hello world!"
- assert sent_messages[4]["is_final"] is True
+ # Verify accumulated text from deltas
+ accumulated = get_accumulated_text(sent_messages)
+ assert accumulated == "Hello world!"
+
+ # Final message should have is_final=True
+ assert sent_messages[-1]["is_final"] is True
- async def test_sync_delta_streaming_accumulates(self):
- """Test that sync generators yielding deltas are accumulated correctly."""
+ async def test_sync_delta_streaming_sends_vercel_chunks(self):
+ """Test that sync generators send Vercel AI SDK format chunks."""
def delta_generator() -> Generator[str, None, None]:
"""Yields individual delta chunks."""
@@ -64,32 +90,20 @@ class TestDeltaStreaming:
yield " "
yield "Three"
- # Create chat with the delta generator
chat_ui = chat(delta_generator)
+ sent_messages: list[dict[str, Any]] = []
- # Track messages sent
- sent_messages: list[dict] = []
-
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
chat_ui._send_message = capture_send # type: ignore
- # Handle the streaming response
result = await chat_ui._handle_streaming_response(delta_generator())
- # Verify final result is fully accumulated
- assert result == "One Two Three"
-
- # Verify incremental chunks were sent correctly
- assert len(sent_messages) == 6 # 5 deltas + 1 final
- assert sent_messages[0]["content"] == "One"
- assert sent_messages[1]["content"] == "One "
- assert sent_messages[2]["content"] == "One Two"
- assert sent_messages[3]["content"] == "One Two "
- assert sent_messages[4]["content"] == "One Two Three"
- assert sent_messages[5]["is_final"] is True
+ assert result is None
+ accumulated = get_accumulated_text(sent_messages)
+ assert accumulated == "One Two Three"
async def test_empty_delta_handled(self):
"""Test that empty deltas are handled gracefully."""
@@ -103,9 +117,9 @@ class TestDeltaStreaming:
yield "world"
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -113,9 +127,11 @@ class TestDeltaStreaming:
result = await chat_ui._handle_streaming_response(delta_generator())
- assert result == "Hello world"
- # Empty deltas still trigger message sends
- assert len(sent_messages) == 6 # 5 deltas (including empty) + 1 final
+ assert result is None
+ # Empty deltas are still sent as text-delta chunks
+ deltas = extract_deltas(sent_messages)
+ assert deltas == ["Hello", "", " ", "", "world"]
+ assert get_accumulated_text(sent_messages) == "Hello world"
async def test_single_delta(self):
"""Test streaming with a single delta."""
@@ -124,9 +140,9 @@ class TestDeltaStreaming:
yield "Single chunk"
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -134,10 +150,12 @@ class TestDeltaStreaming:
result = await chat_ui._handle_streaming_response(delta_generator())
- assert result == "Single chunk"
- assert len(sent_messages) == 2 # 1 delta + 1 final
- assert sent_messages[0]["content"] == "Single chunk"
- assert sent_messages[1]["is_final"] is True
+ assert result is None
+ assert get_accumulated_text(sent_messages) == "Single chunk"
+
+ # Should have: text-start, text-delta, text-end, final
+ assert len(sent_messages) == 4
+ assert sent_messages[-1]["is_final"] is True
async def test_no_deltas(self):
"""Test streaming with no deltas (empty generator)."""
@@ -148,9 +166,9 @@ class TestDeltaStreaming:
yield ""
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -158,9 +176,10 @@ class TestDeltaStreaming:
result = await chat_ui._handle_streaming_response(delta_generator())
- assert result == ""
- # No final message sent if no content
- assert len(sent_messages) == 0
+ assert result is None
+ # Should only have the final message (no text-start/end since no text)
+ assert len(sent_messages) == 1
+ assert sent_messages[0]["is_final"] is True
async def test_unicode_deltas(self):
"""Test that unicode characters in deltas are handled correctly."""
@@ -173,9 +192,9 @@ class TestDeltaStreaming:
yield "!"
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -183,8 +202,8 @@ class TestDeltaStreaming:
result = await chat_ui._handle_streaming_response(delta_generator())
- assert result == "Hello 🌍 世界!"
- assert sent_messages[-1]["content"] == "Hello 🌍 世界!"
+ assert result is None
+ assert get_accumulated_text(sent_messages) == "Hello 🌍 世界!"
async def test_message_id_consistency(self):
"""Test that all chunks in a stream share the same message_id."""
@@ -195,9 +214,9 @@ class TestDeltaStreaming:
yield "C"
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -232,9 +251,9 @@ class TestDeltaStreaming:
await asyncio.sleep(0.001) # Tiny delay
chat_ui = chat(delta_generator)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -242,16 +261,24 @@ class TestDeltaStreaming:
result = await chat_ui._handle_streaming_response(delta_generator())
+ assert result is None
expected = "This is a long streaming response with many deltas."
- assert result == expected
- assert len(sent_messages) == 11 # 10 deltas + 1 final
-
- # Verify progressive accumulation
- assert sent_messages[0]["content"] == "This"
- assert sent_messages[1]["content"] == "This is"
- assert sent_messages[2]["content"] == "This is a"
- assert sent_messages[-1]["content"] == expected
- assert sent_messages[-1]["is_final"] is True
+ assert get_accumulated_text(sent_messages) == expected
+
+ # Verify deltas are individual words
+ deltas = extract_deltas(sent_messages)
+ assert deltas == [
+ "This",
+ " is",
+ " a",
+ " long",
+ " streaming",
+ " response",
+ " with",
+ " many",
+ " deltas",
+ ".",
+ ]
class TestStreamingWithChatModels:
@@ -262,7 +289,7 @@ class TestStreamingWithChatModels:
async def custom_model(
messages: list[ChatMessage],
- config: dict, # noqa: ARG001
+ config: dict[str, Any], # noqa: ARG001
) -> AsyncGenerator[str, None]:
"""Custom model that yields deltas."""
user_msg = messages[-1].content
@@ -272,9 +299,9 @@ class TestStreamingWithChatModels:
await asyncio.sleep(0.001)
chat_ui = chat(custom_model)
- sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_send(msg: dict, **kwargs: object) -> None:
+ def capture_send(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
sent_messages.append(msg)
@@ -286,61 +313,44 @@ class TestStreamingWithChatModels:
result = await chat_ui._handle_streaming_response(generator)
- assert result == "You said: Hello"
- # Should have one chunk per character plus final
- assert len(sent_messages) > 0
+ assert result is None
+ assert get_accumulated_text(sent_messages) == "You said: Hello"
assert sent_messages[-1]["is_final"] is True
class TestStreamingEfficiency:
- """Test that delta streaming is efficient compared to accumulated streaming."""
+ """Test that delta streaming is efficient."""
- async def test_delta_vs_accumulated_bandwidth(self):
- """Demonstrate bandwidth savings of delta vs accumulated streaming."""
+ async def test_delta_streaming_sends_individual_chunks(self):
+ """Verify delta streaming sends individual chunks, not accumulated."""
# Simulate 100-word response
words = ["word" + str(i) for i in range(100)]
- # Delta streaming: each word is sent once
async def delta_stream() -> AsyncGenerator[str, None]:
for word in words:
yield word + " "
chat_ui = chat(delta_stream)
- delta_sent_messages: list[dict] = []
+ sent_messages: list[dict[str, Any]] = []
- def capture_delta(msg: dict, **kwargs: object) -> None:
+ def capture_delta(msg: dict[str, Any], **kwargs: object) -> None:
del kwargs # Unused
- delta_sent_messages.append(msg)
+ sent_messages.append(msg)
chat_ui._send_message = capture_delta # type: ignore
await chat_ui._handle_streaming_response(delta_stream())
- # Calculate bytes sent with delta streaming
- # Backend sends accumulated text, but receives deltas
- delta_bytes_received = sum(len(word + " ") for word in words)
-
- # Each message sends the accumulated content
- delta_bytes_sent = sum(
- len(msg["content"]) for msg in delta_sent_messages
- )
-
- # With old accumulated approach, model would yield:
- # "word0 ", "word0 word1 ", "word0 word1 word2 ", etc.
- accumulated_bytes_received = sum(
- len(" ".join(words[: i + 1]) + " ") for i in range(len(words))
- )
-
- # Delta mode receives much less data from the model
- assert delta_bytes_received < accumulated_bytes_received
-
- # For 100 words, delta receives ~100 words worth of data
- # while accumulated receives ~5000 words worth (1+2+3+...+100)
- efficiency_ratio = accumulated_bytes_received / delta_bytes_received
- assert efficiency_ratio > 40 # Should be ~50x more efficient
-
- print("\nEfficiency Test Results:")
- print(f"Delta bytes received: {delta_bytes_received}")
- print(f"Accumulated bytes received: {accumulated_bytes_received}")
- print(f"Efficiency ratio: {efficiency_ratio:.1f}x")
+ # Extract all deltas sent
+ deltas = extract_deltas(sent_messages)
+
+ # Each delta should be a single word (not accumulated)
+ assert len(deltas) == 100
+ assert deltas[0] == "word0 "
+ assert deltas[1] == "word1 " # NOT "word0 word1 "
+ assert deltas[99] == "word99 "
+
+ # Total accumulated text should be all words
+ accumulated = get_accumulated_text(sent_messages)
+ assert accumulated == " ".join(words) + " "