A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
AI parse from parts in the backend, enable attachments for more providers (#6256)
Shahmir Varqhaf739b6cb804e
diff --git a/frontend/src/components/chat/chat-components.tsx b/frontend/src/components/chat/chat-components.tsxindex 65af3e32b7..338fd1bcdb 100644--- a/frontend/src/components/chat/chat-components.tsx+++ b/frontend/src/components/chat/chat-components.tsx@@ -1,20 +1,20 @@/* Copyright 2024 Marimo. All rights reserved. */+import type { FileUIPart } from "ai";import { FileIcon, FileTextIcon, ImageIcon, XIcon } from "lucide-react";import { useState } from "react";-import type { ChatAttachment } from "@/core/ai/types";import { cn } from "@/utils/cn";export const AttachmentRenderer = ({attachment,}: {- attachment: ChatAttachment;+ attachment: FileUIPart;}) => {- if (attachment.contentType?.startsWith("image/")) {+ if (attachment.mediaType?.startsWith("image/")) {return (<imgsrc={attachment.url}- alt={attachment.name}+ alt={attachment.filename}className="max-h-[100px] max-w-[100px] object-contain mb-1.5"/>);@@ -23,7 +23,7 @@ export const AttachmentRenderer = ({return (<div className="flex flex-row gap-1 items-center text-xs"><FileIcon className="h-3 w-3 mt-0.5" />- {attachment.name}+ {attachment.filename}</div>);};diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsxindex a91d215375..a9288d19c4 100644--- a/frontend/src/components/chat/chat-panel.tsx+++ b/frontend/src/components/chat/chat-panel.tsx@@ -6,7 +6,6 @@ import { storePrompt } from "@marimo-team/codemirror-ai";import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";import {DefaultChatTransport,- type FileUIPart,lastAssistantMessageIsCompleteWithToolCalls,type ToolUIPart,} from "ai";@@ -47,7 +46,6 @@ import {type ChatId,chatStateAtom,} from "@/core/ai/state";-import type { ChatAttachment } from "@/core/ai/types";import { aiAtom, aiEnabledAtom } from "@/core/config/config";import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";import { FeatureFlagged } from "@/core/config/feature-flag";@@ -82,7 +80,11 @@ const DEFAULT_MODE = "manual";// We need to modify the backend to support attachments for other providers// And other types-const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>(["openai"]);+const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>([+ "openai",+ "google",+ "anthropic",+]);const SUPPORTED_ATTACHMENT_TYPES = ["image/*", "text/*"];const MAX_ATTACHMENT_SIZE = 1024 * 1024 * 50; // 50MB@@ -180,7 +182,7 @@ function isToolPart(part: UIMessage["parts"][number]): part is ToolUIPart {return part.type.startsWith("tool-");}-const ChatMessage: React.FC<ChatMessageProps> = memo(+const ChatMessageDisplay: React.FC<ChatMessageProps> = memo(({ message, index, onEdit, isStreamingReasoning, isLast }) => {const renderUserMessage = (message: UIMessage) => {const textParts = message.parts?.filter((p) => p.type === "text");@@ -267,6 +269,11 @@ const ChatMessage: React.FC<ChatMessageProps> = memo(/* handle other part types … */default:+ if (part.type.startsWith("data-")) {+ Logger.log("Found data part", part);+ return null;+ }+Logger.error("Unhandled part type:", part.type);try {return (@@ -299,7 +306,7 @@ const ChatMessage: React.FC<ChatMessageProps> = memo();},);-ChatMessage.displayName = "ChatMessage";+ChatMessageDisplay.displayName = "ChatMessage";interface ChatInputFooterProps {isEmpty: boolean;@@ -550,15 +557,6 @@ const ChatPanelBody = () => {.join("\n");}- // Map from ai/sdk files to our attachment type- function toAttachment(p: FileUIPart): ChatAttachment {- return {- contentType: p.mediaType,- name: p.filename,- url: p.url,- };- }-const input = toContent(options.messages.flatMap((m) => m.parts));const completionBody = await getAICompletionBodyWithAttachments({input,@@ -566,20 +564,15 @@ const ChatPanelBody = () => {// Map from UIMessage to our ChatMessage type// If it's the last message, add the attachments from the completion body- function mapMessage(- m: UIMessage,- isLastMessage: boolean,- ): ChatMessage & { attachments: ChatAttachment[] } {- const attachments = m.parts.flatMap((p) =>- p.type === "file" ? toAttachment(p) : [],- );+ function mapMessage(m: UIMessage, isLastMessage: boolean): ChatMessage {+ const parts = m.parts;if (isLastMessage) {- attachments.push(...completionBody.attachments);+ parts.push(...completionBody.attachments);}return {role: m.role,content: toContent(m.parts),- attachments: attachments,+ parts: parts,};}@@ -649,7 +642,7 @@ const ChatPanelBody = () => {return;}- setFiles([...files]);+ setFiles((prev) => [...(prev ?? []), ...files]);},);@@ -846,7 +839,7 @@ const ChatPanelBody = () => {)}{messages.map((message, idx) => (- <ChatMessage+ <ChatMessageDisplaykey={message.id}message={message}index={idx}diff --git a/frontend/src/components/editor/ai/completion-utils.ts b/frontend/src/components/editor/ai/completion-utils.tsindex 8c631bc6dd..8526b98731 100644--- a/frontend/src/components/editor/ai/completion-utils.ts+++ b/frontend/src/components/editor/ai/completion-utils.ts@@ -5,8 +5,8 @@ import type {CompletionContext,CompletionSource,} from "@codemirror/autocomplete";+import type { FileUIPart } from "ai";import { getAIContextRegistry } from "@/core/ai/context/context";-import type { ChatAttachment } from "@/core/ai/types";import { getCodes } from "@/core/codemirror/copilot/getCodes";import type { AiCompletionRequest } from "@/core/network/types";import { store } from "@/core/state/jotai";@@ -18,7 +18,7 @@ interface Opts {interface AICompletionBodyWithAttachments {body: Omit<AiCompletionRequest, "language" | "prompt" | "code">;- attachments: ChatAttachment[];+ attachments: FileUIPart[];}/**@@ -52,7 +52,7 @@ export async function getAICompletionBodyWithAttachments({input,}: Opts): Promise<AICompletionBodyWithAttachments> {let contextString = "";- let attachments: ChatAttachment[] = [];+ let attachments: FileUIPart[] = [];// Skip if no '@' in the inputif (input.includes("@")) {diff --git a/frontend/src/core/ai/context/__tests__/registry.test.ts b/frontend/src/core/ai/context/__tests__/registry.test.tsindex b73bffb3ce..da854a6854 100644--- a/frontend/src/core/ai/context/__tests__/registry.test.ts+++ b/frontend/src/core/ai/context/__tests__/registry.test.ts@@ -1,10 +1,10 @@/* Copyright 2024 Marimo. All rights reserved. */import type { Completion } from "@codemirror/autocomplete";+import type { FileUIPart } from "ai";import { createStore } from "jotai";import { beforeEach, describe, expect, it, vi } from "vitest";import { MockNotebook } from "@/__mocks__/notebook";-import type { ChatAttachment } from "@/core/ai/types";import { notebookAtom } from "@/core/cells/cells";import { CellId as CellIdClass } from "@/core/cells/ids";import {@@ -28,15 +28,17 @@ interface MockContextItem extends AIContextItem {}// Mock attachment data for testing-const mockAttachment1: ChatAttachment = {- name: "test-image-1.png",- contentType: "image/png",+const mockAttachment1: FileUIPart = {+ type: "file",+ filename: "test-image-1.png",+ mediaType: "image/png",url: "data:image/png;base64,mockdata1",};-const mockAttachment2: ChatAttachment = {- name: "test-image-2.jpg",- contentType: "image/jpeg",+const mockAttachment2: FileUIPart = {+ type: "file",+ filename: "test-image-2.jpg",+ mediaType: "image/jpeg",url: "data:image/jpeg;base64,mockdata2",};@@ -101,7 +103,7 @@ class AttachmentContextProvider extends AIContextProvider<MockContextItem> {constructor(private items: MockContextItem[] = [],- private attachments: ChatAttachment[] = [],+ private attachments: FileUIPart[] = [],) {super();}@@ -120,7 +122,7 @@ class AttachmentContextProvider extends AIContextProvider<MockContextItem> {override async getAttachments(items: MockContextItem[],- ): Promise<ChatAttachment[]> {+ ): Promise<FileUIPart[]> {// Return attachments for items that need themconst itemsNeedingAttachments = items.filter((item) => item.data.needsAttachment,diff --git a/frontend/src/core/ai/context/providers/cell-output.ts b/frontend/src/core/ai/context/providers/cell-output.tsindex f12b29af0f..373f943624 100644--- a/frontend/src/core/ai/context/providers/cell-output.ts+++ b/frontend/src/core/ai/context/providers/cell-output.ts@@ -1,8 +1,8 @@/* Copyright 2024 Marimo. All rights reserved. */import type { Completion } from "@codemirror/autocomplete";+import type { FileUIPart } from "ai";import { toPng } from "html-to-image";-import type { ChatAttachment } from "@/core/ai/types";import { notebookAtom } from "@/core/cells/cells";import { type CellId, CellOutputId } from "@/core/cells/ids";import { displayCellName } from "@/core/cells/names";@@ -32,7 +32,9 @@ function isMediaMimetype(mimetype: OutputMessage["mimetype"] | undefined,htmlString: string,): boolean {- if (!mimetype) return false;+ if (!mimetype) {+ return false;+ }const mediaPrefixes = ["image/", "video/", "audio/", "application/pdf"];if (mediaPrefixes.some((prefix) => mimetype.startsWith(prefix))) {@@ -72,7 +74,7 @@ function parseHtmlContent(htmlString: string): string {const textContent = tempDiv.textContent || tempDiv.innerText || "";// Clean up extra whitespace- return textContent.replace(/\s+/g, " ").trim();+ return textContent.replaceAll(/\s+/g, " ").trim();} catch (error) {Logger.error("Error parsing HTML content:", error);// If parsing fails, return the original string@@ -113,15 +115,14 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputContelet processedContent: string | undefined;let imageUrl: string | undefined;- let shouldDownloadImage: boolean = false;+ let shouldDownloadImage = false;// Process text contentif (outputType === "text" && typeof output.data === "string") {- if (mimetype === "text/html") {- processedContent = parseHtmlContent(output.data);- } else {- processedContent = output.data;- }+ processedContent =+ mimetype === "text/html"+ ? parseHtmlContent(output.data)+ : output.data;}// Process media content - for now, we'll just note that it's media@@ -303,7 +304,7 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputConte/** Get attachments for cell output items that have shouldDownloadImage=true */override async getAttachments(items: CellOutputContextItem[],- ): Promise<ChatAttachment[]> {+ ): Promise<FileUIPart[]> {// Filter items that need image downloadingconst itemsNeedingDownload = items.filter((item) =>@@ -334,8 +335,9 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputContetry {return await Promise.all(downloadRequests.map(async (item) => ({- name: `${item.cellName}-output-screenshot`,- contentType: "image/png",+ type: "file",+ filename: `${item.cellName}-output-screenshot`,+ mediaType: "image/png",url: await toPng(item.element),})),);diff --git a/frontend/src/core/ai/context/registry.ts b/frontend/src/core/ai/context/registry.tsindex 3280c5fa7b..df977bb905 100644--- a/frontend/src/core/ai/context/registry.ts+++ b/frontend/src/core/ai/context/registry.ts@@ -2,8 +2,8 @@import type { Completion } from "@codemirror/autocomplete";import type { Resource } from "@marimo-team/codemirror-mcp";+import type { FileUIPart } from "ai";import { Memoize } from "typescript-memoize";-import type { ChatAttachment } from "@/core/ai/types";import { Logger } from "@/utils/Logger";import { MultiMap } from "@/utils/multi-map";import type { TypedString } from "@/utils/typed";@@ -46,7 +46,7 @@ export abstract class AIContextProvider<abstract formatCompletion(item: T): Completion;/** Get attachments for context items (optional, async) */- async getAttachments(_items: T[]): Promise<ChatAttachment[]> {+ async getAttachments(_items: T[]): Promise<FileUIPart[]> {// Default implementation returns no attachmentsreturn [];}@@ -194,7 +194,7 @@ export class AIContextRegistry<T extends AIContextItem> {*/async getAttachmentsForContext(contextIds: ContextLocatorId[],- ): Promise<ChatAttachment[]> {+ ): Promise<FileUIPart[]> {const allItems = new Map<ContextLocatorId, T>(this.getAllItems().map((item) => [item.uri as ContextLocatorId, item]),);@@ -219,7 +219,7 @@ export class AIContextRegistry<T extends AIContextItem> {}// Collect attachments from all providers- const attachmentPromises = Array.from(itemsByProvider.entries()).map(+ const attachmentPromises = [...itemsByProvider.entries()].map(async ([providerType, items]) => {const provider = this.getProvider(providerType);if (!provider) {diff --git a/frontend/src/core/ai/state.ts b/frontend/src/core/ai/state.tsindex c478ece0b4..a60f1b444f 100644--- a/frontend/src/core/ai/state.ts+++ b/frontend/src/core/ai/state.ts@@ -1,12 +1,12 @@/* Copyright 2024 Marimo. All rights reserved. */import type { UIMessage } from "@ai-sdk/react";+import type { FileUIPart } from "ai";import { atom } from "jotai";import { atomWithStorage } from "jotai/utils";import { adaptForLocalStorage } from "@/utils/storage";import type { TypedString } from "@/utils/typed";import type { CellId } from "../cells/ids";-import type { ChatAttachment } from "./types";const KEY = "marimo:ai:chatState:v5";@@ -29,7 +29,7 @@ export interface Message {content: string;timestamp: number;parts?: UIMessage["parts"];- attachments?: ChatAttachment[];+ attachments?: FileUIPart[];}export interface Chat {diff --git a/frontend/src/core/ai/types.ts b/frontend/src/core/ai/types.tsdeleted file mode 100644index 9f4fbb78d6..0000000000--- a/frontend/src/core/ai/types.ts+++ /dev/null@@ -1,17 +0,0 @@-/* Copyright 2024 Marimo. All rights reserved. */--export interface ChatAttachment {- /**- * The name of the attachment, usually the file name.- */- name?: string;- /**- * A string indicating the [media type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).- * By default, it's extracted from the pathname's extension.- */- contentType?: string;- /**- * The URL of the attachment. It can either be a URL to a hosted file or a [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs).- */- url: string;-}diff --git a/frontend/src/plugins/impl/chat/ChatPlugin.tsx b/frontend/src/plugins/impl/chat/ChatPlugin.tsxindex 424ea7d1a8..87d24de419 100644--- a/frontend/src/plugins/impl/chat/ChatPlugin.tsx+++ b/frontend/src/plugins/impl/chat/ChatPlugin.tsx@@ -41,6 +41,7 @@ export const ChatPlugin = createPlugin<{ messages: ChatMessage[] }>(z.object({role: z.enum(["system", "user", "assistant"]),content: z.string(),+ parts: z.array(z.any()),}),),}),@@ -56,15 +57,7 @@ export const ChatPlugin = createPlugin<{ messages: ChatMessage[] }>(z.object({role: z.enum(["system", "user", "assistant"]),content: z.string(),- attachments: z- .array(- z.object({- name: z.string().optional(),- contentType: z.string().optional(),- url: z.string(),- }),- )- .optional(),+ parts: z.array(z.any()),}),),config: z.object({diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsxindex af029f30b6..bfb5b9e4c8 100644--- a/frontend/src/plugins/impl/chat/chat-ui.tsx+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx@@ -4,7 +4,7 @@ import { type UIMessage, useChat } from "@ai-sdk/react";import { ChatBubbleIcon } from "@radix-ui/react-icons";import { PopoverAnchor } from "@radix-ui/react-popover";import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";-import { DefaultChatTransport } from "ai";+import { DefaultChatTransport, type FileUIPart } from "ai";import { startCase } from "lodash-es";import {BotMessageSquareIcon,@@ -41,7 +41,6 @@ import {} from "@/components/ui/popover";import { Tooltip } from "@/components/ui/tooltip";import { toast } from "@/components/ui/use-toast";-import type { ChatAttachment } from "@/core/ai/types";import { moveToEndOfEditor } from "@/core/codemirror/utils";import { useAsyncData } from "@/hooks/useAsyncData";import { renderHTML } from "@/plugins/core/RenderHTML";@@ -77,7 +76,7 @@ export const Chatbot: React.FC<Props> = (props) => {const messages: UIMessage[] = chatMessages.messages.map((message, idx) => ({id: idx.toString(),role: message.role,- parts: [{ type: "text", text: message.content }],+ parts: message.parts ?? [],}));return messages;}, []);@@ -104,13 +103,15 @@ export const Chatbot: React.FC<Props> = (props) => {messages: UIMessage[];};try {+ const messages = body.messages.map((m) => ({+ role: m.role,+ content: m.parts+ ?.map((p) => ("text" in p ? p.text : ""))+ .join("\n"),+ parts: m.parts,+ }));const response = await props.send_prompt({- messages: body.messages.map((m) => ({- role: m.role,- content: m.parts- ?.map((p) => ("text" in p ? p.text : ""))- .join("\n"),- })),+ messages: messages,config: {max_tokens: config.max_tokens,temperature: config.temperature,@@ -120,6 +121,15 @@ export const Chatbot: React.FC<Props> = (props) => {presence_penalty: config.presence_penalty,},});+ // Update local state with AI response+ setMessages((prev) => [+ ...prev,+ {+ id: Date.now().toString(),+ role: "assistant",+ parts: [{ type: "text", text: response }],+ },+ ]);return new Response(response);// eslint-disable-next-line @typescript-eslint/no-explicit-any} catch (error: any) {@@ -155,12 +165,12 @@ export const Chatbot: React.FC<Props> = (props) => {}};- const renderAttachment = (attachment: ChatAttachment) => {- if (attachment.contentType?.startsWith("image")) {+ const renderAttachment = (attachment: FileUIPart) => {+ if (attachment.mediaType?.startsWith("image")) {return (<imgsrc={attachment.url}- alt={attachment.name || "Attachment"}+ alt={attachment.filename || "Attachment"}className="object-contain rounded-sm"width={100}height={100}@@ -175,7 +185,7 @@ export const Chatbot: React.FC<Props> = (props) => {rel="noopener noreferrer"className="text-background hover:underline">- {attachment.name || "Attachment"}+ {attachment.filename || "Attachment"}</a>);};@@ -291,17 +301,11 @@ export const Chatbot: React.FC<Props> = (props) => {<divclassName={`max-w-[80%] p-3 rounded-lg ${message.role === "user"- ? "bg-(--sky-11) text-(--slate-1)"+ ? "bg-(--sky-11) text-(--slate-1) whitespace-pre-wrap": "bg-(--slate-4) text-(--slate-12)"}`}>- <p- className={cn(- message.role === "user" && "whitespace-pre-wrap",- )}- >- {renderMessage(message)}- </p>+ {renderMessage(message)}</div><div className="flex justify-end text-xs gap-2 invisible group-hover:visible"><button@@ -353,11 +357,12 @@ export const Chatbot: React.FC<Props> = (props) => {</div><formonSubmit={async (evt) => {+ evt.preventDefault();+const fileParts = files? await convertToFileUIPart(files): undefined;- evt.preventDefault();sendMessage({role: "user",parts: [{ type: "text", text: input }, ...(fileParts ?? [])],@@ -402,7 +407,7 @@ export const Chatbot: React.FC<Props> = (props) => {{files && files.length === 1 && (<spantitle={files[0].name}- className="text-sm text-(--slate-11) truncate shrink-0 w-24"+ className="text-sm text-(--slate-11) truncate shrink-0 w-fit max-w-24">{files[0].name}</span>diff --git a/frontend/src/plugins/impl/chat/types.ts b/frontend/src/plugins/impl/chat/types.tsindex d286e06c33..2c79d50255 100644--- a/frontend/src/plugins/impl/chat/types.ts+++ b/frontend/src/plugins/impl/chat/types.ts@@ -1,10 +1,13 @@/* Copyright 2024 Marimo. All rights reserved. */+import type { UIMessage } from "ai";+export type ChatRole = "system" | "user" | "assistant";export interface ChatMessage {role: ChatRole;- content: string;+ content: string; // TODO: Deprecate content+ parts: UIMessage["parts"];}export interface SendMessageRequest {diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.pyindex 31e0518663..260675f727 100644--- a/marimo/_ai/_convert.py+++ b/marimo/_ai/_convert.py@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, castfrom marimo._ai._types import (ChatMessage,ChatPart,+ FilePart,ReasoningPart,TextPart,ToolInvocationPart,@@ -17,10 +18,21 @@ from marimo._ai._types import (from marimo._server.ai.tools.types import Toolif TYPE_CHECKING:- from anthropic.types.message_param import ( # type: ignore[import-not-found]+ from anthropic.types import ( # type: ignore[import-not-found]+ ContentBlock,+ DocumentBlockParam,+ ImageBlockParam,MessageParam,+ RedactedThinkingBlockParam,+ ServerToolUseBlockParam,+ TextBlockParam,+ ThinkingBlockParam,+ ToolResultBlockParam,+ ToolUseBlockParam,+ WebSearchToolResultBlockParam,)from google.genai.types import ( # type: ignore[import-not-found]+ BlobDict,ContentDict,ContentUnionDict,PartDict,@@ -75,171 +87,146 @@ def convert_to_openai_messages(openai_messages: list[dict[Any, Any]] = []for message in messages:- # Handle message without attachments- if not message.attachments:- if not message.parts or len(message.parts) == 0:- openai_messages.append(- {"role": message.role, "content": message.content}- )- else:- parts_messages = get_openai_messages_from_parts(- message.role, message.parts- )- openai_messages.extend(parts_messages)- continue-parts: list[dict[Any, Any]] = []if not message.parts or len(message.parts) == 0:parts.append({"type": "text", "text": message.content})else:- parts_dict = [asdict(part) for part in message.parts]- parts.extend(parts_dict)-- # Handle attachments- for attachment in message.attachments:- content_type = attachment.content_type or "text/plain"-- if content_type.startswith("image"):- parts.append(- {- "type": "image_url",- "image_url": {"url": attachment.url},- }- )- elif content_type.startswith("text"):- parts.append(- {"type": "text", "text": _extract_text(attachment.url)}- )- else:- raise ValueError(f"Unsupported content type {content_type}")+ for part in message.parts:+ if isinstance(part, FilePart):+ if part.media_type.startswith("image"):+ parts.append(+ {+ "type": "image_url",+ "image_url": {"url": part.url},+ }+ )+ elif part.media_type.startswith("text"):+ parts.append(+ {"type": "text", "text": _extract_text(part.url)}+ )+ else:+ raise ValueError(+ f"Unsupported content type {part.media_type}"+ )+ else:+ parts.append(asdict(part))openai_messages.append({"role": message.role, "content": parts})return cast("list[ChatCompletionMessageParam]", openai_messages)-def get_anthropic_messages_from_parts(- role: Literal["system", "user", "assistant"],+AnthropicParts = Union[+ "ServerToolUseBlockParam",+ "WebSearchToolResultBlockParam",+ "TextBlockParam",+ "ImageBlockParam",+ "ToolUseBlockParam",+ "ToolResultBlockParam",+ "DocumentBlockParam",+ "ThinkingBlockParam",+ "RedactedThinkingBlockParam",+ "ContentBlock",+]+++def get_anthropic_parts_from_chat_parts(parts: list[ChatPart],-) -> list[dict[Any, Any]]:- messages: list[dict[Any, Any]] = []- content_parts: list[dict[str, Any]] = []+) -> list[AnthropicParts]:+ anthropic_parts: list[AnthropicParts] = []for part in parts:if isinstance(part, TextPart):- content_parts.append({"type": "text", "text": part.text})+ text_block: TextBlockParam = {+ "type": "text",+ "text": part.text,+ }+ anthropic_parts.append(text_block)elif isinstance(part, ReasoningPart):# Handle reasoning parts with proper Anthropic thinking typesignature = ""if part.details and len(part.details) > 0:signature = part.details[0].signature or ""- content_parts.append(- {- "type": "thinking",- "thinking": part.reasoning,- "signature": signature, # This should be the encrypted signature from Claude's response+ thinking_message: ThinkingBlockParam = {+ "type": "thinking",+ "thinking": part.reasoning,+ "signature": signature,+ }+ anthropic_parts.append(thinking_message)+ elif isinstance(part, FilePart):+ media_type = part.media_type+ if media_type.strip() in [+ "image/jpeg",+ "image/png",+ "image/gif",+ "image/webp",+ ]:+ image_message: ImageBlockParam = {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": media_type, # type: ignore+ "data": _extract_data(part.url),+ },}- )+ anthropic_parts.append(image_message)+ elif media_type.startswith("text"):+ text_block_file: TextBlockParam = {+ "type": "text",+ "text": _extract_text(part.url),+ }+ anthropic_parts.append(text_block_file)+ else:+ raise ValueError(f"Unsupported content type {media_type}")elif isinstance(part, ToolInvocationPart):# Add tool use to current message content- content_parts.append(- {- "type": "tool_use",- "id": part.tool_call_id,- "name": part.tool_name,- "input": part.input,- }- )-- # Create the message with current content (including tool use)- if content_parts:- message_content = (- content_parts[0]["text"]- if len(content_parts) == 1- and content_parts[0]["type"] == "text"- else content_parts- )- messages.append({"role": role, "content": message_content})- content_parts = [] # Reset for next message+ tool_use_message: ToolUseBlockParam = {+ "type": "tool_use",+ "id": part.tool_call_id,+ "name": part.tool_name,+ "input": part.input,+ }+ anthropic_parts.append(tool_use_message)# Create separate tool result message- tool_result_message = {- "role": "user",+ tool_result_message: ToolResultBlockParam = {+ "tool_use_id": part.tool_call_id,+ "type": "tool_result","content": [{- "type": "tool_result",- "tool_use_id": part.tool_call_id,- "content": str(part.output),+ "type": "text",+ "text": str(part.output),}],}- messages.append(tool_result_message)-- # Add remaining content parts as a single message if any- if content_parts:- # If only one text part, use string format; otherwise use array format- if len(content_parts) == 1 and content_parts[0]["type"] == "text":- message_content = content_parts[0]["text"]- else:- message_content = content_parts-- messages.append({"role": role, "content": message_content})+ anthropic_parts.append(tool_result_message)- return messages+ return anthropic_partsdef convert_to_anthropic_messages(messages: list[ChatMessage],) -> list[MessageParam]:- anthropic_messages: list[dict[Any, Any]] = []+ anthropic_messages: list[MessageParam] = []for message in messages:- if not message.attachments:- if not message.parts or len(message.parts) == 0:- anthropic_messages.append(- {"role": message.role, "content": message.content}- )- else:- parts_messages = get_anthropic_messages_from_parts(- message.role, message.parts- )- anthropic_messages.extend(parts_messages)- continue-- # Handle attachments- parts: list[dict[Any, Any]] = []- if not message.parts or len(message.parts) == 0:- parts.append({"type": "text", "text": message.content})+ parts: list[AnthropicParts] = []+ if not message.parts:+ # Convert content to string+ text_block: TextBlockParam = {+ "type": "text",+ "text": str(message.content),+ }+ parts.append(text_block)else:- parts.extend(- get_anthropic_messages_from_parts(message.role, message.parts)- )- for attachment in message.attachments:- content_type = attachment.content_type or "text/plain"- if content_type.startswith("image"):- parts.append(- {- "type": "image",- "source": {- "type": "base64",- "media_type": content_type,- "data": _extract_data(attachment.url),- },- }- )-- elif content_type.startswith("text"):- parts.append(- {"type": "text", "text": _extract_text(attachment.url)}- )-- else:- raise ValueError(f"Unsupported content type {content_type}")+ parts.extend(get_anthropic_parts_from_chat_parts(message.parts))- anthropic_messages.append({"role": message.role, "content": parts})-- return cast("list[MessageParam]", anthropic_messages)+ anthropic_role = (+ "assistant" if message.role == "system" else message.role+ )+ anthropic_messages.append({"role": anthropic_role, "content": parts})+ return anthropic_messagesdef convert_to_groq_messages(@@ -248,15 +235,21 @@ def convert_to_groq_messages(groq_messages: list[dict[Any, Any]] = []for message in messages:- # Currently only supports text content (Llava is deprecated now)- # See here - https://console.groq.com/docs/deprecations- if message.attachments:+ if message.parts:+ # Currently only supports text content (Llava is deprecated now)+ # See here - https://console.groq.com/docs/deprecations+ file_parts = [+ part for part in message.parts if isinstance(part, FilePart)+ ]# Convert attachments to text if possibletext_content = str(message.content) # Explicitly convert to string- for attachment in message.attachments:- content_type = attachment.content_type or "text/plain"- if content_type.startswith("text"):- text_content += "\n" + _extract_text(attachment.url)+ for file in file_parts:+ if file.media_type.startswith("text"):+ text_content += "\n" + _extract_text(file.url)+ else:+ raise ValueError(+ f"Unsupported content type {file.media_type}. Only text content is supported."+ )groq_messages.append({"role": message.role, "content": text_content}@@ -296,6 +289,20 @@ def get_google_messages_from_parts("parts": [{"text": part.reasoning, "thought": True}],}messages.append(reasoning_message)+ elif isinstance(part, FilePart):+ media_type = part.media_type+ if not media_type.startswith(("image", "text")):+ raise ValueError(f"Unsupported content type {media_type}")+ inline_data: BlobDict = {+ "mime_type": media_type,+ "data": base64.b64decode(_extract_data(part.url)),+ }+ messages.append(+ {+ "role": "user" if role == "user" else "model",+ "parts": [{"inline_data": inline_data}],+ }+ )elif isinstance(part, ToolInvocationPart):# Create function call message for Googlefunction_call_message: ContentDict = {@@ -334,23 +341,6 @@ def convert_to_google_messages(google_messages: list[ContentUnionDict] = []for message in messages:- # Handle message without attachments- if not message.attachments:- if not message.parts or len(message.parts) == 0:- google_messages.append(- {- "role": "user" if message.role == "user" else "model",- "parts": [{"text": str(message.content)}],- }- )- else:- parts_messages = get_google_messages_from_parts(- message.role, message.parts- )- google_messages.extend(parts_messages)- continue-- # Handle attachmentsparts: list[PartDict] = []if not message.parts or len(message.parts) == 0:parts.append({"text": str(message.content)})@@ -362,34 +352,6 @@ def convert_to_google_messages(if "parts" in parts_message:parts.extend(parts_message["parts"] or [])- for attachment in message.attachments:- content_type = attachment.content_type or "text/plain"-- if content_type.startswith("image"):- parts.append(- {- "inline_data": {- "mime_type": content_type,- "data": base64.b64decode(- _extract_data(attachment.url)- ),- },- }- )- elif content_type.startswith("text"):- parts.append(- {- "inline_data": {- "mime_type": content_type,- "data": base64.b64decode(- _extract_data(attachment.url)- ),- },- }- )- else:- raise ValueError(f"Unsupported content type {content_type}")-google_messages.append({"role": "user" if message.role == "user" else "model",diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.pyindex f0750e490c..a7e207bcb9 100644--- a/marimo/_ai/_types.py+++ b/marimo/_ai/_types.py@@ -20,6 +20,13 @@ class TextPartDict(TypedDict):text: str+class FilePartDict(TypedDict):+ type: Literal["file"]+ media_type: str+ filename: Optional[str]+ url: str++class ReasoningPartDict(TypedDict):type: Literal["reasoning"]reasoning: str@@ -40,7 +47,9 @@ class ToolInvocationPartDict(TypedDict):output: Optional[Any]-ChatPartDict = Union[TextPartDict, ReasoningPartDict, ToolInvocationPartDict]+ChatPartDict = Union[+ TextPartDict, ReasoningPartDict, ToolInvocationPartDict, FilePartDict+]class ChatMessageDict(TypedDict):@@ -123,8 +132,10 @@ class ToolInvocationPart:@dataclassclass FilePart:+ """Represents a FileUIPart from the AI SDK."""+type: Literal["file"]- media_type: Optional[str]+ media_type: strfilename: Optional[str]url: str@@ -147,9 +158,11 @@ class ChatMessage(msgspec.Struct):content: Any# Optional attachments to the message.+ # TODO: Deprecate in favour of partsattachments: Optional[list[ChatAttachment]] = None- # Optional parts from AI SDK. (see types above)+ # Parts from AI SDK. (see types above)+ # TODO: Make this requiredparts: Optional[list[ChatPart]] = Nonediff --git a/marimo/_plugins/ui/_impl/chat/utils.py b/marimo/_plugins/ui/_impl/chat/utils.pyindex a42d5212a4..b889698d64 100644--- a/marimo/_plugins/ui/_impl/chat/utils.py+++ b/marimo/_plugins/ui/_impl/chat/utils.py@@ -4,10 +4,11 @@ from __future__ import annotationsfrom typing import Optional, castfrom marimo._ai._types import (- ChatAttachment,ChatMessage,ChatMessageDict,ChatPart,+ FilePart,+ FilePartDict,ReasoningDetails,ReasoningPart,ReasoningPartDict,@@ -22,20 +23,6 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:if isinstance(d, ChatMessage):return d- attachments_dict = d.get("attachments", None)- attachments: Optional[list[ChatAttachment]] = None- if attachments_dict is not None:- attachments = [- ChatAttachment(- name=attachment["name"] or "attachment",- content_type=attachment["content_type"],- url=attachment["url"],- )- for attachment in attachments_dict- ]- else:- attachments = None-# Handle partsparts_dict = d.get("parts", None)parts: Optional[list[ChatPart]] = None@@ -45,6 +32,16 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:if part_dict["type"] == "text":part_dict = cast(TextPartDict, part_dict)parts.append(TextPart(type="text", text=part_dict["text"]))+ elif part_dict["type"] == "file":+ part_dict = cast(FilePartDict, part_dict)+ parts.append(+ FilePart(+ type="file",+ media_type=part_dict["media_type"],+ filename=part_dict["filename"],+ url=part_dict["url"],+ )+ )elif part_dict["type"] == "reasoning":part_dict = cast(ReasoningPartDict, part_dict)# Handle ReasoningDetails if present@@ -96,9 +93,4 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:))- return ChatMessage(- role=d["role"],- content=d["content"],- attachments=attachments,- parts=parts,- )+ return ChatMessage(role=d["role"], content=d["content"], parts=parts)diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.pyindex c4e8c2802e..8680b43538 100644--- a/tests/_ai/llm/test_impl.py+++ b/tests/_ai/llm/test_impl.py@@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patchimport pytest-from marimo._ai._types import ChatMessage, ChatModelConfig+from marimo._ai._types import ChatMessage, ChatModelConfig, TextPartfrom marimo._ai.llm._impl import (DEFAULT_SYSTEM_MESSAGE,anthropic,@@ -274,9 +274,13 @@ class TestOpenAI:assert call_args["model"] == "gpt-4"assert len(call_args["messages"]) == 2assert call_args["messages"][0]["role"] == "system"- assert call_args["messages"][0]["content"] == DEFAULT_SYSTEM_MESSAGE+ assert call_args["messages"][0]["content"] == [+ {"type": "text", "text": DEFAULT_SYSTEM_MESSAGE}+ ]assert call_args["messages"][1]["role"] == "user"- assert call_args["messages"][1]["content"] == "Test prompt"+ assert call_args["messages"][1]["content"] == [+ {"type": "text", "text": "Test prompt"}+ ]assert call_args["max_completion_tokens"] == 100# Use pytest.approx for floating point comparisonsassert call_args["temperature"] == pytest.approx(0.7)@@ -436,10 +440,26 @@ class TestOpenAI:# Test with multiple messagesmessages = [- ChatMessage(role="system", content="Custom system"),- ChatMessage(role="user", content="Hello"),- ChatMessage(role="assistant", content="Hi there"),- ChatMessage(role="user", content="How are you?"),+ ChatMessage(+ role="system",+ content="Custom system",+ parts=[TextPart(type="text", text="Custom system")],+ ),+ ChatMessage(+ role="user",+ content="Hello",+ parts=[TextPart(type="text", text="Hello")],+ ),+ ChatMessage(+ role="assistant",+ content="Hi there",+ parts=[TextPart(type="text", text="Hi there")],+ ),+ ChatMessage(+ role="user",+ content="How are you?",+ parts=[TextPart(type="text", text="How are you?")],+ ),]model(messages, ChatModelConfig())@@ -447,18 +467,31 @@ class TestOpenAI:# Verify the messages were properly convertedcall_args = mock_client.chat.completions.create.call_args[1]assert len(call_args["messages"]) == 5 # system + all messages- assert call_args["messages"][0]["role"] == "system"- assert (- call_args["messages"][0]["content"] == DEFAULT_SYSTEM_MESSAGE- )- assert call_args["messages"][1]["role"] == "system"- assert call_args["messages"][1]["content"] == "Custom system"- assert call_args["messages"][2]["role"] == "user"- assert call_args["messages"][2]["content"] == "Hello"- assert call_args["messages"][3]["role"] == "assistant"- assert call_args["messages"][3]["content"] == "Hi there"- assert call_args["messages"][4]["role"] == "user"- assert call_args["messages"][4]["content"] == "How are you?"++ assert call_args["messages"] == [+ {+ "role": "system",+ "content": [+ {"type": "text", "text": DEFAULT_SYSTEM_MESSAGE}+ ],+ },+ {+ "role": "system",+ "content": [{"type": "text", "text": "Custom system"}],+ },+ {+ "role": "user",+ "content": [{"type": "text", "text": "Hello"}],+ },+ {+ "role": "assistant",+ "content": [{"type": "text", "text": "Hi there"}],+ },+ {+ "role": "user",+ "content": [{"type": "text", "text": "How are you?"}],+ },+ ]@pytest.mark.skipif(diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.pyindex f1e0a35ad3..c5ceaeea0f 100644--- a/tests/_ai/test_chat_convert.py+++ b/tests/_ai/test_chat_convert.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport base64import json+from copy import deepcopyimport pytest@@ -14,13 +15,13 @@ from marimo._ai._convert import (convert_to_groq_messages,convert_to_openai_messages,convert_to_openai_tools,- get_anthropic_messages_from_parts,+ get_anthropic_parts_from_chat_parts,get_google_messages_from_parts,get_openai_messages_from_parts,)from marimo._ai._types import (- ChatAttachment,ChatMessage,+ FilePart,ReasoningDetails,ReasoningPart,TextPart,@@ -36,34 +37,35 @@ def sample_messages() -> list[ChatMessage]:ChatMessage(role="user",content="Hello, I have a question.",- attachments=[- ChatAttachment(- name="image.png",- content_type="image/png",+ parts=[+ TextPart(type="text", text="Hello, I have a question."),+ FilePart(+ type="file",+ media_type="image/png",+ filename="image.png",url=f"data:image/png;base64,{base64.b64encode(b'hello')}",),- ChatAttachment(- name="text.txt",- content_type="text/plain",- url="data:text/csv;base64,QQoxCjIKMwo=",- ),+ TextPart(type="text", text="A\n1\n2\n3\n"),],),ChatMessage(role="assistant",content="Sure, I'd be happy to help. What's your question?",- attachments=[],+ parts=[+ TextPart(+ type="text",+ text="Sure, I'd be happy to help. What's your question?",+ ),+ ],),]-def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):+def test_convert_to_openai_messages(+ sample_messages: list[ChatMessage],+):result = convert_to_openai_messages(sample_messages)-- assert len(result) == 2-assert result == [- # User message{"role": "user","content": [@@ -75,26 +77,23 @@ def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):{"type": "text", "text": "A\n1\n2\n3\n"},],},- # Assistant message{"role": "assistant",- "content": "Sure, I'd be happy to help. What's your question?",+ "content": [+ {+ "type": "text",+ "text": "Sure, I'd be happy to help. What's your question?",+ }+ ],},]-def test_convert_to_openai_messages_with_parts_and_attachments():+def test_convert_to_openai_messages_with_parts():sample_messages = [ChatMessage(role="user",content="Message with parts and attachments",- attachments=[- ChatAttachment(- name="image.png",- content_type="image/png",- url=f"data:image/png;base64,{base64.b64encode(b'hello')}",- ),- ],parts=[TextPart(type="text", text="Message with parts and attachments"@@ -115,6 +114,12 @@ def test_convert_to_openai_messages_with_parts_and_attachments():input={"expression": "2 + 2"},output={"answer": 4},),+ FilePart(+ type="file",+ media_type="image/png",+ filename="image.png",+ url=f"data:image/png;base64,{base64.b64encode(b'hello')}",+ ),],),]@@ -147,84 +152,91 @@ def test_convert_to_openai_messages_with_parts_and_attachments():]-def test_convert_to_anthropic_messages(sample_messages: list[ChatMessage]):+def test_convert_to_anthropic_messages(+ sample_messages: list[ChatMessage],+):result = convert_to_anthropic_messages(sample_messages)-- assert len(result) == 2-- # Check user message- assert result[0]["role"] == "user"- assert len(result[0]["content"]) == 3- assert result[0]["content"][0] == {- "type": "text",- "text": "Hello, I have a question.",- }- assert result[0]["content"][1] == {- "type": "image",- "source": {- "data": "b'aGVsbG8='",- "media_type": "image/png",- "type": "base64",+ assert result == [+ # User message+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Hello, I have a question."},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "b'aGVsbG8='",+ },+ },+ {"type": "text", "text": "A\n1\n2\n3\n"},+ ],},- }- assert result[0]["content"][2] == {- "type": "text",- "text": "A\n1\n2\n3\n",- }-- # Check assistant message- assert result[1]["role"] == "assistant"- assert (- result[1]["content"]- == "Sure, I'd be happy to help. What's your question?"- )+ # Assistant message+ {+ "role": "assistant",+ "content": [+ {+ "type": "text",+ "text": "Sure, I'd be happy to help. What's your question?",+ }+ ],+ },+ ]-def test_convert_to_google_messages(sample_messages: list[ChatMessage]):+def test_convert_to_google_messages(+ sample_messages: list[ChatMessage],+):result = convert_to_google_messages(sample_messages)-- assert len(result) == 2-- # Check user message- assert result[0]["role"] == "user"- assert result[0]["parts"] == [- {"text": "Hello, I have a question."},+ assert result == [+ # User message{- "inline_data": {- "data": b"m\xa1\x95\xb1\xb1\xbc",- "mime_type": "image/png",- }+ "role": "user",+ "parts": [+ {"text": "Hello, I have a question."},+ {+ "inline_data": {+ "mime_type": "image/png",+ "data": b"m\xa1\x95\xb1\xb1\xbc",+ }+ },+ {"text": "A\n1\n2\n3\n"},+ ],},+ # Assistant message{- "inline_data": {- "data": b"A\n1\n2\n3\n",- "mime_type": "text/plain",- }+ "role": "model",+ "parts": [+ {"text": "Sure, I'd be happy to help. What's your question?"}+ ],},]- # Check assistant message- assert result[1]["role"] == "model"- assert result[1]["parts"] == [- {"text": "Sure, I'd be happy to help. What's your question?"}- ]--def test_convert_to_groq_messages(sample_messages: list[ChatMessage]):- result = convert_to_groq_messages(sample_messages)-- assert len(result) == 2-- # Check user message with text attachment- assert result[0]["role"] == "user"- assert result[0]["content"] == "Hello, I have a question.\nA\n1\n2\n3\n"-- # Check assistant message- assert result[1]["role"] == "assistant"- assert (- result[1]["content"]- == "Sure, I'd be happy to help. What's your question?"+def test_convert_to_groq_messages(+ sample_messages: list[ChatMessage],+):+ new_messages = deepcopy(sample_messages)+ # replace image file part with text file part+ new_messages[0].parts[1] = FilePart(+ filename="text.txt",+ type="file",+ media_type="text/plain",+ url="data:text/plain;base64,R29vZGJ5ZQ==",)+ result = convert_to_groq_messages(new_messages)++ assert result == [+ # User message+ {"role": "user", "content": "Hello, I have a question.\nGoodbye"},+ # Assistant message+ {+ "role": "assistant",+ "content": "Sure, I'd be happy to help. What's your question?",+ },+ ]def test_empty_messages():@@ -238,29 +250,41 @@ def test_empty_messages():def test_message_without_attachments():messages = [ChatMessage(- role="user", content="Just a simple message", attachments=[]+ role="user",+ content="Just a simple message",+ attachments=[],+ parts=[TextPart(type="text", text="Just a simple message")],)]openai_result = convert_to_openai_messages(messages)- assert len(openai_result) == 1- assert openai_result[0]["role"] == "user"- assert openai_result[0]["content"] == "Just a simple message"+ assert openai_result == [+ {+ "role": "user",+ "content": [{"type": "text", "text": "Just a simple message"}],+ }+ ]anthropic_result = convert_to_anthropic_messages(messages)- assert len(anthropic_result) == 1- assert anthropic_result[0]["role"] == "user"- assert anthropic_result[0]["content"] == "Just a simple message"+ assert anthropic_result == [+ {+ "role": "user",+ "content": [{"type": "text", "text": "Just a simple message"}],+ }+ ]google_result = convert_to_google_messages(messages)- assert len(google_result) == 1- assert google_result[0]["role"] == "user"- assert google_result[0]["parts"] == [{"text": "Just a simple message"}]+ assert google_result == [+ {+ "role": "user",+ "parts": [{"text": "Just a simple message"}],+ }+ ]groq_result = convert_to_groq_messages(messages)- assert len(groq_result) == 1- assert groq_result[0]["role"] == "user"- assert groq_result[0]["content"] == "Just a simple message"+ assert groq_result == [+ {"role": "user", "content": "Just a simple message"}+ ]def test_from_chat_message_dict():@@ -268,25 +292,30 @@ def test_from_chat_message_dict():message_dict = {"role": "user","content": "Hello, this is a test message.",- "attachments": [+ "parts": [{- "name": "test.png",- "content_type": "image/png",+ "type": "file",+ "media_type": "image/png",+ "filename": "test.png","url": "http://example.com/test.png",}],}result = from_chat_message_dict(message_dict)-- assert isinstance(result, ChatMessage)- assert result.role == "user"- assert result.content == "Hello, this is a test message."- assert len(result.attachments) == 1- assert isinstance(result.attachments[0], ChatAttachment)- assert result.attachments[0].name == "test.png"- assert result.attachments[0].content_type == "image/png"- assert result.attachments[0].url == "http://example.com/test.png"+ assert result == ChatMessage(+ role="user",+ content="Hello, this is a test message.",+ attachments=None,+ parts=[+ FilePart(+ type="file",+ media_type="image/png",+ filename="test.png",+ url="http://example.com/test.png",+ )+ ],+ )# Test case 2: ChatMessage without attachmentsmessage_dict_no_attachments = {@@ -327,32 +356,7 @@ def test_from_chat_message_dict():result_with_parts.parts[1].reasoning == "This is my reasoning process")- # Test case 4: ChatMessage with both attachments and parts- message_dict_full = {- "role": "user",- "content": "Complex message with everything.",- "attachments": [- {- "name": "doc.pdf",- "content_type": "application/pdf",- "url": "http://example.com/doc.pdf",- }- ],- "parts": [{"type": "text", "text": "Additional text content"}],- }-- result_full = from_chat_message_dict(message_dict_full)-- assert isinstance(result_full, ChatMessage)- assert result_full.role == "user"- assert result_full.content == "Complex message with everything."- assert len(result_full.attachments) == 1- assert result_full.attachments[0].name == "doc.pdf"- assert len(result_full.parts) == 1- assert result_full.parts[0].type == "text"- assert result_full.parts[0].text == "Additional text content"-- # Test case 5: ChatMessage with tool invocation part (result state only)+ # Test case 4: ChatMessage with tool invocation part (result state only)message_dict_tool_result = {"role": "assistant","content": "Here's the tool result.",@@ -383,7 +387,7 @@ def test_from_chat_message_dict():assert result_tool_result.parts[0].tool_name == "weather_tool"assert result_tool_result.parts[0].input == {"location": "New York"}- # Test case 6: Existing ChatMessage input (should return as-is)+ # Test case 5: Existing ChatMessage input (should return as-is)existing_message = ChatMessage(role="user",content="I'm already a ChatMessage object",@@ -628,24 +632,21 @@ def test_get_openai_messages_from_parts_empty():assert result == []-def test_get_anthropic_messages_from_parts_text_only():+def test_get_anthropic_parts_from_chat_parts_text_only():"""Test converting TextPart to Anthropic format."""parts = [TextPart(type="text", text="Hello"),TextPart(type="text", text="World"),]- result = get_anthropic_messages_from_parts("user", parts)-- assert len(result) == 1- assert result[0]["role"] == "user"- assert result[0]["content"] == [+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [{"type": "text", "text": "Hello"},{"type": "text", "text": "World"},]-def test_get_anthropic_messages_from_parts_with_reasoning():+def test_get_anthropic_parts_from_chat_parts_with_reasoning():"""Test converting ReasoningPart to Anthropic thinking format."""reasoning_details = [ReasoningDetails(@@ -662,11 +663,8 @@ def test_get_anthropic_messages_from_parts_with_reasoning():),]- result = get_anthropic_messages_from_parts("assistant", parts)-- assert len(result) == 1- assert result[0]["role"] == "assistant"- assert result[0]["content"] == [+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [{"type": "text", "text": "Let me think about this"},{"type": "thinking",@@ -676,7 +674,7 @@ def test_get_anthropic_messages_from_parts_with_reasoning():]-def test_get_anthropic_messages_from_parts_reasoning_no_signature():+def test_get_anthropic_parts_from_chat_parts_reasoning_no_signature():"""Test converting ReasoningPart without signature to Anthropic format."""reasoning_details = [ReasoningDetails(type="text", text="Step 1", signature=None)@@ -690,20 +688,17 @@ def test_get_anthropic_messages_from_parts_reasoning_no_signature():),]- result = get_anthropic_messages_from_parts("assistant", parts)-- assert len(result) == 1- assert result[0]["role"] == "assistant"- assert result[0]["content"] == [+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [{"type": "thinking","thinking": "Basic reasoning","signature": "", # Should be empty string when no signature- },+ }]-def test_get_anthropic_messages_from_parts_reasoning_empty_details():+def test_get_anthropic_parts_from_empty_reasoning_details_chat_part():"""Test converting ReasoningPart with empty details list."""parts = [ReasoningPart(@@ -713,20 +708,17 @@ def test_get_anthropic_messages_from_parts_reasoning_empty_details():),]- result = get_anthropic_messages_from_parts("assistant", parts)-- assert len(result) == 1- assert result[0]["role"] == "assistant"- assert result[0]["content"] == [+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [{"type": "thinking","thinking": "Some reasoning","signature": "", # Should be empty when no details- },+ }]-def test_get_anthropic_messages_from_parts_with_tool_invocation():+def test_get_anthropic_parts_from_chat_parts_with_tool_invocation():"""Test converting ToolInvocationPart to Anthropic format."""parts = [TextPart(type="text", text="I'll search for you"),@@ -739,47 +731,37 @@ def test_get_anthropic_messages_from_parts_with_tool_invocation():),]- result = get_anthropic_messages_from_parts("assistant", parts)-- assert (- len(result) == 2- ) # One message with tool use, one tool result message-- # Check first message with tool use- assert result[0]["role"] == "assistant"- expected_content = [+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [+ # Text message{"type": "text", "text": "I'll search for you"},+ # Tool use message{"type": "tool_use","id": "call_123","name": "search_tool","input": {"query": "Python tutorials"},},- ]- assert result[0]["content"] == expected_content-- # Check tool result message- assert result[1]["role"] == "user"- assert result[1]["content"] == [+ # Tool result message{- "type": "tool_result","tool_use_id": "call_123",- "content": str({"results": ["tutorial1", "tutorial2"]}),- }+ "type": "tool_result",+ "content": [+ {+ "type": "text",+ "text": "{'results': ['tutorial1', 'tutorial2']}",+ }+ ],+ },]-def test_get_anthropic_messages_from_parts_single_text():- """Test converting single TextPart uses string format instead of array."""+def test_get_anthropic_parts_from_chat_parts_single_text():+ """Test converting single text part."""parts = [TextPart(type="text", text="Single message")]- result = get_anthropic_messages_from_parts("user", parts)-- assert len(result) == 1- assert result[0]["role"] == "user"- assert (- result[0]["content"] == "Single message"- ) # Should be string, not array+ result = get_anthropic_parts_from_chat_parts(parts)+ assert result == [{"type": "text", "text": "Single message"}]def test_get_google_messages_from_parts_text_only():diff --git a/tests/_server/api/endpoints/test_ai.py b/tests/_server/api/endpoints/test_ai.pyindex e50ee642ca..6caee24f5d 100644--- a/tests/_server/api/endpoints/test_ai.py+++ b/tests/_server/api/endpoints/test_ai.py@@ -139,7 +139,7 @@ class TestOpenAiEndpoints:# Assert the prompt it was called withprompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][1]["content"]+ ][1]["content"][0]["text"]assert prompt == ("Help me create a dataframe")# Assert the model it was called withmodel = oaiclient.chat.completions.create.call_args.kwargs["model"]@@ -184,7 +184,7 @@ class TestOpenAiEndpoints:# Assert the prompt it was called withprompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][1]["content"]+ ][1]["content"][0]["text"]assert prompt == "Help me create a dataframe"@staticmethod@@ -305,7 +305,7 @@ class TestOpenAiEndpoints:# Assert the prompt it was called withprompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][1]["content"]+ ][1]["content"][0]["text"]assert (prompt== f"{FIM_PREFIX_TAG}import pandas as pd\n{FIM_SUFFIX_TAG}\ndf.head(){FIM_MIDDLE_TAG}"@@ -313,7 +313,7 @@ class TestOpenAiEndpoints:# Assert the system prompt for FIM modelssystem_prompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][0]["content"]+ ][0]["content"][0]["text"]assert (system_prompt== f"You are a python code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."@@ -385,7 +385,7 @@ class TestOpenAiEndpoints:# Assert the system prompt for FIM modelssystem_prompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][0]["content"]+ ][0]["content"][0]["text"]assert (system_prompt== f"You are a sql code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."@@ -462,7 +462,7 @@ class TestAnthropicAiEndpoints:# Assert the prompt it was called withprompt: str = anthropic_client.messages.create.call_args.kwargs["messages"- ][0]["content"]+ ][0]["content"][0]["text"]assert prompt == "Help me create a dataframe"# Assert the model it was called withmodel = anthropic_client.messages.create.call_args.kwargs["model"]@@ -503,7 +503,7 @@ class TestAnthropicAiEndpoints:# Assert the prompt it was called withprompt: str = anthropic_client.messages.create.call_args.kwargs["messages"- ][0]["content"]+ ][0]["content"][0]["text"]assert (prompt== f"{FIM_PREFIX_TAG}import pandas as pd\n{FIM_SUFFIX_TAG}\ndf.head(){FIM_MIDDLE_TAG}"@@ -781,7 +781,7 @@ def test_chat_without_code(client: TestClient) -> None:# Assert the prompt it was called withprompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][1]["content"]+ ][1]["content"][0]["text"]assert prompt == "Hello"@@ -827,7 +827,7 @@ def test_chat_with_code(client: TestClient) -> None:# Assert the prompt it was called withprompt = oaiclient.chat.completions.create.call_args.kwargs["messages"- ][1]["content"]+ ][1]["content"][0]["text"]assert prompt == "Help me create a dataframe"