Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

AI parse from parts in the backend, enable attachments for more providers (#6256)

Shahmir Varqhaf739b6cb804e
diff --git a/frontend/src/components/chat/chat-components.tsx b/frontend/src/components/chat/chat-components.tsx
index 65af3e32b7..338fd1bcdb 100644
--- a/frontend/src/components/chat/chat-components.tsx
+++ b/frontend/src/components/chat/chat-components.tsx
@@ -1,20 +1,20 @@
/* Copyright 2024 Marimo. All rights reserved. */
+import type { FileUIPart } from "ai";
import { FileIcon, FileTextIcon, ImageIcon, XIcon } from "lucide-react";
import { useState } from "react";
-import type { ChatAttachment } from "@/core/ai/types";
import { cn } from "@/utils/cn";
export const AttachmentRenderer = ({
attachment,
}: {
- attachment: ChatAttachment;
+ attachment: FileUIPart;
}) => {
- if (attachment.contentType?.startsWith("image/")) {
+ if (attachment.mediaType?.startsWith("image/")) {
return (
<img
src={attachment.url}
- alt={attachment.name}
+ alt={attachment.filename}
className="max-h-[100px] max-w-[100px] object-contain mb-1.5"
/>
);
@@ -23,7 +23,7 @@ export const AttachmentRenderer = ({
return (
<div className="flex flex-row gap-1 items-center text-xs">
<FileIcon className="h-3 w-3 mt-0.5" />
- {attachment.name}
+ {attachment.filename}
</div>
);
};
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsx
index a91d215375..a9288d19c4 100644
--- a/frontend/src/components/chat/chat-panel.tsx
+++ b/frontend/src/components/chat/chat-panel.tsx
@@ -6,7 +6,6 @@ import { storePrompt } from "@marimo-team/codemirror-ai";
import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";
import {
DefaultChatTransport,
- type FileUIPart,
lastAssistantMessageIsCompleteWithToolCalls,
type ToolUIPart,
} from "ai";
@@ -47,7 +46,6 @@ import {
type ChatId,
chatStateAtom,
} from "@/core/ai/state";
-import type { ChatAttachment } from "@/core/ai/types";
import { aiAtom, aiEnabledAtom } from "@/core/config/config";
import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";
import { FeatureFlagged } from "@/core/config/feature-flag";
@@ -82,7 +80,11 @@ const DEFAULT_MODE = "manual";
// We need to modify the backend to support attachments for other providers
// And other types
-const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>(["openai"]);
+const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>([
+ "openai",
+ "google",
+ "anthropic",
+]);
const SUPPORTED_ATTACHMENT_TYPES = ["image/*", "text/*"];
const MAX_ATTACHMENT_SIZE = 1024 * 1024 * 50; // 50MB
@@ -180,7 +182,7 @@ function isToolPart(part: UIMessage["parts"][number]): part is ToolUIPart {
return part.type.startsWith("tool-");
}
-const ChatMessage: React.FC<ChatMessageProps> = memo(
+const ChatMessageDisplay: React.FC<ChatMessageProps> = memo(
({ message, index, onEdit, isStreamingReasoning, isLast }) => {
const renderUserMessage = (message: UIMessage) => {
const textParts = message.parts?.filter((p) => p.type === "text");
@@ -267,6 +269,11 @@ const ChatMessage: React.FC<ChatMessageProps> = memo(
/* handle other part types … */
default:
+ if (part.type.startsWith("data-")) {
+ Logger.log("Found data part", part);
+ return null;
+ }
+
Logger.error("Unhandled part type:", part.type);
try {
return (
@@ -299,7 +306,7 @@ const ChatMessage: React.FC<ChatMessageProps> = memo(
);
},
);
-ChatMessage.displayName = "ChatMessage";
+ChatMessageDisplay.displayName = "ChatMessage";
interface ChatInputFooterProps {
isEmpty: boolean;
@@ -550,15 +557,6 @@ const ChatPanelBody = () => {
.join("\n");
}
- // Map from ai/sdk files to our attachment type
- function toAttachment(p: FileUIPart): ChatAttachment {
- return {
- contentType: p.mediaType,
- name: p.filename,
- url: p.url,
- };
- }
-
const input = toContent(options.messages.flatMap((m) => m.parts));
const completionBody = await getAICompletionBodyWithAttachments({
input,
@@ -566,20 +564,15 @@ const ChatPanelBody = () => {
// Map from UIMessage to our ChatMessage type
// If it's the last message, add the attachments from the completion body
- function mapMessage(
- m: UIMessage,
- isLastMessage: boolean,
- ): ChatMessage & { attachments: ChatAttachment[] } {
- const attachments = m.parts.flatMap((p) =>
- p.type === "file" ? toAttachment(p) : [],
- );
+ function mapMessage(m: UIMessage, isLastMessage: boolean): ChatMessage {
+ const parts = m.parts;
if (isLastMessage) {
- attachments.push(...completionBody.attachments);
+ parts.push(...completionBody.attachments);
}
return {
role: m.role,
content: toContent(m.parts),
- attachments: attachments,
+ parts: parts,
};
}
@@ -649,7 +642,7 @@ const ChatPanelBody = () => {
return;
}
- setFiles([...files]);
+ setFiles((prev) => [...(prev ?? []), ...files]);
},
);
@@ -846,7 +839,7 @@ const ChatPanelBody = () => {
)}
{messages.map((message, idx) => (
- <ChatMessage
+ <ChatMessageDisplay
key={message.id}
message={message}
index={idx}
diff --git a/frontend/src/components/editor/ai/completion-utils.ts b/frontend/src/components/editor/ai/completion-utils.ts
index 8c631bc6dd..8526b98731 100644
--- a/frontend/src/components/editor/ai/completion-utils.ts
+++ b/frontend/src/components/editor/ai/completion-utils.ts
@@ -5,8 +5,8 @@ import type {
CompletionContext,
CompletionSource,
} from "@codemirror/autocomplete";
+import type { FileUIPart } from "ai";
import { getAIContextRegistry } from "@/core/ai/context/context";
-import type { ChatAttachment } from "@/core/ai/types";
import { getCodes } from "@/core/codemirror/copilot/getCodes";
import type { AiCompletionRequest } from "@/core/network/types";
import { store } from "@/core/state/jotai";
@@ -18,7 +18,7 @@ interface Opts {
interface AICompletionBodyWithAttachments {
body: Omit<AiCompletionRequest, "language" | "prompt" | "code">;
- attachments: ChatAttachment[];
+ attachments: FileUIPart[];
}
/**
@@ -52,7 +52,7 @@ export async function getAICompletionBodyWithAttachments({
input,
}: Opts): Promise<AICompletionBodyWithAttachments> {
let contextString = "";
- let attachments: ChatAttachment[] = [];
+ let attachments: FileUIPart[] = [];
// Skip if no '@' in the input
if (input.includes("@")) {
diff --git a/frontend/src/core/ai/context/__tests__/registry.test.ts b/frontend/src/core/ai/context/__tests__/registry.test.ts
index b73bffb3ce..da854a6854 100644
--- a/frontend/src/core/ai/context/__tests__/registry.test.ts
+++ b/frontend/src/core/ai/context/__tests__/registry.test.ts
@@ -1,10 +1,10 @@
/* Copyright 2024 Marimo. All rights reserved. */
import type { Completion } from "@codemirror/autocomplete";
+import type { FileUIPart } from "ai";
import { createStore } from "jotai";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { MockNotebook } from "@/__mocks__/notebook";
-import type { ChatAttachment } from "@/core/ai/types";
import { notebookAtom } from "@/core/cells/cells";
import { CellId as CellIdClass } from "@/core/cells/ids";
import {
@@ -28,15 +28,17 @@ interface MockContextItem extends AIContextItem {
}
// Mock attachment data for testing
-const mockAttachment1: ChatAttachment = {
- name: "test-image-1.png",
- contentType: "image/png",
+const mockAttachment1: FileUIPart = {
+ type: "file",
+ filename: "test-image-1.png",
+ mediaType: "image/png",
url: "data:image/png;base64,mockdata1",
};
-const mockAttachment2: ChatAttachment = {
- name: "test-image-2.jpg",
- contentType: "image/jpeg",
+const mockAttachment2: FileUIPart = {
+ type: "file",
+ filename: "test-image-2.jpg",
+ mediaType: "image/jpeg",
url: "data:image/jpeg;base64,mockdata2",
};
@@ -101,7 +103,7 @@ class AttachmentContextProvider extends AIContextProvider<MockContextItem> {
constructor(
private items: MockContextItem[] = [],
- private attachments: ChatAttachment[] = [],
+ private attachments: FileUIPart[] = [],
) {
super();
}
@@ -120,7 +122,7 @@ class AttachmentContextProvider extends AIContextProvider<MockContextItem> {
override async getAttachments(
items: MockContextItem[],
- ): Promise<ChatAttachment[]> {
+ ): Promise<FileUIPart[]> {
// Return attachments for items that need them
const itemsNeedingAttachments = items.filter(
(item) => item.data.needsAttachment,
diff --git a/frontend/src/core/ai/context/providers/cell-output.ts b/frontend/src/core/ai/context/providers/cell-output.ts
index f12b29af0f..373f943624 100644
--- a/frontend/src/core/ai/context/providers/cell-output.ts
+++ b/frontend/src/core/ai/context/providers/cell-output.ts
@@ -1,8 +1,8 @@
/* Copyright 2024 Marimo. All rights reserved. */
import type { Completion } from "@codemirror/autocomplete";
+import type { FileUIPart } from "ai";
import { toPng } from "html-to-image";
-import type { ChatAttachment } from "@/core/ai/types";
import { notebookAtom } from "@/core/cells/cells";
import { type CellId, CellOutputId } from "@/core/cells/ids";
import { displayCellName } from "@/core/cells/names";
@@ -32,7 +32,9 @@ function isMediaMimetype(
mimetype: OutputMessage["mimetype"] | undefined,
htmlString: string,
): boolean {
- if (!mimetype) return false;
+ if (!mimetype) {
+ return false;
+ }
const mediaPrefixes = ["image/", "video/", "audio/", "application/pdf"];
if (mediaPrefixes.some((prefix) => mimetype.startsWith(prefix))) {
@@ -72,7 +74,7 @@ function parseHtmlContent(htmlString: string): string {
const textContent = tempDiv.textContent || tempDiv.innerText || "";
// Clean up extra whitespace
- return textContent.replace(/\s+/g, " ").trim();
+ return textContent.replaceAll(/\s+/g, " ").trim();
} catch (error) {
Logger.error("Error parsing HTML content:", error);
// If parsing fails, return the original string
@@ -113,15 +115,14 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputConte
let processedContent: string | undefined;
let imageUrl: string | undefined;
- let shouldDownloadImage: boolean = false;
+ let shouldDownloadImage = false;
// Process text content
if (outputType === "text" && typeof output.data === "string") {
- if (mimetype === "text/html") {
- processedContent = parseHtmlContent(output.data);
- } else {
- processedContent = output.data;
- }
+ processedContent =
+ mimetype === "text/html"
+ ? parseHtmlContent(output.data)
+ : output.data;
}
// Process media content - for now, we'll just note that it's media
@@ -303,7 +304,7 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputConte
/** Get attachments for cell output items that have shouldDownloadImage=true */
override async getAttachments(
items: CellOutputContextItem[],
- ): Promise<ChatAttachment[]> {
+ ): Promise<FileUIPart[]> {
// Filter items that need image downloading
const itemsNeedingDownload = items.filter(
(item) =>
@@ -334,8 +335,9 @@ export class CellOutputContextProvider extends AIContextProvider<CellOutputConte
try {
return await Promise.all(
downloadRequests.map(async (item) => ({
- name: `${item.cellName}-output-screenshot`,
- contentType: "image/png",
+ type: "file",
+ filename: `${item.cellName}-output-screenshot`,
+ mediaType: "image/png",
url: await toPng(item.element),
})),
);
diff --git a/frontend/src/core/ai/context/registry.ts b/frontend/src/core/ai/context/registry.ts
index 3280c5fa7b..df977bb905 100644
--- a/frontend/src/core/ai/context/registry.ts
+++ b/frontend/src/core/ai/context/registry.ts
@@ -2,8 +2,8 @@
import type { Completion } from "@codemirror/autocomplete";
import type { Resource } from "@marimo-team/codemirror-mcp";
+import type { FileUIPart } from "ai";
import { Memoize } from "typescript-memoize";
-import type { ChatAttachment } from "@/core/ai/types";
import { Logger } from "@/utils/Logger";
import { MultiMap } from "@/utils/multi-map";
import type { TypedString } from "@/utils/typed";
@@ -46,7 +46,7 @@ export abstract class AIContextProvider<
abstract formatCompletion(item: T): Completion;
/** Get attachments for context items (optional, async) */
- async getAttachments(_items: T[]): Promise<ChatAttachment[]> {
+ async getAttachments(_items: T[]): Promise<FileUIPart[]> {
// Default implementation returns no attachments
return [];
}
@@ -194,7 +194,7 @@ export class AIContextRegistry<T extends AIContextItem> {
*/
async getAttachmentsForContext(
contextIds: ContextLocatorId[],
- ): Promise<ChatAttachment[]> {
+ ): Promise<FileUIPart[]> {
const allItems = new Map<ContextLocatorId, T>(
this.getAllItems().map((item) => [item.uri as ContextLocatorId, item]),
);
@@ -219,7 +219,7 @@ export class AIContextRegistry<T extends AIContextItem> {
}
// Collect attachments from all providers
- const attachmentPromises = Array.from(itemsByProvider.entries()).map(
+ const attachmentPromises = [...itemsByProvider.entries()].map(
async ([providerType, items]) => {
const provider = this.getProvider(providerType);
if (!provider) {
diff --git a/frontend/src/core/ai/state.ts b/frontend/src/core/ai/state.ts
index c478ece0b4..a60f1b444f 100644
--- a/frontend/src/core/ai/state.ts
+++ b/frontend/src/core/ai/state.ts
@@ -1,12 +1,12 @@
/* Copyright 2024 Marimo. All rights reserved. */
import type { UIMessage } from "@ai-sdk/react";
+import type { FileUIPart } from "ai";
import { atom } from "jotai";
import { atomWithStorage } from "jotai/utils";
import { adaptForLocalStorage } from "@/utils/storage";
import type { TypedString } from "@/utils/typed";
import type { CellId } from "../cells/ids";
-import type { ChatAttachment } from "./types";
const KEY = "marimo:ai:chatState:v5";
@@ -29,7 +29,7 @@ export interface Message {
content: string;
timestamp: number;
parts?: UIMessage["parts"];
- attachments?: ChatAttachment[];
+ attachments?: FileUIPart[];
}
export interface Chat {
diff --git a/frontend/src/core/ai/types.ts b/frontend/src/core/ai/types.ts
deleted file mode 100644
index 9f4fbb78d6..0000000000
--- a/frontend/src/core/ai/types.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-/* Copyright 2024 Marimo. All rights reserved. */
-
-export interface ChatAttachment {
- /**
- * The name of the attachment, usually the file name.
- */
- name?: string;
- /**
- * A string indicating the [media type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
- * By default, it's extracted from the pathname's extension.
- */
- contentType?: string;
- /**
- * The URL of the attachment. It can either be a URL to a hosted file or a [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs).
- */
- url: string;
-}
diff --git a/frontend/src/plugins/impl/chat/ChatPlugin.tsx b/frontend/src/plugins/impl/chat/ChatPlugin.tsx
index 424ea7d1a8..87d24de419 100644
--- a/frontend/src/plugins/impl/chat/ChatPlugin.tsx
+++ b/frontend/src/plugins/impl/chat/ChatPlugin.tsx
@@ -41,6 +41,7 @@ export const ChatPlugin = createPlugin<{ messages: ChatMessage[] }>(
z.object({
role: z.enum(["system", "user", "assistant"]),
content: z.string(),
+ parts: z.array(z.any()),
}),
),
}),
@@ -56,15 +57,7 @@ export const ChatPlugin = createPlugin<{ messages: ChatMessage[] }>(
z.object({
role: z.enum(["system", "user", "assistant"]),
content: z.string(),
- attachments: z
- .array(
- z.object({
- name: z.string().optional(),
- contentType: z.string().optional(),
- url: z.string(),
- }),
- )
- .optional(),
+ parts: z.array(z.any()),
}),
),
config: z.object({
diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsx
index af029f30b6..bfb5b9e4c8 100644
--- a/frontend/src/plugins/impl/chat/chat-ui.tsx
+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx
@@ -4,7 +4,7 @@ import { type UIMessage, useChat } from "@ai-sdk/react";
import { ChatBubbleIcon } from "@radix-ui/react-icons";
import { PopoverAnchor } from "@radix-ui/react-popover";
import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";
-import { DefaultChatTransport } from "ai";
+import { DefaultChatTransport, type FileUIPart } from "ai";
import { startCase } from "lodash-es";
import {
BotMessageSquareIcon,
@@ -41,7 +41,6 @@ import {
} from "@/components/ui/popover";
import { Tooltip } from "@/components/ui/tooltip";
import { toast } from "@/components/ui/use-toast";
-import type { ChatAttachment } from "@/core/ai/types";
import { moveToEndOfEditor } from "@/core/codemirror/utils";
import { useAsyncData } from "@/hooks/useAsyncData";
import { renderHTML } from "@/plugins/core/RenderHTML";
@@ -77,7 +76,7 @@ export const Chatbot: React.FC<Props> = (props) => {
const messages: UIMessage[] = chatMessages.messages.map((message, idx) => ({
id: idx.toString(),
role: message.role,
- parts: [{ type: "text", text: message.content }],
+ parts: message.parts ?? [],
}));
return messages;
}, []);
@@ -104,13 +103,15 @@ export const Chatbot: React.FC<Props> = (props) => {
messages: UIMessage[];
};
try {
+ const messages = body.messages.map((m) => ({
+ role: m.role,
+ content: m.parts
+ ?.map((p) => ("text" in p ? p.text : ""))
+ .join("\n"),
+ parts: m.parts,
+ }));
const response = await props.send_prompt({
- messages: body.messages.map((m) => ({
- role: m.role,
- content: m.parts
- ?.map((p) => ("text" in p ? p.text : ""))
- .join("\n"),
- })),
+ messages: messages,
config: {
max_tokens: config.max_tokens,
temperature: config.temperature,
@@ -120,6 +121,15 @@ export const Chatbot: React.FC<Props> = (props) => {
presence_penalty: config.presence_penalty,
},
});
+ // Update local state with AI response
+ setMessages((prev) => [
+ ...prev,
+ {
+ id: Date.now().toString(),
+ role: "assistant",
+ parts: [{ type: "text", text: response }],
+ },
+ ]);
return new Response(response);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} catch (error: any) {
@@ -155,12 +165,12 @@ export const Chatbot: React.FC<Props> = (props) => {
}
};
- const renderAttachment = (attachment: ChatAttachment) => {
- if (attachment.contentType?.startsWith("image")) {
+ const renderAttachment = (attachment: FileUIPart) => {
+ if (attachment.mediaType?.startsWith("image")) {
return (
<img
src={attachment.url}
- alt={attachment.name || "Attachment"}
+ alt={attachment.filename || "Attachment"}
className="object-contain rounded-sm"
width={100}
height={100}
@@ -175,7 +185,7 @@ export const Chatbot: React.FC<Props> = (props) => {
rel="noopener noreferrer"
className="text-background hover:underline"
>
- {attachment.name || "Attachment"}
+ {attachment.filename || "Attachment"}
</a>
);
};
@@ -291,17 +301,11 @@ export const Chatbot: React.FC<Props> = (props) => {
<div
className={`max-w-[80%] p-3 rounded-lg ${
message.role === "user"
- ? "bg-(--sky-11) text-(--slate-1)"
+ ? "bg-(--sky-11) text-(--slate-1) whitespace-pre-wrap"
: "bg-(--slate-4) text-(--slate-12)"
}`}
>
- <p
- className={cn(
- message.role === "user" && "whitespace-pre-wrap",
- )}
- >
- {renderMessage(message)}
- </p>
+ {renderMessage(message)}
</div>
<div className="flex justify-end text-xs gap-2 invisible group-hover:visible">
<button
@@ -353,11 +357,12 @@ export const Chatbot: React.FC<Props> = (props) => {
</div>
<form
onSubmit={async (evt) => {
+ evt.preventDefault();
+
const fileParts = files
? await convertToFileUIPart(files)
: undefined;
- evt.preventDefault();
sendMessage({
role: "user",
parts: [{ type: "text", text: input }, ...(fileParts ?? [])],
@@ -402,7 +407,7 @@ export const Chatbot: React.FC<Props> = (props) => {
{files && files.length === 1 && (
<span
title={files[0].name}
- className="text-sm text-(--slate-11) truncate shrink-0 w-24"
+ className="text-sm text-(--slate-11) truncate shrink-0 w-fit max-w-24"
>
{files[0].name}
</span>
diff --git a/frontend/src/plugins/impl/chat/types.ts b/frontend/src/plugins/impl/chat/types.ts
index d286e06c33..2c79d50255 100644
--- a/frontend/src/plugins/impl/chat/types.ts
+++ b/frontend/src/plugins/impl/chat/types.ts
@@ -1,10 +1,13 @@
/* Copyright 2024 Marimo. All rights reserved. */
+import type { UIMessage } from "ai";
+
export type ChatRole = "system" | "user" | "assistant";
export interface ChatMessage {
role: ChatRole;
- content: string;
+ content: string; // TODO: Deprecate content
+ parts: UIMessage["parts"];
}
export interface SendMessageRequest {
diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.py
index 31e0518663..260675f727 100644
--- a/marimo/_ai/_convert.py
+++ b/marimo/_ai/_convert.py
@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
from marimo._ai._types import (
ChatMessage,
ChatPart,
+ FilePart,
ReasoningPart,
TextPart,
ToolInvocationPart,
@@ -17,10 +18,21 @@ from marimo._ai._types import (
from marimo._server.ai.tools.types import Tool
if TYPE_CHECKING:
- from anthropic.types.message_param import ( # type: ignore[import-not-found]
+ from anthropic.types import ( # type: ignore[import-not-found]
+ ContentBlock,
+ DocumentBlockParam,
+ ImageBlockParam,
MessageParam,
+ RedactedThinkingBlockParam,
+ ServerToolUseBlockParam,
+ TextBlockParam,
+ ThinkingBlockParam,
+ ToolResultBlockParam,
+ ToolUseBlockParam,
+ WebSearchToolResultBlockParam,
)
from google.genai.types import ( # type: ignore[import-not-found]
+ BlobDict,
ContentDict,
ContentUnionDict,
PartDict,
@@ -75,171 +87,146 @@ def convert_to_openai_messages(
openai_messages: list[dict[Any, Any]] = []
for message in messages:
- # Handle message without attachments
- if not message.attachments:
- if not message.parts or len(message.parts) == 0:
- openai_messages.append(
- {"role": message.role, "content": message.content}
- )
- else:
- parts_messages = get_openai_messages_from_parts(
- message.role, message.parts
- )
- openai_messages.extend(parts_messages)
- continue
-
parts: list[dict[Any, Any]] = []
if not message.parts or len(message.parts) == 0:
parts.append({"type": "text", "text": message.content})
else:
- parts_dict = [asdict(part) for part in message.parts]
- parts.extend(parts_dict)
-
- # Handle attachments
- for attachment in message.attachments:
- content_type = attachment.content_type or "text/plain"
-
- if content_type.startswith("image"):
- parts.append(
- {
- "type": "image_url",
- "image_url": {"url": attachment.url},
- }
- )
- elif content_type.startswith("text"):
- parts.append(
- {"type": "text", "text": _extract_text(attachment.url)}
- )
- else:
- raise ValueError(f"Unsupported content type {content_type}")
+ for part in message.parts:
+ if isinstance(part, FilePart):
+ if part.media_type.startswith("image"):
+ parts.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": part.url},
+ }
+ )
+ elif part.media_type.startswith("text"):
+ parts.append(
+ {"type": "text", "text": _extract_text(part.url)}
+ )
+ else:
+ raise ValueError(
+ f"Unsupported content type {part.media_type}"
+ )
+ else:
+ parts.append(asdict(part))
openai_messages.append({"role": message.role, "content": parts})
return cast("list[ChatCompletionMessageParam]", openai_messages)
-def get_anthropic_messages_from_parts(
- role: Literal["system", "user", "assistant"],
+AnthropicParts = Union[
+ "ServerToolUseBlockParam",
+ "WebSearchToolResultBlockParam",
+ "TextBlockParam",
+ "ImageBlockParam",
+ "ToolUseBlockParam",
+ "ToolResultBlockParam",
+ "DocumentBlockParam",
+ "ThinkingBlockParam",
+ "RedactedThinkingBlockParam",
+ "ContentBlock",
+]
+
+
+def get_anthropic_parts_from_chat_parts(
parts: list[ChatPart],
-) -> list[dict[Any, Any]]:
- messages: list[dict[Any, Any]] = []
- content_parts: list[dict[str, Any]] = []
+) -> list[AnthropicParts]:
+ anthropic_parts: list[AnthropicParts] = []
for part in parts:
if isinstance(part, TextPart):
- content_parts.append({"type": "text", "text": part.text})
+ text_block: TextBlockParam = {
+ "type": "text",
+ "text": part.text,
+ }
+ anthropic_parts.append(text_block)
elif isinstance(part, ReasoningPart):
# Handle reasoning parts with proper Anthropic thinking type
signature = ""
if part.details and len(part.details) > 0:
signature = part.details[0].signature or ""
- content_parts.append(
- {
- "type": "thinking",
- "thinking": part.reasoning,
- "signature": signature, # This should be the encrypted signature from Claude's response
+ thinking_message: ThinkingBlockParam = {
+ "type": "thinking",
+ "thinking": part.reasoning,
+ "signature": signature,
+ }
+ anthropic_parts.append(thinking_message)
+ elif isinstance(part, FilePart):
+ media_type = part.media_type
+ if media_type.strip() in [
+ "image/jpeg",
+ "image/png",
+ "image/gif",
+ "image/webp",
+ ]:
+ image_message: ImageBlockParam = {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": media_type, # type: ignore
+ "data": _extract_data(part.url),
+ },
}
- )
+ anthropic_parts.append(image_message)
+ elif media_type.startswith("text"):
+ text_block_file: TextBlockParam = {
+ "type": "text",
+ "text": _extract_text(part.url),
+ }
+ anthropic_parts.append(text_block_file)
+ else:
+ raise ValueError(f"Unsupported content type {media_type}")
elif isinstance(part, ToolInvocationPart):
# Add tool use to current message content
- content_parts.append(
- {
- "type": "tool_use",
- "id": part.tool_call_id,
- "name": part.tool_name,
- "input": part.input,
- }
- )
-
- # Create the message with current content (including tool use)
- if content_parts:
- message_content = (
- content_parts[0]["text"]
- if len(content_parts) == 1
- and content_parts[0]["type"] == "text"
- else content_parts
- )
- messages.append({"role": role, "content": message_content})
- content_parts = [] # Reset for next message
+ tool_use_message: ToolUseBlockParam = {
+ "type": "tool_use",
+ "id": part.tool_call_id,
+ "name": part.tool_name,
+ "input": part.input,
+ }
+ anthropic_parts.append(tool_use_message)
# Create separate tool result message
- tool_result_message = {
- "role": "user",
+ tool_result_message: ToolResultBlockParam = {
+ "tool_use_id": part.tool_call_id,
+ "type": "tool_result",
"content": [
{
- "type": "tool_result",
- "tool_use_id": part.tool_call_id,
- "content": str(part.output),
+ "type": "text",
+ "text": str(part.output),
}
],
}
- messages.append(tool_result_message)
-
- # Add remaining content parts as a single message if any
- if content_parts:
- # If only one text part, use string format; otherwise use array format
- if len(content_parts) == 1 and content_parts[0]["type"] == "text":
- message_content = content_parts[0]["text"]
- else:
- message_content = content_parts
-
- messages.append({"role": role, "content": message_content})
+ anthropic_parts.append(tool_result_message)
- return messages
+ return anthropic_parts
def convert_to_anthropic_messages(
messages: list[ChatMessage],
) -> list[MessageParam]:
- anthropic_messages: list[dict[Any, Any]] = []
+ anthropic_messages: list[MessageParam] = []
for message in messages:
- if not message.attachments:
- if not message.parts or len(message.parts) == 0:
- anthropic_messages.append(
- {"role": message.role, "content": message.content}
- )
- else:
- parts_messages = get_anthropic_messages_from_parts(
- message.role, message.parts
- )
- anthropic_messages.extend(parts_messages)
- continue
-
- # Handle attachments
- parts: list[dict[Any, Any]] = []
- if not message.parts or len(message.parts) == 0:
- parts.append({"type": "text", "text": message.content})
+ parts: list[AnthropicParts] = []
+ if not message.parts:
+ # Convert content to string
+ text_block: TextBlockParam = {
+ "type": "text",
+ "text": str(message.content),
+ }
+ parts.append(text_block)
else:
- parts.extend(
- get_anthropic_messages_from_parts(message.role, message.parts)
- )
- for attachment in message.attachments:
- content_type = attachment.content_type or "text/plain"
- if content_type.startswith("image"):
- parts.append(
- {
- "type": "image",
- "source": {
- "type": "base64",
- "media_type": content_type,
- "data": _extract_data(attachment.url),
- },
- }
- )
-
- elif content_type.startswith("text"):
- parts.append(
- {"type": "text", "text": _extract_text(attachment.url)}
- )
-
- else:
- raise ValueError(f"Unsupported content type {content_type}")
+ parts.extend(get_anthropic_parts_from_chat_parts(message.parts))
- anthropic_messages.append({"role": message.role, "content": parts})
-
- return cast("list[MessageParam]", anthropic_messages)
+ anthropic_role = (
+ "assistant" if message.role == "system" else message.role
+ )
+ anthropic_messages.append({"role": anthropic_role, "content": parts})
+ return anthropic_messages
def convert_to_groq_messages(
@@ -248,15 +235,21 @@ def convert_to_groq_messages(
groq_messages: list[dict[Any, Any]] = []
for message in messages:
- # Currently only supports text content (Llava is deprecated now)
- # See here - https://console.groq.com/docs/deprecations
- if message.attachments:
+ if message.parts:
+ # Currently only supports text content (Llava is deprecated now)
+ # See here - https://console.groq.com/docs/deprecations
+ file_parts = [
+ part for part in message.parts if isinstance(part, FilePart)
+ ]
# Convert attachments to text if possible
text_content = str(message.content) # Explicitly convert to string
- for attachment in message.attachments:
- content_type = attachment.content_type or "text/plain"
- if content_type.startswith("text"):
- text_content += "\n" + _extract_text(attachment.url)
+ for file in file_parts:
+ if file.media_type.startswith("text"):
+ text_content += "\n" + _extract_text(file.url)
+ else:
+ raise ValueError(
+ f"Unsupported content type {file.media_type}. Only text content is supported."
+ )
groq_messages.append(
{"role": message.role, "content": text_content}
@@ -296,6 +289,20 @@ def get_google_messages_from_parts(
"parts": [{"text": part.reasoning, "thought": True}],
}
messages.append(reasoning_message)
+ elif isinstance(part, FilePart):
+ media_type = part.media_type
+ if not media_type.startswith(("image", "text")):
+ raise ValueError(f"Unsupported content type {media_type}")
+ inline_data: BlobDict = {
+ "mime_type": media_type,
+ "data": base64.b64decode(_extract_data(part.url)),
+ }
+ messages.append(
+ {
+ "role": "user" if role == "user" else "model",
+ "parts": [{"inline_data": inline_data}],
+ }
+ )
elif isinstance(part, ToolInvocationPart):
# Create function call message for Google
function_call_message: ContentDict = {
@@ -334,23 +341,6 @@ def convert_to_google_messages(
google_messages: list[ContentUnionDict] = []
for message in messages:
- # Handle message without attachments
- if not message.attachments:
- if not message.parts or len(message.parts) == 0:
- google_messages.append(
- {
- "role": "user" if message.role == "user" else "model",
- "parts": [{"text": str(message.content)}],
- }
- )
- else:
- parts_messages = get_google_messages_from_parts(
- message.role, message.parts
- )
- google_messages.extend(parts_messages)
- continue
-
- # Handle attachments
parts: list[PartDict] = []
if not message.parts or len(message.parts) == 0:
parts.append({"text": str(message.content)})
@@ -362,34 +352,6 @@ def convert_to_google_messages(
if "parts" in parts_message:
parts.extend(parts_message["parts"] or [])
- for attachment in message.attachments:
- content_type = attachment.content_type or "text/plain"
-
- if content_type.startswith("image"):
- parts.append(
- {
- "inline_data": {
- "mime_type": content_type,
- "data": base64.b64decode(
- _extract_data(attachment.url)
- ),
- },
- }
- )
- elif content_type.startswith("text"):
- parts.append(
- {
- "inline_data": {
- "mime_type": content_type,
- "data": base64.b64decode(
- _extract_data(attachment.url)
- ),
- },
- }
- )
- else:
- raise ValueError(f"Unsupported content type {content_type}")
-
google_messages.append(
{
"role": "user" if message.role == "user" else "model",
diff --git a/marimo/_ai/_types.py b/marimo/_ai/_types.py
index f0750e490c..a7e207bcb9 100644
--- a/marimo/_ai/_types.py
+++ b/marimo/_ai/_types.py
@@ -20,6 +20,13 @@ class TextPartDict(TypedDict):
text: str
+class FilePartDict(TypedDict):
+ type: Literal["file"]
+ media_type: str
+ filename: Optional[str]
+ url: str
+
+
class ReasoningPartDict(TypedDict):
type: Literal["reasoning"]
reasoning: str
@@ -40,7 +47,9 @@ class ToolInvocationPartDict(TypedDict):
output: Optional[Any]
-ChatPartDict = Union[TextPartDict, ReasoningPartDict, ToolInvocationPartDict]
+ChatPartDict = Union[
+ TextPartDict, ReasoningPartDict, ToolInvocationPartDict, FilePartDict
+]
class ChatMessageDict(TypedDict):
@@ -123,8 +132,10 @@ class ToolInvocationPart:
@dataclass
class FilePart:
+ """Represents a FileUIPart from the AI SDK."""
+
type: Literal["file"]
- media_type: Optional[str]
+ media_type: str
filename: Optional[str]
url: str
@@ -147,9 +158,11 @@ class ChatMessage(msgspec.Struct):
content: Any
# Optional attachments to the message.
+ # TODO: Deprecate in favour of parts
attachments: Optional[list[ChatAttachment]] = None
- # Optional parts from AI SDK. (see types above)
+ # Parts from AI SDK. (see types above)
+ # TODO: Make this required
parts: Optional[list[ChatPart]] = None
diff --git a/marimo/_plugins/ui/_impl/chat/utils.py b/marimo/_plugins/ui/_impl/chat/utils.py
index a42d5212a4..b889698d64 100644
--- a/marimo/_plugins/ui/_impl/chat/utils.py
+++ b/marimo/_plugins/ui/_impl/chat/utils.py
@@ -4,10 +4,11 @@ from __future__ import annotations
from typing import Optional, cast
from marimo._ai._types import (
- ChatAttachment,
ChatMessage,
ChatMessageDict,
ChatPart,
+ FilePart,
+ FilePartDict,
ReasoningDetails,
ReasoningPart,
ReasoningPartDict,
@@ -22,20 +23,6 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:
if isinstance(d, ChatMessage):
return d
- attachments_dict = d.get("attachments", None)
- attachments: Optional[list[ChatAttachment]] = None
- if attachments_dict is not None:
- attachments = [
- ChatAttachment(
- name=attachment["name"] or "attachment",
- content_type=attachment["content_type"],
- url=attachment["url"],
- )
- for attachment in attachments_dict
- ]
- else:
- attachments = None
-
# Handle parts
parts_dict = d.get("parts", None)
parts: Optional[list[ChatPart]] = None
@@ -45,6 +32,16 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:
if part_dict["type"] == "text":
part_dict = cast(TextPartDict, part_dict)
parts.append(TextPart(type="text", text=part_dict["text"]))
+ elif part_dict["type"] == "file":
+ part_dict = cast(FilePartDict, part_dict)
+ parts.append(
+ FilePart(
+ type="file",
+ media_type=part_dict["media_type"],
+ filename=part_dict["filename"],
+ url=part_dict["url"],
+ )
+ )
elif part_dict["type"] == "reasoning":
part_dict = cast(ReasoningPartDict, part_dict)
# Handle ReasoningDetails if present
@@ -96,9 +93,4 @@ def from_chat_message_dict(d: ChatMessageDict) -> ChatMessage:
)
)
- return ChatMessage(
- role=d["role"],
- content=d["content"],
- attachments=attachments,
- parts=parts,
- )
+ return ChatMessage(role=d["role"], content=d["content"], parts=parts)
diff --git a/tests/_ai/llm/test_impl.py b/tests/_ai/llm/test_impl.py
index c4e8c2802e..8680b43538 100644
--- a/tests/_ai/llm/test_impl.py
+++ b/tests/_ai/llm/test_impl.py
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch
import pytest
-from marimo._ai._types import ChatMessage, ChatModelConfig
+from marimo._ai._types import ChatMessage, ChatModelConfig, TextPart
from marimo._ai.llm._impl import (
DEFAULT_SYSTEM_MESSAGE,
anthropic,
@@ -274,9 +274,13 @@ class TestOpenAI:
assert call_args["model"] == "gpt-4"
assert len(call_args["messages"]) == 2
assert call_args["messages"][0]["role"] == "system"
- assert call_args["messages"][0]["content"] == DEFAULT_SYSTEM_MESSAGE
+ assert call_args["messages"][0]["content"] == [
+ {"type": "text", "text": DEFAULT_SYSTEM_MESSAGE}
+ ]
assert call_args["messages"][1]["role"] == "user"
- assert call_args["messages"][1]["content"] == "Test prompt"
+ assert call_args["messages"][1]["content"] == [
+ {"type": "text", "text": "Test prompt"}
+ ]
assert call_args["max_completion_tokens"] == 100
# Use pytest.approx for floating point comparisons
assert call_args["temperature"] == pytest.approx(0.7)
@@ -436,10 +440,26 @@ class TestOpenAI:
# Test with multiple messages
messages = [
- ChatMessage(role="system", content="Custom system"),
- ChatMessage(role="user", content="Hello"),
- ChatMessage(role="assistant", content="Hi there"),
- ChatMessage(role="user", content="How are you?"),
+ ChatMessage(
+ role="system",
+ content="Custom system",
+ parts=[TextPart(type="text", text="Custom system")],
+ ),
+ ChatMessage(
+ role="user",
+ content="Hello",
+ parts=[TextPart(type="text", text="Hello")],
+ ),
+ ChatMessage(
+ role="assistant",
+ content="Hi there",
+ parts=[TextPart(type="text", text="Hi there")],
+ ),
+ ChatMessage(
+ role="user",
+ content="How are you?",
+ parts=[TextPart(type="text", text="How are you?")],
+ ),
]
model(messages, ChatModelConfig())
@@ -447,18 +467,31 @@ class TestOpenAI:
# Verify the messages were properly converted
call_args = mock_client.chat.completions.create.call_args[1]
assert len(call_args["messages"]) == 5 # system + all messages
- assert call_args["messages"][0]["role"] == "system"
- assert (
- call_args["messages"][0]["content"] == DEFAULT_SYSTEM_MESSAGE
- )
- assert call_args["messages"][1]["role"] == "system"
- assert call_args["messages"][1]["content"] == "Custom system"
- assert call_args["messages"][2]["role"] == "user"
- assert call_args["messages"][2]["content"] == "Hello"
- assert call_args["messages"][3]["role"] == "assistant"
- assert call_args["messages"][3]["content"] == "Hi there"
- assert call_args["messages"][4]["role"] == "user"
- assert call_args["messages"][4]["content"] == "How are you?"
+
+ assert call_args["messages"] == [
+ {
+ "role": "system",
+ "content": [
+ {"type": "text", "text": DEFAULT_SYSTEM_MESSAGE}
+ ],
+ },
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "Custom system"}],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Hello"}],
+ },
+ {
+ "role": "assistant",
+ "content": [{"type": "text", "text": "Hi there"}],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "How are you?"}],
+ },
+ ]
@pytest.mark.skipif(
diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.py
index f1e0a35ad3..c5ceaeea0f 100644
--- a/tests/_ai/test_chat_convert.py
+++ b/tests/_ai/test_chat_convert.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import base64
import json
+from copy import deepcopy
import pytest
@@ -14,13 +15,13 @@ from marimo._ai._convert import (
convert_to_groq_messages,
convert_to_openai_messages,
convert_to_openai_tools,
- get_anthropic_messages_from_parts,
+ get_anthropic_parts_from_chat_parts,
get_google_messages_from_parts,
get_openai_messages_from_parts,
)
from marimo._ai._types import (
- ChatAttachment,
ChatMessage,
+ FilePart,
ReasoningDetails,
ReasoningPart,
TextPart,
@@ -36,34 +37,35 @@ def sample_messages() -> list[ChatMessage]:
ChatMessage(
role="user",
content="Hello, I have a question.",
- attachments=[
- ChatAttachment(
- name="image.png",
- content_type="image/png",
+ parts=[
+ TextPart(type="text", text="Hello, I have a question."),
+ FilePart(
+ type="file",
+ media_type="image/png",
+ filename="image.png",
url=f"data:image/png;base64,{base64.b64encode(b'hello')}",
),
- ChatAttachment(
- name="text.txt",
- content_type="text/plain",
- url="data:text/csv;base64,QQoxCjIKMwo=",
- ),
+ TextPart(type="text", text="A\n1\n2\n3\n"),
],
),
ChatMessage(
role="assistant",
content="Sure, I'd be happy to help. What's your question?",
- attachments=[],
+ parts=[
+ TextPart(
+ type="text",
+ text="Sure, I'd be happy to help. What's your question?",
+ ),
+ ],
),
]
-def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):
+def test_convert_to_openai_messages(
+ sample_messages: list[ChatMessage],
+):
result = convert_to_openai_messages(sample_messages)
-
- assert len(result) == 2
-
assert result == [
- # User message
{
"role": "user",
"content": [
@@ -75,26 +77,23 @@ def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):
{"type": "text", "text": "A\n1\n2\n3\n"},
],
},
- # Assistant message
{
"role": "assistant",
- "content": "Sure, I'd be happy to help. What's your question?",
+ "content": [
+ {
+ "type": "text",
+ "text": "Sure, I'd be happy to help. What's your question?",
+ }
+ ],
},
]
-def test_convert_to_openai_messages_with_parts_and_attachments():
+def test_convert_to_openai_messages_with_parts():
sample_messages = [
ChatMessage(
role="user",
content="Message with parts and attachments",
- attachments=[
- ChatAttachment(
- name="image.png",
- content_type="image/png",
- url=f"data:image/png;base64,{base64.b64encode(b'hello')}",
- ),
- ],
parts=[
TextPart(
type="text", text="Message with parts and attachments"
@@ -115,6 +114,12 @@ def test_convert_to_openai_messages_with_parts_and_attachments():
input={"expression": "2 + 2"},
output={"answer": 4},
),
+ FilePart(
+ type="file",
+ media_type="image/png",
+ filename="image.png",
+ url=f"data:image/png;base64,{base64.b64encode(b'hello')}",
+ ),
],
),
]
@@ -147,84 +152,91 @@ def test_convert_to_openai_messages_with_parts_and_attachments():
]
-def test_convert_to_anthropic_messages(sample_messages: list[ChatMessage]):
+def test_convert_to_anthropic_messages(
+ sample_messages: list[ChatMessage],
+):
result = convert_to_anthropic_messages(sample_messages)
-
- assert len(result) == 2
-
- # Check user message
- assert result[0]["role"] == "user"
- assert len(result[0]["content"]) == 3
- assert result[0]["content"][0] == {
- "type": "text",
- "text": "Hello, I have a question.",
- }
- assert result[0]["content"][1] == {
- "type": "image",
- "source": {
- "data": "b'aGVsbG8='",
- "media_type": "image/png",
- "type": "base64",
+ assert result == [
+ # User message
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Hello, I have a question."},
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "b'aGVsbG8='",
+ },
+ },
+ {"type": "text", "text": "A\n1\n2\n3\n"},
+ ],
},
- }
- assert result[0]["content"][2] == {
- "type": "text",
- "text": "A\n1\n2\n3\n",
- }
-
- # Check assistant message
- assert result[1]["role"] == "assistant"
- assert (
- result[1]["content"]
- == "Sure, I'd be happy to help. What's your question?"
- )
+ # Assistant message
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": "Sure, I'd be happy to help. What's your question?",
+ }
+ ],
+ },
+ ]
-def test_convert_to_google_messages(sample_messages: list[ChatMessage]):
+def test_convert_to_google_messages(
+ sample_messages: list[ChatMessage],
+):
result = convert_to_google_messages(sample_messages)
-
- assert len(result) == 2
-
- # Check user message
- assert result[0]["role"] == "user"
- assert result[0]["parts"] == [
- {"text": "Hello, I have a question."},
+ assert result == [
+ # User message
{
- "inline_data": {
- "data": b"m\xa1\x95\xb1\xb1\xbc",
- "mime_type": "image/png",
- }
+ "role": "user",
+ "parts": [
+ {"text": "Hello, I have a question."},
+ {
+ "inline_data": {
+ "mime_type": "image/png",
+ "data": b"m\xa1\x95\xb1\xb1\xbc",
+ }
+ },
+ {"text": "A\n1\n2\n3\n"},
+ ],
},
+ # Assistant message
{
- "inline_data": {
- "data": b"A\n1\n2\n3\n",
- "mime_type": "text/plain",
- }
+ "role": "model",
+ "parts": [
+ {"text": "Sure, I'd be happy to help. What's your question?"}
+ ],
},
]
- # Check assistant message
- assert result[1]["role"] == "model"
- assert result[1]["parts"] == [
- {"text": "Sure, I'd be happy to help. What's your question?"}
- ]
-
-def test_convert_to_groq_messages(sample_messages: list[ChatMessage]):
- result = convert_to_groq_messages(sample_messages)
-
- assert len(result) == 2
-
- # Check user message with text attachment
- assert result[0]["role"] == "user"
- assert result[0]["content"] == "Hello, I have a question.\nA\n1\n2\n3\n"
-
- # Check assistant message
- assert result[1]["role"] == "assistant"
- assert (
- result[1]["content"]
- == "Sure, I'd be happy to help. What's your question?"
+def test_convert_to_groq_messages(
+ sample_messages: list[ChatMessage],
+):
+ new_messages = deepcopy(sample_messages)
+ # replace image file part with text file part
+ new_messages[0].parts[1] = FilePart(
+ filename="text.txt",
+ type="file",
+ media_type="text/plain",
+ url="data:text/plain;base64,R29vZGJ5ZQ==",
)
+ result = convert_to_groq_messages(new_messages)
+
+ assert result == [
+ # User message
+ {"role": "user", "content": "Hello, I have a question.\nGoodbye"},
+ # Assistant message
+ {
+ "role": "assistant",
+ "content": "Sure, I'd be happy to help. What's your question?",
+ },
+ ]
def test_empty_messages():
@@ -238,29 +250,41 @@ def test_empty_messages():
def test_message_without_attachments():
messages = [
ChatMessage(
- role="user", content="Just a simple message", attachments=[]
+ role="user",
+ content="Just a simple message",
+ attachments=[],
+ parts=[TextPart(type="text", text="Just a simple message")],
)
]
openai_result = convert_to_openai_messages(messages)
- assert len(openai_result) == 1
- assert openai_result[0]["role"] == "user"
- assert openai_result[0]["content"] == "Just a simple message"
+ assert openai_result == [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Just a simple message"}],
+ }
+ ]
anthropic_result = convert_to_anthropic_messages(messages)
- assert len(anthropic_result) == 1
- assert anthropic_result[0]["role"] == "user"
- assert anthropic_result[0]["content"] == "Just a simple message"
+ assert anthropic_result == [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "Just a simple message"}],
+ }
+ ]
google_result = convert_to_google_messages(messages)
- assert len(google_result) == 1
- assert google_result[0]["role"] == "user"
- assert google_result[0]["parts"] == [{"text": "Just a simple message"}]
+ assert google_result == [
+ {
+ "role": "user",
+ "parts": [{"text": "Just a simple message"}],
+ }
+ ]
groq_result = convert_to_groq_messages(messages)
- assert len(groq_result) == 1
- assert groq_result[0]["role"] == "user"
- assert groq_result[0]["content"] == "Just a simple message"
+ assert groq_result == [
+ {"role": "user", "content": "Just a simple message"}
+ ]
def test_from_chat_message_dict():
@@ -268,25 +292,30 @@ def test_from_chat_message_dict():
message_dict = {
"role": "user",
"content": "Hello, this is a test message.",
- "attachments": [
+ "parts": [
{
- "name": "test.png",
- "content_type": "image/png",
+ "type": "file",
+ "media_type": "image/png",
+ "filename": "test.png",
"url": "http://example.com/test.png",
}
],
}
result = from_chat_message_dict(message_dict)
-
- assert isinstance(result, ChatMessage)
- assert result.role == "user"
- assert result.content == "Hello, this is a test message."
- assert len(result.attachments) == 1
- assert isinstance(result.attachments[0], ChatAttachment)
- assert result.attachments[0].name == "test.png"
- assert result.attachments[0].content_type == "image/png"
- assert result.attachments[0].url == "http://example.com/test.png"
+ assert result == ChatMessage(
+ role="user",
+ content="Hello, this is a test message.",
+ attachments=None,
+ parts=[
+ FilePart(
+ type="file",
+ media_type="image/png",
+ filename="test.png",
+ url="http://example.com/test.png",
+ )
+ ],
+ )
# Test case 2: ChatMessage without attachments
message_dict_no_attachments = {
@@ -327,32 +356,7 @@ def test_from_chat_message_dict():
result_with_parts.parts[1].reasoning == "This is my reasoning process"
)
- # Test case 4: ChatMessage with both attachments and parts
- message_dict_full = {
- "role": "user",
- "content": "Complex message with everything.",
- "attachments": [
- {
- "name": "doc.pdf",
- "content_type": "application/pdf",
- "url": "http://example.com/doc.pdf",
- }
- ],
- "parts": [{"type": "text", "text": "Additional text content"}],
- }
-
- result_full = from_chat_message_dict(message_dict_full)
-
- assert isinstance(result_full, ChatMessage)
- assert result_full.role == "user"
- assert result_full.content == "Complex message with everything."
- assert len(result_full.attachments) == 1
- assert result_full.attachments[0].name == "doc.pdf"
- assert len(result_full.parts) == 1
- assert result_full.parts[0].type == "text"
- assert result_full.parts[0].text == "Additional text content"
-
- # Test case 5: ChatMessage with tool invocation part (result state only)
+ # Test case 4: ChatMessage with tool invocation part (result state only)
message_dict_tool_result = {
"role": "assistant",
"content": "Here's the tool result.",
@@ -383,7 +387,7 @@ def test_from_chat_message_dict():
assert result_tool_result.parts[0].tool_name == "weather_tool"
assert result_tool_result.parts[0].input == {"location": "New York"}
- # Test case 6: Existing ChatMessage input (should return as-is)
+ # Test case 5: Existing ChatMessage input (should return as-is)
existing_message = ChatMessage(
role="user",
content="I'm already a ChatMessage object",
@@ -628,24 +632,21 @@ def test_get_openai_messages_from_parts_empty():
assert result == []
-def test_get_anthropic_messages_from_parts_text_only():
+def test_get_anthropic_parts_from_chat_parts_text_only():
"""Test converting TextPart to Anthropic format."""
parts = [
TextPart(type="text", text="Hello"),
TextPart(type="text", text="World"),
]
- result = get_anthropic_messages_from_parts("user", parts)
-
- assert len(result) == 1
- assert result[0]["role"] == "user"
- assert result[0]["content"] == [
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [
{"type": "text", "text": "Hello"},
{"type": "text", "text": "World"},
]
-def test_get_anthropic_messages_from_parts_with_reasoning():
+def test_get_anthropic_parts_from_chat_parts_with_reasoning():
"""Test converting ReasoningPart to Anthropic thinking format."""
reasoning_details = [
ReasoningDetails(
@@ -662,11 +663,8 @@ def test_get_anthropic_messages_from_parts_with_reasoning():
),
]
- result = get_anthropic_messages_from_parts("assistant", parts)
-
- assert len(result) == 1
- assert result[0]["role"] == "assistant"
- assert result[0]["content"] == [
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [
{"type": "text", "text": "Let me think about this"},
{
"type": "thinking",
@@ -676,7 +674,7 @@ def test_get_anthropic_messages_from_parts_with_reasoning():
]
-def test_get_anthropic_messages_from_parts_reasoning_no_signature():
+def test_get_anthropic_parts_from_chat_parts_reasoning_no_signature():
"""Test converting ReasoningPart without signature to Anthropic format."""
reasoning_details = [
ReasoningDetails(type="text", text="Step 1", signature=None)
@@ -690,20 +688,17 @@ def test_get_anthropic_messages_from_parts_reasoning_no_signature():
),
]
- result = get_anthropic_messages_from_parts("assistant", parts)
-
- assert len(result) == 1
- assert result[0]["role"] == "assistant"
- assert result[0]["content"] == [
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [
{
"type": "thinking",
"thinking": "Basic reasoning",
"signature": "", # Should be empty string when no signature
- },
+ }
]
-def test_get_anthropic_messages_from_parts_reasoning_empty_details():
+def test_get_anthropic_parts_from_empty_reasoning_details_chat_part():
"""Test converting ReasoningPart with empty details list."""
parts = [
ReasoningPart(
@@ -713,20 +708,17 @@ def test_get_anthropic_messages_from_parts_reasoning_empty_details():
),
]
- result = get_anthropic_messages_from_parts("assistant", parts)
-
- assert len(result) == 1
- assert result[0]["role"] == "assistant"
- assert result[0]["content"] == [
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [
{
"type": "thinking",
"thinking": "Some reasoning",
"signature": "", # Should be empty when no details
- },
+ }
]
-def test_get_anthropic_messages_from_parts_with_tool_invocation():
+def test_get_anthropic_parts_from_chat_parts_with_tool_invocation():
"""Test converting ToolInvocationPart to Anthropic format."""
parts = [
TextPart(type="text", text="I'll search for you"),
@@ -739,47 +731,37 @@ def test_get_anthropic_messages_from_parts_with_tool_invocation():
),
]
- result = get_anthropic_messages_from_parts("assistant", parts)
-
- assert (
- len(result) == 2
- ) # One message with tool use, one tool result message
-
- # Check first message with tool use
- assert result[0]["role"] == "assistant"
- expected_content = [
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [
+ # Text message
{"type": "text", "text": "I'll search for you"},
+ # Tool use message
{
"type": "tool_use",
"id": "call_123",
"name": "search_tool",
"input": {"query": "Python tutorials"},
},
- ]
- assert result[0]["content"] == expected_content
-
- # Check tool result message
- assert result[1]["role"] == "user"
- assert result[1]["content"] == [
+ # Tool result message
{
- "type": "tool_result",
"tool_use_id": "call_123",
- "content": str({"results": ["tutorial1", "tutorial2"]}),
- }
+ "type": "tool_result",
+ "content": [
+ {
+ "type": "text",
+ "text": "{'results': ['tutorial1', 'tutorial2']}",
+ }
+ ],
+ },
]
-def test_get_anthropic_messages_from_parts_single_text():
- """Test converting single TextPart uses string format instead of array."""
+def test_get_anthropic_parts_from_chat_parts_single_text():
+ """Test converting single text part."""
parts = [TextPart(type="text", text="Single message")]
- result = get_anthropic_messages_from_parts("user", parts)
-
- assert len(result) == 1
- assert result[0]["role"] == "user"
- assert (
- result[0]["content"] == "Single message"
- ) # Should be string, not array
+ result = get_anthropic_parts_from_chat_parts(parts)
+ assert result == [{"type": "text", "text": "Single message"}]
def test_get_google_messages_from_parts_text_only():
diff --git a/tests/_server/api/endpoints/test_ai.py b/tests/_server/api/endpoints/test_ai.py
index e50ee642ca..6caee24f5d 100644
--- a/tests/_server/api/endpoints/test_ai.py
+++ b/tests/_server/api/endpoints/test_ai.py
@@ -139,7 +139,7 @@ class TestOpenAiEndpoints:
# Assert the prompt it was called with
prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][1]["content"]
+ ][1]["content"][0]["text"]
assert prompt == ("Help me create a dataframe")
# Assert the model it was called with
model = oaiclient.chat.completions.create.call_args.kwargs["model"]
@@ -184,7 +184,7 @@ class TestOpenAiEndpoints:
# Assert the prompt it was called with
prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][1]["content"]
+ ][1]["content"][0]["text"]
assert prompt == "Help me create a dataframe"
@staticmethod
@@ -305,7 +305,7 @@ class TestOpenAiEndpoints:
# Assert the prompt it was called with
prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][1]["content"]
+ ][1]["content"][0]["text"]
assert (
prompt
== f"{FIM_PREFIX_TAG}import pandas as pd\n{FIM_SUFFIX_TAG}\ndf.head(){FIM_MIDDLE_TAG}"
@@ -313,7 +313,7 @@ class TestOpenAiEndpoints:
# Assert the system prompt for FIM models
system_prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][0]["content"]
+ ][0]["content"][0]["text"]
assert (
system_prompt
== f"You are a python code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."
@@ -385,7 +385,7 @@ class TestOpenAiEndpoints:
# Assert the system prompt for FIM models
system_prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][0]["content"]
+ ][0]["content"][0]["text"]
assert (
system_prompt
== f"You are a sql code completion assistant. Complete the missing code between the prefix and suffix while maintaining proper syntax, style, and functionality.Only output the code that goes after the {FIM_SUFFIX_TAG} part. Do not add any explanation or markdown."
@@ -462,7 +462,7 @@ class TestAnthropicAiEndpoints:
# Assert the prompt it was called with
prompt: str = anthropic_client.messages.create.call_args.kwargs[
"messages"
- ][0]["content"]
+ ][0]["content"][0]["text"]
assert prompt == "Help me create a dataframe"
# Assert the model it was called with
model = anthropic_client.messages.create.call_args.kwargs["model"]
@@ -503,7 +503,7 @@ class TestAnthropicAiEndpoints:
# Assert the prompt it was called with
prompt: str = anthropic_client.messages.create.call_args.kwargs[
"messages"
- ][0]["content"]
+ ][0]["content"][0]["text"]
assert (
prompt
== f"{FIM_PREFIX_TAG}import pandas as pd\n{FIM_SUFFIX_TAG}\ndf.head(){FIM_MIDDLE_TAG}"
@@ -781,7 +781,7 @@ def test_chat_without_code(client: TestClient) -> None:
# Assert the prompt it was called with
prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][1]["content"]
+ ][1]["content"][0]["text"]
assert prompt == "Hello"
@@ -827,7 +827,7 @@ def test_chat_with_code(client: TestClient) -> None:
# Assert the prompt it was called with
prompt = oaiclient.chat.completions.create.call_args.kwargs[
"messages"
- ][1]["content"]
+ ][1]["content"][0]["text"]
assert prompt == "Help me create a dataframe"