A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
attachments in chat sidebar (#6181)
Shahmir Varqha4dcf8ec0a752
diff --git a/frontend/src/__tests__/chat-utils.test.ts b/frontend/src/__tests__/chat-utils.test.tsindex 6342d3573d..91ec485b98 100644--- a/frontend/src/__tests__/chat-utils.test.ts+++ b/frontend/src/__tests__/chat-utils.test.ts@@ -58,13 +58,13 @@ describe("addMessageToChat", () => {};it("should add a new message to an existing chat", () => {- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-4",- "user",- "New message",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-4",+ role: "user",+ content: "New message",+ });expect(result.chats).toHaveLength(2);const updatedChat = result.chats.get(CHAT_1);@@ -81,13 +81,13 @@ describe("addMessageToChat", () => {});it("should update an existing message", () => {- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-1",- "user",- "Updated content",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-1",+ role: "user",+ content: "Updated content",+ });expect(result.chats).toHaveLength(2);const updatedChat = result.chats.get(CHAT_1);@@ -105,14 +105,14 @@ describe("addMessageToChat", () => {it("should handle message parts", () => {const parts = [{ type: "text" as const, text: "Part content" }];- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-5",- "assistant",- "Message with parts",+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-5",+ role: "assistant",+ content: "Message with parts",parts,- );+ });const updatedChat = result.chats.get(CHAT_1);expect(updatedChat?.messages[2].parts).toEqual(parts);@@ -140,64 +140,64 @@ describe("addMessageToChat", () => {]),};- const result = addMessageToChat(- stateWithParts,- CHAT_1,- "msg-1",- "user",- "Updated content",- updatedParts,- );+ const result = addMessageToChat({+ chatState: stateWithParts,+ chatId: CHAT_1,+ messageId: "msg-1",+ role: "user",+ content: "Updated content",+ parts: updatedParts,+ });const updatedChat = result.chats.get(CHAT_1);expect(updatedChat?.messages[0].parts).toEqual(updatedParts);});it("should return unchanged state when chatId is null", () => {- const result = addMessageToChat(- mockChatState,- null,- "msg-4",- "user",- "New message",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: null,+ messageId: "msg-4",+ role: "user",+ content: "New message",+ });expect(result).toEqual(mockChatState);});it("should return unchanged state when chatId does not exist", () => {- const result = addMessageToChat(- mockChatState,- "non-existent-chat" as ChatId,- "msg-4",- "user",- "New message",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: "non-existent-chat" as ChatId,+ messageId: "msg-4",+ role: "user",+ content: "New message",+ });expect(result).toEqual(mockChatState);});it("should not modify other chats when updating a specific chat", () => {- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-4",- "user",- "New message",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-4",+ role: "user",+ content: "New message",+ });const unchangedChat = result.chats.get(CHAT_2);expect(unchangedChat).toEqual([...mockChatState.chats.values()][1]);});it("should preserve message order when adding new messages", () => {- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-4",- "user",- "New message",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-4",+ role: "user",+ content: "New message",+ });const updatedChat = result.chats.get(CHAT_1);expect(updatedChat?.messages[0].id).toBe("msg-1");@@ -220,13 +220,13 @@ describe("addMessageToChat", () => {activeChatId: chatId,};- const result = addMessageToChat(- emptyChatState,- chatId,- "msg-1",- "user",- "First message",- );+ const result = addMessageToChat({+ chatState: emptyChatState,+ chatId: chatId,+ messageId: "msg-1",+ role: "user",+ content: "First message",+ });const updatedChat = result.chats.get(chatId);expect(updatedChat?.messages).toHaveLength(1);@@ -234,13 +234,13 @@ describe("addMessageToChat", () => {});it("should handle different message roles", () => {- const result = addMessageToChat(- mockChatState,- CHAT_1,- "msg-4",- "assistant",- "Assistant response",- );+ const result = addMessageToChat({+ chatState: mockChatState,+ chatId: CHAT_1,+ messageId: "msg-4",+ role: "assistant",+ content: "Assistant response",+ });const updatedChat = result.chats.get(CHAT_1);expect(updatedChat?.messages[2].role).toBe("assistant");diff --git a/frontend/src/components/chat/chat-components.tsx b/frontend/src/components/chat/chat-components.tsxnew file mode 100644index 0000000000..65af3e32b7--- /dev/null+++ b/frontend/src/components/chat/chat-components.tsx@@ -0,0 +1,71 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { FileIcon, FileTextIcon, ImageIcon, XIcon } from "lucide-react";+import { useState } from "react";+import type { ChatAttachment } from "@/core/ai/types";+import { cn } from "@/utils/cn";++export const AttachmentRenderer = ({+ attachment,+}: {+ attachment: ChatAttachment;+}) => {+ if (attachment.contentType?.startsWith("image/")) {+ return (+ <img+ src={attachment.url}+ alt={attachment.name}+ className="max-h-[100px] max-w-[100px] object-contain mb-1.5"+ />+ );+ }++ return (+ <div className="flex flex-row gap-1 items-center text-xs">+ <FileIcon className="h-3 w-3 mt-0.5" />+ {attachment.name}+ </div>+ );+};++export const FileAttachmentPill = ({+ file,+ className,+ onRemove,+}: {+ file: File;+ className?: string;+ onRemove: () => void;+}) => {+ const [isHovered, setIsHovered] = useState(false);++ return (+ <div+ className={cn(+ "py-1 px-1.5 bg-muted rounded-md cursor-pointer flex flex-row gap-1 items-center text-xs",+ className,+ )}+ onMouseEnter={() => setIsHovered(true)}+ onMouseLeave={() => setIsHovered(false)}+ >+ {isHovered ? (+ <XIcon className="h-3 w-3 mt-0.5" onClick={onRemove} />+ ) : (+ renderFileIcon(file)+ )}+ {file.name}+ </div>+ );+};++function renderFileIcon(file: File): React.ReactNode {+ const classNames = "h-3 w-3 mt-0.5";++ if (file.type.startsWith("image/")) {+ return <ImageIcon className={classNames} />;+ } else if (file.type.startsWith("text/")) {+ return <FileTextIcon className={classNames} />;+ }++ return <FileIcon className={classNames} />;+}diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsxindex 51693ba1c3..7134f1552d 100644--- a/frontend/src/components/chat/chat-panel.tsx+++ b/frontend/src/components/chat/chat-panel.tsx@@ -9,6 +9,7 @@ import {BotMessageSquareIcon,ClockIcon,Loader2,+ PaperclipIcon,PlusIcon,SendIcon,SettingsIcon,@@ -41,6 +42,7 @@ import {} from "@/components/ui/select";import { addMessageToChat } from "@/core/ai/chat-utils";import { useModelChange } from "@/core/ai/config";+import { AiModelId, type ProviderId } from "@/core/ai/ids/ids";import {activeChatAtom,type Chat,@@ -48,7 +50,7 @@ import {type ChatState,chatStateAtom,} from "@/core/ai/state";-import { getCodes } from "@/core/codemirror/copilot/getCodes";+import type { ChatAttachment } from "@/core/ai/types";import { aiAtom, aiEnabledAtom } from "@/core/config/config";import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";import { FeatureFlagged } from "@/core/config/feature-flag";@@ -57,6 +59,7 @@ import { useRuntimeManager } from "@/core/runtime/config";import { ErrorBanner } from "@/plugins/impl/common/error-banner";import { cn } from "@/utils/cn";import { timeAgo } from "@/utils/dates";+import { blobToString } from "@/utils/fileToBase64";import { Logger } from "@/utils/Logger";import { generateUUID } from "@/utils/uuid";import { AIModelDropdown } from "../ai/ai-model-dropdown";@@ -65,11 +68,23 @@ import { PromptInput } from "../editor/ai/add-cell-with-ai";import { getAICompletionBody } from "../editor/ai/completion-utils";import { PanelEmptyState } from "../editor/chrome/panels/empty-state";import { CopyClipboardIcon } from "../icons/copy-icon";+import { Input } from "../ui/input";import { Tooltip, TooltipProvider } from "../ui/tooltip";+import { toast } from "../ui/use-toast";+import { AttachmentRenderer, FileAttachmentPill } from "./chat-components";import { MarkdownRenderer } from "./markdown-renderer";import { ReasoningAccordion } from "./reasoning-accordion";import { ToolCallAccordion } from "./tool-call-accordion";+// Default mode for the AI+const DEFAULT_MODE = "manual";++// We need to modify the backend to support attachments for other providers+// And other types+const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>(["openai"]);+const SUPPORTED_ATTACHMENT_TYPES = ["image/*", "text/*"];+const MAX_ATTACHMENT_SIZE = 1024 * 1024 * 50; // 50MB+interface ChatHeaderProps {onNewChat: () => void;activeChatId: ChatId | undefined;@@ -168,6 +183,9 @@ const ChatMessage: React.FC<ChatMessageProps> = memo(>{message.role === "user" ? (<div className="w-[95%] bg-background border p-1 rounded-sm">+ {message.experimental_attachments?.map((attachment, idx) => (+ <AttachmentRenderer attachment={attachment} key={idx} />+ ))}<PromptInputkey={message.id}value={message.content}@@ -243,15 +261,24 @@ interface ChatInputFooterProps {onSendClick: () => void;isLoading: boolean;onStop: () => void;+ handleFileChange: (event: React.ChangeEvent<HTMLInputElement>) => void;+ fileInputRef: React.RefObject<HTMLInputElement | null>;}-const DEFAULT_MODE = "manual";-const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(- ({ isEmpty, onSendClick, isLoading, onStop }) => {+ ({+ isEmpty,+ onSendClick,+ isLoading,+ onStop,+ fileInputRef,+ handleFileChange,+ }) => {const ai = useAtomValue(aiAtom);const currentMode = ai?.mode || DEFAULT_MODE;const currentModel = ai?.models?.chat_model || DEFAULT_AI_MODEL;+ const currentProvider = AiModelId.parse(currentModel).providerId;+const { saveModeChange, saveModelChange } = useModelChange();const modeOptions = [@@ -268,8 +295,11 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(},];+ const isAttachmentSupported =+ PROVIDERS_THAT_SUPPORT_ATTACHMENTS.has(currentProvider);+return (- <div className="px-3 py-2 border-t border-border/20 flex items-center justify-between">+ <div className="px-3 py-2 border-t border-border/20 flex flex-row items-center justify-between"><div className="flex items-center gap-2"><FeatureFlagged feature="mcp_docs"><Select value={currentMode} onValueChange={saveModeChange}>@@ -307,19 +337,43 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(forRole="chat"/></div>- <Button- variant="ghost"- size="sm"- className="h-6 w-6 p-0 hover:bg-muted/30"- onClick={isLoading ? onStop : onSendClick}- disabled={isLoading ? false : isEmpty}- >- {isLoading ? (- <SquareIcon className="h-3 w-3 fill-current" />- ) : (- <SendIcon className="h-3 w-3" />+ <div className="flex flex-row">+ {isAttachmentSupported && (+ <>+ <Button+ variant="text"+ size="icon"+ className="cursor-pointer"+ onClick={() => fileInputRef.current?.click()}+ title="Attach a file"+ >+ <PaperclipIcon className="h-3.5 w-3.5" />+ </Button>+ <Input+ ref={fileInputRef}+ type="file"+ multiple={true}+ hidden={true}+ onChange={handleFileChange}+ accept={SUPPORTED_ATTACHMENT_TYPES.join(",")}+ />+ </>)}- </Button>++ <Button+ variant="text"+ size="sm"+ className="h-6 w-6 p-0 hover:bg-muted/30 cursor-pointer"+ onClick={isLoading ? onStop : onSendClick}+ disabled={isLoading ? false : isEmpty}+ >+ {isLoading ? (+ <SquareIcon className="h-3 w-3 fill-current" />+ ) : (+ <SendIcon className="h-3 w-3" />+ )}+ </Button>+ </div></div>);},@@ -328,16 +382,33 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(ChatInputFooter.displayName = "ChatInputFooter";interface ChatInputProps {+ placeholder?: string;input: string;+ inputClassName?: string;setInput: (value: string) => void;onSubmit: (e: KeyboardEvent | undefined, value: string) => void;inputRef: React.RefObject<ReactCodeMirrorRef | null>;isLoading: boolean;onStop: () => void;+ onClose: () => void;+ fileInputRef: React.RefObject<HTMLInputElement | null>;+ handleFileChange: (event: React.ChangeEvent<HTMLInputElement>) => void;}const ChatInput: React.FC<ChatInputProps> = memo(- ({ input, setInput, onSubmit, inputRef, isLoading, onStop }) => {+ ({+ placeholder,+ input,+ inputClassName,+ setInput,+ onSubmit,+ inputRef,+ isLoading,+ onStop,+ fileInputRef,+ handleFileChange,+ onClose,+ }) => {const handleSendClick = useEvent(() => {if (input.trim()) {onSubmit(undefined, input);@@ -346,14 +417,14 @@ const ChatInput: React.FC<ChatInputProps> = memo(return (<div className="border-t relative shrink-0 min-h-[80px] flex flex-col">- <div className="px-2 py-3 flex-1">+ <div className={cn("px-2 py-3 flex-1", inputClassName)}><PromptInputinputRef={inputRef}value={input}onChange={setInput}onSubmit={onSubmit}- onClose={() => inputRef.current?.editor?.blur()}- placeholder="Type your message..."+ onClose={onClose}+ placeholder={placeholder || "Type your message..."}/></div><ChatInputFooter@@ -361,6 +432,8 @@ const ChatInput: React.FC<ChatInputProps> = memo(onSendClick={handleSendClick}isLoading={isLoading}onStop={onStop}+ fileInputRef={fileInputRef}+ handleFileChange={handleFileChange}/></div>);@@ -395,9 +468,11 @@ const ChatPanelBody = () => {const [chatState, setChatState] = useAtom(chatStateAtom);const [activeChat, setActiveChat] = useAtom(activeChatAtom);const [newThreadInput, setNewThreadInput] = useState("");+ const [files, setFiles] = useState<File[]>();const newThreadInputRef = useRef<ReactCodeMirrorRef>(null);const newMessageInputRef = useRef<ReactCodeMirrorRef>(null);const scrollContainerRef = useRef<HTMLDivElement>(null);+ const fileInputRef = useRef<HTMLInputElement>(null);const messagesEndRef = useRef<HTMLDivElement>(null);const runtimeManager = useRuntimeManager();const { invokeAiTool } = useRequestClient();@@ -423,24 +498,36 @@ const ChatPanelBody = () => {api: runtimeManager.getAiURL("chat").toString(),headers: runtimeManager.headers(),experimental_prepareRequestBody: (options) => {- return {+ const completionBody = getAICompletionBody({+ input: options.messages.map((m) => m.content).join("\n"),+ });++ // Backend accepts attachments, so we convert the key+ const newOptions = {...options,- ...getAICompletionBody({- input: options.messages.map((m) => m.content).join("\n"),- }),- includeOtherCode: getCodes(""),+ messages: options.messages.map((m) => ({+ ...m,+ attachments: m.experimental_attachments,+ experimental_attachments: undefined,+ })),+ };++ return {+ ...newOptions,+ ...completionBody,};},onFinish: (message) => {setChatState((prev) => {- return addMessageToChat(- prev,- prev.activeChatId,- message.id,- "assistant",- message.content,- message.parts,- );+ return addMessageToChat({+ chatState: prev,+ chatId: prev.activeChatId,+ messageId: message.id,+ role: "assistant",+ content: message.content,+ parts: message.parts,+ attachments: message.experimental_attachments,+ });});},onToolCall: async ({ toolCall }) => {@@ -465,6 +552,36 @@ const ChatPanelBody = () => {},});+ const handleFileChange = useEvent(+ (event: React.ChangeEvent<HTMLInputElement>) => {+ const files = event.target.files;+ if (!files) {+ return;+ }++ let fileSize = 0;+ for (const file of files) {+ fileSize += file.size;+ }++ if (fileSize > MAX_ATTACHMENT_SIZE) {+ toast({+ title: "File size exceeds 50MB limit",+ description: "Please remove some files and try again.",+ });+ return;+ }++ setFiles([...files]);+ },+ );++ const removeFile = useEvent((file: File) => {+ if (files) {+ setFiles(files.filter((f) => f !== file));+ }+ });+const isLoading = status === "submitted" || status === "streaming";// Sync user messages from useChat to storage when they become available@@ -497,13 +614,15 @@ const ChatPanelBody = () => {let result = prev;for (const userMessage of missingUserMessages) {- result = addMessageToChat(- result,- prev.activeChatId,- userMessage.id,- "user",- userMessage.content,- );+ result = addMessageToChat({+ chatState: result,+ chatId: prev.activeChatId,+ messageId: userMessage.id,+ role: "user",+ content: userMessage.content,+ parts: userMessage.parts,+ attachments: userMessage.experimental_attachments,+ });}return result;@@ -533,7 +652,10 @@ const ChatPanelBody = () => {requestAnimationFrame(scrollToBottom);}, [chatState.activeChatId]);- const createNewThread = (initialMessage: string) => {+ const createNewThread = async (+ initialMessage: string,+ initialAttachments?: File[],+ ) => {const CURRENT_TIME = Date.now();const newChat: Chat = {id: generateUUID() as ChatId,@@ -558,12 +680,19 @@ const ChatPanelBody = () => {return newState;});+ const attachments =+ initialAttachments && initialAttachments.length > 0+ ? await convertToChatAttachments(initialAttachments)+ : undefined;+// Trigger AI conversation with appendappend({id: generateUUID(),role: "user",content: initialMessage,+ experimental_attachments: attachments,});+ setFiles(undefined);setInput("");};@@ -571,9 +700,13 @@ const ChatPanelBody = () => {setActiveChat(null);setInput("");setNewThreadInput("");+ setFiles(undefined);});const handleMessageEdit = useEvent((index: number, newValue: string) => {+ const editedMessage = messages[index];+ const attachments = editedMessage.experimental_attachments;+// Truncate both useChat and storagesetMessages((messages) => messages.slice(0, index));const activeChatId = chatState.activeChatId;@@ -599,18 +732,26 @@ const ChatPanelBody = () => {append({role: "user",content: newValue,+ experimental_attachments: attachments,});});const handleChatInputSubmit = useEvent(- (e: KeyboardEvent | undefined, newValue: string): void => {+ async (e: KeyboardEvent | undefined, newValue: string): Promise<void> => {if (!newValue.trim()) {return;}if (newMessageInputRef.current?.view) {storePrompt(newMessageInputRef.current.view);}- handleSubmit(e);+ const attachments = files+ ? await convertToChatAttachments(files)+ : undefined;++ handleSubmit(e, {+ experimental_attachments: attachments,+ });+ setFiles(undefined);},);@@ -625,7 +766,7 @@ const ChatPanelBody = () => {if (newThreadInputRef.current?.view) {storePrompt(newThreadInputRef.current.view);}- createNewThread(newThreadInput.trim());+ createNewThread(newThreadInput.trim(), files);});const handleOnCloseThread = () => newThreadInputRef.current?.editor?.blur();@@ -636,6 +777,53 @@ const ChatPanelBody = () => {);}, [chatState.chats]);+ const isNewThread = messages.length === 0;+ const chatInput = isNewThread ? (+ <ChatInput+ key="new-thread-input"+ placeholder="Ask anything, @ to include context about tables or dataframes"+ input={newThreadInput}+ inputRef={newThreadInputRef}+ inputClassName="px-1 py-0"+ setInput={setNewThreadInput}+ onSubmit={handleNewThreadSubmit}+ isLoading={isLoading}+ onStop={stop}+ fileInputRef={fileInputRef}+ handleFileChange={handleFileChange}+ onClose={handleOnCloseThread}+ />+ ) : (+ <ChatInput+ input={input}+ setInput={setInput}+ onSubmit={handleChatInputSubmit}+ inputRef={newMessageInputRef}+ isLoading={isLoading}+ onStop={stop}+ onClose={() => newMessageInputRef.current?.editor?.blur()}+ fileInputRef={fileInputRef}+ handleFileChange={handleFileChange}+ />+ );++ const filesPills = files && files.length > 0 && (+ <div+ className={cn(+ "flex flex-row gap-1 flex-wrap p-1",+ isNewThread && "py-2 px-1",+ )}+ >+ {files?.map((file) => (+ <FileAttachmentPill+ file={file}+ key={file.name}+ onRemove={() => removeFile(file)}+ />+ ))}+ </div>+ );+return (<div className="flex flex-col h-[calc(100%-53px)]"><TooltipProvider>@@ -651,25 +839,10 @@ const ChatPanelBody = () => {className="flex-1 px-3 bg-(--slate-1) gap-4 py-3 flex flex-col overflow-y-auto"ref={scrollContainerRef}>- {(!messages || messages.length === 0) && (- <div className="flex flex-col rounded-md border bg-background">- <div className="px-1">- <PromptInput- inputRef={newThreadInputRef}- key="new-thread-input"- value={newThreadInput}- placeholder="Ask anything, @ to include context about tables or dataframes"- onClose={handleOnCloseThread}- onChange={setNewThreadInput}- onSubmit={handleNewThreadSubmit}- />- </div>- <ChatInputFooter- isEmpty={!newThreadInput.trim()}- onSendClick={handleNewThreadSubmit}- isLoading={isLoading}- onStop={stop}- />+ {isNewThread && (+ <div className="rounded-md border bg-background">+ {filesPills}+ {chatInput}</div>)}@@ -712,15 +885,12 @@ const ChatPanelBody = () => {</div>)}- {messages && messages.length > 0 && (- <ChatInput- input={input}- setInput={setInput}- onSubmit={handleChatInputSubmit}- inputRef={newMessageInputRef}- isLoading={isLoading}- onStop={stop}- />+ {/* For existing threads, we place the chat input at the bottom */}+ {!isNewThread && (+ <>+ {filesPills}+ {chatInput}+ </>)}</div>);@@ -749,3 +919,19 @@ function isLastMessageReasoning(messages: Message[]): boolean {const lastPart = parts[parts.length - 1];return lastPart.type === "reasoning";}++async function convertToChatAttachments(+ files: File[],+): Promise<ChatAttachment[]> {+ const attachments = await Promise.all(+ files.map(async (file) => {+ return {+ name: file.name,+ url: await blobToString(file, "dataUrl"),+ contentType: file.type,+ };+ }),+ );++ return attachments;+}diff --git a/frontend/src/core/ai/chat-utils.ts b/frontend/src/core/ai/chat-utils.tsindex 51892b46f4..f6b9e0656e 100644--- a/frontend/src/core/ai/chat-utils.ts+++ b/frontend/src/core/ai/chat-utils.ts@@ -3,15 +3,27 @@import type { Message as AIMessage } from "@ai-sdk/react";import { Logger } from "@/utils/Logger";import type { ChatId, ChatState } from "./state";+import type { ChatAttachment } from "./types";-export const addMessageToChat = (- chatState: ChatState,- chatId: ChatId | null,- messageId: string,- role: "user" | "assistant",- content: string,- parts?: AIMessage["parts"],-): ChatState => {+interface AddMessageToChatParams {+ chatState: ChatState;+ chatId: ChatId | null;+ messageId: string;+ role: "user" | "assistant";+ content: string;+ parts?: AIMessage["parts"];+ attachments?: ChatAttachment[];+}++export const addMessageToChat = ({+ chatState,+ chatId,+ messageId,+ role,+ content,+ parts,+ attachments,+}: AddMessageToChatParams): ChatState => {if (!chatId) {Logger.warn("No active chat");return chatState;@@ -42,6 +54,7 @@ export const addMessageToChat = (content,timestamp: timestamp,parts,+ attachments,},],updatedAt: timestamp,@@ -53,6 +66,7 @@ export const addMessageToChat = (...newMessages[messageIndex],content,parts,+ attachments,};newChats.set(chat.id, {...chat,diff --git a/frontend/src/core/ai/state.ts b/frontend/src/core/ai/state.tsindex 7e6b3cbbcb..c4a8123de3 100644--- a/frontend/src/core/ai/state.ts+++ b/frontend/src/core/ai/state.ts@@ -6,6 +6,7 @@ import { atomWithStorage } from "jotai/utils";import { adaptForLocalStorage } from "@/utils/storage";import type { TypedString } from "@/utils/typed";import type { CellId } from "../cells/ids";+import type { ChatAttachment } from "./types";const KEY = "marimo:ai:chatState:v4";@@ -28,6 +29,7 @@ export interface Message {content: string;timestamp: number;parts?: AIMessage["parts"];+ attachments?: ChatAttachment[];}export interface Chat {diff --git a/frontend/src/core/ai/types.ts b/frontend/src/core/ai/types.tsnew file mode 100644index 0000000000..9f4fbb78d6--- /dev/null+++ b/frontend/src/core/ai/types.ts@@ -0,0 +1,17 @@+/* Copyright 2024 Marimo. All rights reserved. */++export interface ChatAttachment {+ /**+ * The name of the attachment, usually the file name.+ */+ name?: string;+ /**+ * A string indicating the [media type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).+ * By default, it's extracted from the pathname's extension.+ */+ contentType?: string;+ /**+ * The URL of the attachment. It can either be a URL to a hosted file or a [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs).+ */+ url: string;+}diff --git a/frontend/src/plugins/impl/MicrophonePlugin.tsx b/frontend/src/plugins/impl/MicrophonePlugin.tsxindex ba0f370c97..87a65a4052 100644--- a/frontend/src/plugins/impl/MicrophonePlugin.tsx+++ b/frontend/src/plugins/impl/MicrophonePlugin.tsx@@ -5,7 +5,7 @@ import { z } from "zod";import { AudioRecorder } from "@/components/audio/audio-recorder";import { useAudioRecorder } from "@/hooks/useAudioRecorder";import type { IPlugin, IPluginProps } from "@/plugins/types";-import { blobToBase64 } from "@/utils/fileToBase64";+import { blobToString } from "@/utils/fileToBase64";import { Labeled } from "./common/labeled";/**@@ -33,7 +33,7 @@ const Microphone = ({ setValue, data }: IPluginProps<Value, Data>) => {const { start, stop, pauseResume, recordingStatus, recordingTime, allowed } =useAudioRecorder({onDone: async (file) => {- const base64 = await blobToBase64(file);+ const base64 = await blobToString(file, "base64");setValue(base64);},});diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsxindex baf75dbbb5..c3d0aafac9 100644--- a/frontend/src/plugins/impl/chat/chat-ui.tsx+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx@@ -40,6 +40,7 @@ import {} from "@/components/ui/popover";import { Tooltip } from "@/components/ui/tooltip";import { toast } from "@/components/ui/use-toast";+import type { ChatAttachment } from "@/core/ai/types";import { moveToEndOfEditor } from "@/core/codemirror/utils";import { useAsyncData } from "@/hooks/useAsyncData";import { renderHTML } from "@/plugins/core/RenderHTML";@@ -49,12 +50,7 @@ import { Logger } from "@/utils/Logger";import { Objects } from "@/utils/objects";import { ErrorBanner } from "../common/error-banner";import type { PluginFunctions } from "./ChatPlugin";-import type {- ChatAttachment,- ChatConfig,- ChatMessage,- ChatRole,-} from "./types";+import type { ChatConfig, ChatMessage, ChatRole } from "./types";interface Props extends PluginFunctions {prompts: string[];diff --git a/frontend/src/plugins/impl/chat/types.ts b/frontend/src/plugins/impl/chat/types.tsindex ce0afcd448..2c035bdcc7 100644--- a/frontend/src/plugins/impl/chat/types.ts+++ b/frontend/src/plugins/impl/chat/types.ts@@ -1,4 +1,6 @@/* Copyright 2024 Marimo. All rights reserved. */++import type { ChatAttachment } from "@/core/ai/types";export type ChatRole = "system" | "user" | "assistant";export interface ChatMessage {@@ -7,12 +9,6 @@ export interface ChatMessage {attachments?: ChatAttachment[];}-export interface ChatAttachment {- name?: string;- contentType?: string;- url: string;-}-export interface SendMessageRequest {messages: ChatMessage[];config: {diff --git a/frontend/src/utils/__tests__/blob.test.ts b/frontend/src/utils/__tests__/blob.test.tsindex 48f46eea96..1e30336dd5 100644--- a/frontend/src/utils/__tests__/blob.test.ts+++ b/frontend/src/utils/__tests__/blob.test.ts@@ -1,6 +1,7 @@/* Copyright 2024 Marimo. All rights reserved. */import { beforeEach, describe, expect, test } from "vitest";import { deserializeBlob, serializeBlob } from "../blob";+import { blobToString } from "../fileToBase64";describe("Blob serialization and deserialization", () => {const testString = "Hello, world!";@@ -50,3 +51,39 @@ describe("Blob serialization and deserialization", () => {expect(deserialized.type).toBe(imageBlob.type);});});++describe("blobToString", () => {+ const testString = "Hello, world!";+ const mimeType = "text/plain";++ test("should convert a Blob to a base64 string", async () => {+ const blob = new Blob([testString], { type: mimeType });+ const base64 = await blobToString(blob, "base64");+ expect(base64).toBe(btoa(testString));+ });++ test("should convert a File to a base64 string", async () => {+ const file = new File([testString], "test.txt", { type: mimeType });+ const base64 = await blobToString(file, "base64");+ expect(base64).toBe(btoa(testString));+ });++ test("should convert a Blob to a data URL", async () => {+ const blob = new Blob([testString], { type: mimeType });+ const dataUrl = await blobToString(blob, "dataUrl");+ expect(dataUrl).toBe(`data:${mimeType};base64,${btoa(testString)}`);+ });++ test("should handle empty Blob", async () => {+ const blob = new Blob([], { type: mimeType });+ const base64 = await blobToString(blob, "base64");+ expect(base64).toBe("");+ });++ test("should handle binary data", async () => {+ const binaryData = new Uint8Array([0, 1, 2, 3, 4, 5]);+ const blob = new Blob([binaryData], { type: "application/octet-stream" });+ const base64 = await blobToString(blob, "base64");+ expect(base64).toBe(btoa(String.fromCharCode(...binaryData)));+ });+});diff --git a/frontend/src/utils/fileToBase64.ts b/frontend/src/utils/fileToBase64.tsindex 4fac3f060f..6038f95cca 100644--- a/frontend/src/utils/fileToBase64.ts+++ b/frontend/src/utils/fileToBase64.ts@@ -1,14 +1,29 @@/* Copyright 2024 Marimo. All rights reserved. */-export function blobToBase64(blob: Blob): Promise<string> {++/**+ * Converts a Blob or File to either a base64-encoded string or a data URL.+ *+ * @param input - The Blob or File to convert.+ * @param format - The output format: "base64" for the base64-encoded string, "dataUrl" for the full data URL.+ * @returns A promise that resolves to the requested string representation.+ */+export function blobToString(+ input: Blob | File,+ format: "base64" | "dataUrl",+): Promise<string> {return new Promise((resolve) => {const reader = new FileReader();- reader.readAsDataURL(blob);+ reader.readAsDataURL(input);reader.onload = (e) => {if (e.target?.result) {const dataURL = e.target.result as string;- // Get contents from a string of the form: data:*/*;base64,contents- const b64EncodedContents = dataURL.slice(dataURL.indexOf(",") + 1);- resolve(b64EncodedContents);+ if (format === "base64") {+ // Extract base64 content from data URL. data:*/*;base64,contents+ const base64Content = dataURL.slice(dataURL.indexOf(",") + 1);+ resolve(base64Content);+ } else {+ resolve(dataURL);+ }}};});@@ -22,7 +37,7 @@ export function blobToBase64(blob: Blob): Promise<string> {export function filesToBase64(files: File[]): Promise<Array<[string, string]>> {return Promise.all(files.map((file) =>- blobToBase64(file).then(+ blobToString(file, "base64").then((contents) => [file.name, contents] as [string, string],),),diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.pyindex b01e7149c7..a9c250ae3e 100644--- a/marimo/_ai/_convert.py+++ b/marimo/_ai/_convert.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport base64import json+from dataclasses import asdictfrom typing import TYPE_CHECKING, Any, Literal, Union, castfrom marimo._ai._types import (@@ -85,15 +86,14 @@ def convert_to_openai_messages(openai_messages.extend(parts_messages)continue- # Handle attachmentsparts: list[dict[Any, Any]] = []if not message.parts or len(message.parts) == 0:parts.append({"type": "text", "text": message.content})else:- parts.extend(- get_openai_messages_from_parts(message.role, message.parts)- )+ parts_dict = [asdict(part) for part in message.parts]+ parts.extend(parts_dict)+ # Handle attachmentsfor attachment in message.attachments:content_type = attachment.content_type or "text/plain"diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.pyindex b58f5857f3..52ad53f038 100644--- a/tests/_ai/test_chat_convert.py+++ b/tests/_ai/test_chat_convert.py@@ -63,28 +63,97 @@ def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):assert len(result) == 2- # Check user message- assert result[0]["role"] == "user"- assert len(result[0]["content"]) == 3- assert result[0]["content"][0] == {- "type": "text",- "text": "Hello, I have a question.",- }- assert result[0]["content"][1] == {- "type": "image_url",- "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},- }- assert result[0]["content"][2] == {- "type": "text",- "text": "A\n1\n2\n3\n",- }+ assert result == [+ # User message+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Hello, I have a question."},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},+ },+ {"type": "text", "text": "A\n1\n2\n3\n"},+ ],+ },+ # Assistant message+ {+ "role": "assistant",+ "content": "Sure, I'd be happy to help. What's your question?",+ },+ ]- # Check assistant message- assert result[1]["role"] == "assistant"- assert (- result[1]["content"]- == "Sure, I'd be happy to help. What's your question?"- )++def test_convert_to_openai_messages_with_parts_and_attachments():+ sample_messages = [+ ChatMessage(+ role="user",+ content="Message with parts and attachments",+ attachments=[+ ChatAttachment(+ name="image.png",+ content_type="image/png",+ url=f"data:image/png;base64,{base64.b64encode(b'hello')}",+ ),+ ],+ parts=[+ TextPart(+ type="text", text="Message with parts and attachments"+ ),+ ReasoningPart(+ type="reasoning",+ reasoning="Deep thinking process",+ details=[+ ReasoningDetails(+ type="text", text="Analysis", signature=None+ )+ ],+ ),+ ToolInvocationPart(+ type="tool-invocation",+ tool_invocation=ToolInvocationResult(+ state="result",+ result={"answer": 4},+ tool_call_id="call_123",+ tool_name="calculator",+ step=1,+ args={"expression": "2 + 2"},+ ),+ ),+ ],+ ),+ ]+ result = convert_to_openai_messages(sample_messages)+ assert result == [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Message with parts and attachments"},+ {+ "type": "reasoning",+ "reasoning": "Deep thinking process",+ "details": [+ {"type": "text", "text": "Analysis", "signature": None}+ ],+ },+ {+ "type": "tool-invocation",+ "tool_invocation": {+ "state": "result",+ "result": {"answer": 4},+ "tool_call_id": "call_123",+ "tool_name": "calculator",+ "step": 1,+ "args": {"expression": "2 + 2"},+ },+ },+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},+ },+ ],+ }+ ]def test_convert_to_anthropic_messages(sample_messages: list[ChatMessage]):