Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

attachments in chat sidebar (#6181)

Shahmir Varqha4dcf8ec0a752
diff --git a/frontend/src/__tests__/chat-utils.test.ts b/frontend/src/__tests__/chat-utils.test.ts
index 6342d3573d..91ec485b98 100644
--- a/frontend/src/__tests__/chat-utils.test.ts
+++ b/frontend/src/__tests__/chat-utils.test.ts
@@ -58,13 +58,13 @@ describe("addMessageToChat", () => {
};
it("should add a new message to an existing chat", () => {
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-4",
- "user",
- "New message",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-4",
+ role: "user",
+ content: "New message",
+ });
expect(result.chats).toHaveLength(2);
const updatedChat = result.chats.get(CHAT_1);
@@ -81,13 +81,13 @@ describe("addMessageToChat", () => {
});
it("should update an existing message", () => {
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-1",
- "user",
- "Updated content",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-1",
+ role: "user",
+ content: "Updated content",
+ });
expect(result.chats).toHaveLength(2);
const updatedChat = result.chats.get(CHAT_1);
@@ -105,14 +105,14 @@ describe("addMessageToChat", () => {
it("should handle message parts", () => {
const parts = [{ type: "text" as const, text: "Part content" }];
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-5",
- "assistant",
- "Message with parts",
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-5",
+ role: "assistant",
+ content: "Message with parts",
parts,
- );
+ });
const updatedChat = result.chats.get(CHAT_1);
expect(updatedChat?.messages[2].parts).toEqual(parts);
@@ -140,64 +140,64 @@ describe("addMessageToChat", () => {
]),
};
- const result = addMessageToChat(
- stateWithParts,
- CHAT_1,
- "msg-1",
- "user",
- "Updated content",
- updatedParts,
- );
+ const result = addMessageToChat({
+ chatState: stateWithParts,
+ chatId: CHAT_1,
+ messageId: "msg-1",
+ role: "user",
+ content: "Updated content",
+ parts: updatedParts,
+ });
const updatedChat = result.chats.get(CHAT_1);
expect(updatedChat?.messages[0].parts).toEqual(updatedParts);
});
it("should return unchanged state when chatId is null", () => {
- const result = addMessageToChat(
- mockChatState,
- null,
- "msg-4",
- "user",
- "New message",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: null,
+ messageId: "msg-4",
+ role: "user",
+ content: "New message",
+ });
expect(result).toEqual(mockChatState);
});
it("should return unchanged state when chatId does not exist", () => {
- const result = addMessageToChat(
- mockChatState,
- "non-existent-chat" as ChatId,
- "msg-4",
- "user",
- "New message",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: "non-existent-chat" as ChatId,
+ messageId: "msg-4",
+ role: "user",
+ content: "New message",
+ });
expect(result).toEqual(mockChatState);
});
it("should not modify other chats when updating a specific chat", () => {
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-4",
- "user",
- "New message",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-4",
+ role: "user",
+ content: "New message",
+ });
const unchangedChat = result.chats.get(CHAT_2);
expect(unchangedChat).toEqual([...mockChatState.chats.values()][1]);
});
it("should preserve message order when adding new messages", () => {
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-4",
- "user",
- "New message",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-4",
+ role: "user",
+ content: "New message",
+ });
const updatedChat = result.chats.get(CHAT_1);
expect(updatedChat?.messages[0].id).toBe("msg-1");
@@ -220,13 +220,13 @@ describe("addMessageToChat", () => {
activeChatId: chatId,
};
- const result = addMessageToChat(
- emptyChatState,
- chatId,
- "msg-1",
- "user",
- "First message",
- );
+ const result = addMessageToChat({
+ chatState: emptyChatState,
+ chatId: chatId,
+ messageId: "msg-1",
+ role: "user",
+ content: "First message",
+ });
const updatedChat = result.chats.get(chatId);
expect(updatedChat?.messages).toHaveLength(1);
@@ -234,13 +234,13 @@ describe("addMessageToChat", () => {
});
it("should handle different message roles", () => {
- const result = addMessageToChat(
- mockChatState,
- CHAT_1,
- "msg-4",
- "assistant",
- "Assistant response",
- );
+ const result = addMessageToChat({
+ chatState: mockChatState,
+ chatId: CHAT_1,
+ messageId: "msg-4",
+ role: "assistant",
+ content: "Assistant response",
+ });
const updatedChat = result.chats.get(CHAT_1);
expect(updatedChat?.messages[2].role).toBe("assistant");
diff --git a/frontend/src/components/chat/chat-components.tsx b/frontend/src/components/chat/chat-components.tsx
new file mode 100644
index 0000000000..65af3e32b7
--- /dev/null
+++ b/frontend/src/components/chat/chat-components.tsx
@@ -0,0 +1,71 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { FileIcon, FileTextIcon, ImageIcon, XIcon } from "lucide-react";
+import { useState } from "react";
+import type { ChatAttachment } from "@/core/ai/types";
+import { cn } from "@/utils/cn";
+
+export const AttachmentRenderer = ({
+ attachment,
+}: {
+ attachment: ChatAttachment;
+}) => {
+ if (attachment.contentType?.startsWith("image/")) {
+ return (
+ <img
+ src={attachment.url}
+ alt={attachment.name}
+ className="max-h-[100px] max-w-[100px] object-contain mb-1.5"
+ />
+ );
+ }
+
+ return (
+ <div className="flex flex-row gap-1 items-center text-xs">
+ <FileIcon className="h-3 w-3 mt-0.5" />
+ {attachment.name}
+ </div>
+ );
+};
+
+export const FileAttachmentPill = ({
+ file,
+ className,
+ onRemove,
+}: {
+ file: File;
+ className?: string;
+ onRemove: () => void;
+}) => {
+ const [isHovered, setIsHovered] = useState(false);
+
+ return (
+ <div
+ className={cn(
+ "py-1 px-1.5 bg-muted rounded-md cursor-pointer flex flex-row gap-1 items-center text-xs",
+ className,
+ )}
+ onMouseEnter={() => setIsHovered(true)}
+ onMouseLeave={() => setIsHovered(false)}
+ >
+ {isHovered ? (
+ <XIcon className="h-3 w-3 mt-0.5" onClick={onRemove} />
+ ) : (
+ renderFileIcon(file)
+ )}
+ {file.name}
+ </div>
+ );
+};
+
+function renderFileIcon(file: File): React.ReactNode {
+ const classNames = "h-3 w-3 mt-0.5";
+
+ if (file.type.startsWith("image/")) {
+ return <ImageIcon className={classNames} />;
+ } else if (file.type.startsWith("text/")) {
+ return <FileTextIcon className={classNames} />;
+ }
+
+ return <FileIcon className={classNames} />;
+}
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsx
index 51693ba1c3..7134f1552d 100644
--- a/frontend/src/components/chat/chat-panel.tsx
+++ b/frontend/src/components/chat/chat-panel.tsx
@@ -9,6 +9,7 @@ import {
BotMessageSquareIcon,
ClockIcon,
Loader2,
+ PaperclipIcon,
PlusIcon,
SendIcon,
SettingsIcon,
@@ -41,6 +42,7 @@ import {
} from "@/components/ui/select";
import { addMessageToChat } from "@/core/ai/chat-utils";
import { useModelChange } from "@/core/ai/config";
+import { AiModelId, type ProviderId } from "@/core/ai/ids/ids";
import {
activeChatAtom,
type Chat,
@@ -48,7 +50,7 @@ import {
type ChatState,
chatStateAtom,
} from "@/core/ai/state";
-import { getCodes } from "@/core/codemirror/copilot/getCodes";
+import type { ChatAttachment } from "@/core/ai/types";
import { aiAtom, aiEnabledAtom } from "@/core/config/config";
import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";
import { FeatureFlagged } from "@/core/config/feature-flag";
@@ -57,6 +59,7 @@ import { useRuntimeManager } from "@/core/runtime/config";
import { ErrorBanner } from "@/plugins/impl/common/error-banner";
import { cn } from "@/utils/cn";
import { timeAgo } from "@/utils/dates";
+import { blobToString } from "@/utils/fileToBase64";
import { Logger } from "@/utils/Logger";
import { generateUUID } from "@/utils/uuid";
import { AIModelDropdown } from "../ai/ai-model-dropdown";
@@ -65,11 +68,23 @@ import { PromptInput } from "../editor/ai/add-cell-with-ai";
import { getAICompletionBody } from "../editor/ai/completion-utils";
import { PanelEmptyState } from "../editor/chrome/panels/empty-state";
import { CopyClipboardIcon } from "../icons/copy-icon";
+import { Input } from "../ui/input";
import { Tooltip, TooltipProvider } from "../ui/tooltip";
+import { toast } from "../ui/use-toast";
+import { AttachmentRenderer, FileAttachmentPill } from "./chat-components";
import { MarkdownRenderer } from "./markdown-renderer";
import { ReasoningAccordion } from "./reasoning-accordion";
import { ToolCallAccordion } from "./tool-call-accordion";
+// Default mode for the AI
+const DEFAULT_MODE = "manual";
+
+// We need to modify the backend to support attachments for other providers
+// And other types
+const PROVIDERS_THAT_SUPPORT_ATTACHMENTS = new Set<ProviderId>(["openai"]);
+const SUPPORTED_ATTACHMENT_TYPES = ["image/*", "text/*"];
+const MAX_ATTACHMENT_SIZE = 1024 * 1024 * 50; // 50MB
+
interface ChatHeaderProps {
onNewChat: () => void;
activeChatId: ChatId | undefined;
@@ -168,6 +183,9 @@ const ChatMessage: React.FC<ChatMessageProps> = memo(
>
{message.role === "user" ? (
<div className="w-[95%] bg-background border p-1 rounded-sm">
+ {message.experimental_attachments?.map((attachment, idx) => (
+ <AttachmentRenderer attachment={attachment} key={idx} />
+ ))}
<PromptInput
key={message.id}
value={message.content}
@@ -243,15 +261,24 @@ interface ChatInputFooterProps {
onSendClick: () => void;
isLoading: boolean;
onStop: () => void;
+ handleFileChange: (event: React.ChangeEvent<HTMLInputElement>) => void;
+ fileInputRef: React.RefObject<HTMLInputElement | null>;
}
-const DEFAULT_MODE = "manual";
-
const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(
- ({ isEmpty, onSendClick, isLoading, onStop }) => {
+ ({
+ isEmpty,
+ onSendClick,
+ isLoading,
+ onStop,
+ fileInputRef,
+ handleFileChange,
+ }) => {
const ai = useAtomValue(aiAtom);
const currentMode = ai?.mode || DEFAULT_MODE;
const currentModel = ai?.models?.chat_model || DEFAULT_AI_MODEL;
+ const currentProvider = AiModelId.parse(currentModel).providerId;
+
const { saveModeChange, saveModelChange } = useModelChange();
const modeOptions = [
@@ -268,8 +295,11 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(
},
];
+ const isAttachmentSupported =
+ PROVIDERS_THAT_SUPPORT_ATTACHMENTS.has(currentProvider);
+
return (
- <div className="px-3 py-2 border-t border-border/20 flex items-center justify-between">
+ <div className="px-3 py-2 border-t border-border/20 flex flex-row items-center justify-between">
<div className="flex items-center gap-2">
<FeatureFlagged feature="mcp_docs">
<Select value={currentMode} onValueChange={saveModeChange}>
@@ -307,19 +337,43 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(
forRole="chat"
/>
</div>
- <Button
- variant="ghost"
- size="sm"
- className="h-6 w-6 p-0 hover:bg-muted/30"
- onClick={isLoading ? onStop : onSendClick}
- disabled={isLoading ? false : isEmpty}
- >
- {isLoading ? (
- <SquareIcon className="h-3 w-3 fill-current" />
- ) : (
- <SendIcon className="h-3 w-3" />
+ <div className="flex flex-row">
+ {isAttachmentSupported && (
+ <>
+ <Button
+ variant="text"
+ size="icon"
+ className="cursor-pointer"
+ onClick={() => fileInputRef.current?.click()}
+ title="Attach a file"
+ >
+ <PaperclipIcon className="h-3.5 w-3.5" />
+ </Button>
+ <Input
+ ref={fileInputRef}
+ type="file"
+ multiple={true}
+ hidden={true}
+ onChange={handleFileChange}
+ accept={SUPPORTED_ATTACHMENT_TYPES.join(",")}
+ />
+ </>
)}
- </Button>
+
+ <Button
+ variant="text"
+ size="sm"
+ className="h-6 w-6 p-0 hover:bg-muted/30 cursor-pointer"
+ onClick={isLoading ? onStop : onSendClick}
+ disabled={isLoading ? false : isEmpty}
+ >
+ {isLoading ? (
+ <SquareIcon className="h-3 w-3 fill-current" />
+ ) : (
+ <SendIcon className="h-3 w-3" />
+ )}
+ </Button>
+ </div>
</div>
);
},
@@ -328,16 +382,33 @@ const ChatInputFooter: React.FC<ChatInputFooterProps> = memo(
ChatInputFooter.displayName = "ChatInputFooter";
interface ChatInputProps {
+ placeholder?: string;
input: string;
+ inputClassName?: string;
setInput: (value: string) => void;
onSubmit: (e: KeyboardEvent | undefined, value: string) => void;
inputRef: React.RefObject<ReactCodeMirrorRef | null>;
isLoading: boolean;
onStop: () => void;
+ onClose: () => void;
+ fileInputRef: React.RefObject<HTMLInputElement | null>;
+ handleFileChange: (event: React.ChangeEvent<HTMLInputElement>) => void;
}
const ChatInput: React.FC<ChatInputProps> = memo(
- ({ input, setInput, onSubmit, inputRef, isLoading, onStop }) => {
+ ({
+ placeholder,
+ input,
+ inputClassName,
+ setInput,
+ onSubmit,
+ inputRef,
+ isLoading,
+ onStop,
+ fileInputRef,
+ handleFileChange,
+ onClose,
+ }) => {
const handleSendClick = useEvent(() => {
if (input.trim()) {
onSubmit(undefined, input);
@@ -346,14 +417,14 @@ const ChatInput: React.FC<ChatInputProps> = memo(
return (
<div className="border-t relative shrink-0 min-h-[80px] flex flex-col">
- <div className="px-2 py-3 flex-1">
+ <div className={cn("px-2 py-3 flex-1", inputClassName)}>
<PromptInput
inputRef={inputRef}
value={input}
onChange={setInput}
onSubmit={onSubmit}
- onClose={() => inputRef.current?.editor?.blur()}
- placeholder="Type your message..."
+ onClose={onClose}
+ placeholder={placeholder || "Type your message..."}
/>
</div>
<ChatInputFooter
@@ -361,6 +432,8 @@ const ChatInput: React.FC<ChatInputProps> = memo(
onSendClick={handleSendClick}
isLoading={isLoading}
onStop={onStop}
+ fileInputRef={fileInputRef}
+ handleFileChange={handleFileChange}
/>
</div>
);
@@ -395,9 +468,11 @@ const ChatPanelBody = () => {
const [chatState, setChatState] = useAtom(chatStateAtom);
const [activeChat, setActiveChat] = useAtom(activeChatAtom);
const [newThreadInput, setNewThreadInput] = useState("");
+ const [files, setFiles] = useState<File[]>();
const newThreadInputRef = useRef<ReactCodeMirrorRef>(null);
const newMessageInputRef = useRef<ReactCodeMirrorRef>(null);
const scrollContainerRef = useRef<HTMLDivElement>(null);
+ const fileInputRef = useRef<HTMLInputElement>(null);
const messagesEndRef = useRef<HTMLDivElement>(null);
const runtimeManager = useRuntimeManager();
const { invokeAiTool } = useRequestClient();
@@ -423,24 +498,36 @@ const ChatPanelBody = () => {
api: runtimeManager.getAiURL("chat").toString(),
headers: runtimeManager.headers(),
experimental_prepareRequestBody: (options) => {
- return {
+ const completionBody = getAICompletionBody({
+ input: options.messages.map((m) => m.content).join("\n"),
+ });
+
+ // Backend accepts attachments, so we convert the key
+ const newOptions = {
...options,
- ...getAICompletionBody({
- input: options.messages.map((m) => m.content).join("\n"),
- }),
- includeOtherCode: getCodes(""),
+ messages: options.messages.map((m) => ({
+ ...m,
+ attachments: m.experimental_attachments,
+ experimental_attachments: undefined,
+ })),
+ };
+
+ return {
+ ...newOptions,
+ ...completionBody,
};
},
onFinish: (message) => {
setChatState((prev) => {
- return addMessageToChat(
- prev,
- prev.activeChatId,
- message.id,
- "assistant",
- message.content,
- message.parts,
- );
+ return addMessageToChat({
+ chatState: prev,
+ chatId: prev.activeChatId,
+ messageId: message.id,
+ role: "assistant",
+ content: message.content,
+ parts: message.parts,
+ attachments: message.experimental_attachments,
+ });
});
},
onToolCall: async ({ toolCall }) => {
@@ -465,6 +552,36 @@ const ChatPanelBody = () => {
},
});
+ const handleFileChange = useEvent(
+ (event: React.ChangeEvent<HTMLInputElement>) => {
+ const files = event.target.files;
+ if (!files) {
+ return;
+ }
+
+ let fileSize = 0;
+ for (const file of files) {
+ fileSize += file.size;
+ }
+
+ if (fileSize > MAX_ATTACHMENT_SIZE) {
+ toast({
+ title: "File size exceeds 50MB limit",
+ description: "Please remove some files and try again.",
+ });
+ return;
+ }
+
+ setFiles([...files]);
+ },
+ );
+
+ const removeFile = useEvent((file: File) => {
+ if (files) {
+ setFiles(files.filter((f) => f !== file));
+ }
+ });
+
const isLoading = status === "submitted" || status === "streaming";
// Sync user messages from useChat to storage when they become available
@@ -497,13 +614,15 @@ const ChatPanelBody = () => {
let result = prev;
for (const userMessage of missingUserMessages) {
- result = addMessageToChat(
- result,
- prev.activeChatId,
- userMessage.id,
- "user",
- userMessage.content,
- );
+ result = addMessageToChat({
+ chatState: result,
+ chatId: prev.activeChatId,
+ messageId: userMessage.id,
+ role: "user",
+ content: userMessage.content,
+ parts: userMessage.parts,
+ attachments: userMessage.experimental_attachments,
+ });
}
return result;
@@ -533,7 +652,10 @@ const ChatPanelBody = () => {
requestAnimationFrame(scrollToBottom);
}, [chatState.activeChatId]);
- const createNewThread = (initialMessage: string) => {
+ const createNewThread = async (
+ initialMessage: string,
+ initialAttachments?: File[],
+ ) => {
const CURRENT_TIME = Date.now();
const newChat: Chat = {
id: generateUUID() as ChatId,
@@ -558,12 +680,19 @@ const ChatPanelBody = () => {
return newState;
});
+ const attachments =
+ initialAttachments && initialAttachments.length > 0
+ ? await convertToChatAttachments(initialAttachments)
+ : undefined;
+
// Trigger AI conversation with append
append({
id: generateUUID(),
role: "user",
content: initialMessage,
+ experimental_attachments: attachments,
});
+ setFiles(undefined);
setInput("");
};
@@ -571,9 +700,13 @@ const ChatPanelBody = () => {
setActiveChat(null);
setInput("");
setNewThreadInput("");
+ setFiles(undefined);
});
const handleMessageEdit = useEvent((index: number, newValue: string) => {
+ const editedMessage = messages[index];
+ const attachments = editedMessage.experimental_attachments;
+
// Truncate both useChat and storage
setMessages((messages) => messages.slice(0, index));
const activeChatId = chatState.activeChatId;
@@ -599,18 +732,26 @@ const ChatPanelBody = () => {
append({
role: "user",
content: newValue,
+ experimental_attachments: attachments,
});
});
const handleChatInputSubmit = useEvent(
- (e: KeyboardEvent | undefined, newValue: string): void => {
+ async (e: KeyboardEvent | undefined, newValue: string): Promise<void> => {
if (!newValue.trim()) {
return;
}
if (newMessageInputRef.current?.view) {
storePrompt(newMessageInputRef.current.view);
}
- handleSubmit(e);
+ const attachments = files
+ ? await convertToChatAttachments(files)
+ : undefined;
+
+ handleSubmit(e, {
+ experimental_attachments: attachments,
+ });
+ setFiles(undefined);
},
);
@@ -625,7 +766,7 @@ const ChatPanelBody = () => {
if (newThreadInputRef.current?.view) {
storePrompt(newThreadInputRef.current.view);
}
- createNewThread(newThreadInput.trim());
+ createNewThread(newThreadInput.trim(), files);
});
const handleOnCloseThread = () => newThreadInputRef.current?.editor?.blur();
@@ -636,6 +777,53 @@ const ChatPanelBody = () => {
);
}, [chatState.chats]);
+ const isNewThread = messages.length === 0;
+ const chatInput = isNewThread ? (
+ <ChatInput
+ key="new-thread-input"
+ placeholder="Ask anything, @ to include context about tables or dataframes"
+ input={newThreadInput}
+ inputRef={newThreadInputRef}
+ inputClassName="px-1 py-0"
+ setInput={setNewThreadInput}
+ onSubmit={handleNewThreadSubmit}
+ isLoading={isLoading}
+ onStop={stop}
+ fileInputRef={fileInputRef}
+ handleFileChange={handleFileChange}
+ onClose={handleOnCloseThread}
+ />
+ ) : (
+ <ChatInput
+ input={input}
+ setInput={setInput}
+ onSubmit={handleChatInputSubmit}
+ inputRef={newMessageInputRef}
+ isLoading={isLoading}
+ onStop={stop}
+ onClose={() => newMessageInputRef.current?.editor?.blur()}
+ fileInputRef={fileInputRef}
+ handleFileChange={handleFileChange}
+ />
+ );
+
+ const filesPills = files && files.length > 0 && (
+ <div
+ className={cn(
+ "flex flex-row gap-1 flex-wrap p-1",
+ isNewThread && "py-2 px-1",
+ )}
+ >
+ {files?.map((file) => (
+ <FileAttachmentPill
+ file={file}
+ key={file.name}
+ onRemove={() => removeFile(file)}
+ />
+ ))}
+ </div>
+ );
+
return (
<div className="flex flex-col h-[calc(100%-53px)]">
<TooltipProvider>
@@ -651,25 +839,10 @@ const ChatPanelBody = () => {
className="flex-1 px-3 bg-(--slate-1) gap-4 py-3 flex flex-col overflow-y-auto"
ref={scrollContainerRef}
>
- {(!messages || messages.length === 0) && (
- <div className="flex flex-col rounded-md border bg-background">
- <div className="px-1">
- <PromptInput
- inputRef={newThreadInputRef}
- key="new-thread-input"
- value={newThreadInput}
- placeholder="Ask anything, @ to include context about tables or dataframes"
- onClose={handleOnCloseThread}
- onChange={setNewThreadInput}
- onSubmit={handleNewThreadSubmit}
- />
- </div>
- <ChatInputFooter
- isEmpty={!newThreadInput.trim()}
- onSendClick={handleNewThreadSubmit}
- isLoading={isLoading}
- onStop={stop}
- />
+ {isNewThread && (
+ <div className="rounded-md border bg-background">
+ {filesPills}
+ {chatInput}
</div>
)}
@@ -712,15 +885,12 @@ const ChatPanelBody = () => {
</div>
)}
- {messages && messages.length > 0 && (
- <ChatInput
- input={input}
- setInput={setInput}
- onSubmit={handleChatInputSubmit}
- inputRef={newMessageInputRef}
- isLoading={isLoading}
- onStop={stop}
- />
+ {/* For existing threads, we place the chat input at the bottom */}
+ {!isNewThread && (
+ <>
+ {filesPills}
+ {chatInput}
+ </>
)}
</div>
);
@@ -749,3 +919,19 @@ function isLastMessageReasoning(messages: Message[]): boolean {
const lastPart = parts[parts.length - 1];
return lastPart.type === "reasoning";
}
+
+async function convertToChatAttachments(
+ files: File[],
+): Promise<ChatAttachment[]> {
+ const attachments = await Promise.all(
+ files.map(async (file) => {
+ return {
+ name: file.name,
+ url: await blobToString(file, "dataUrl"),
+ contentType: file.type,
+ };
+ }),
+ );
+
+ return attachments;
+}
diff --git a/frontend/src/core/ai/chat-utils.ts b/frontend/src/core/ai/chat-utils.ts
index 51892b46f4..f6b9e0656e 100644
--- a/frontend/src/core/ai/chat-utils.ts
+++ b/frontend/src/core/ai/chat-utils.ts
@@ -3,15 +3,27 @@
import type { Message as AIMessage } from "@ai-sdk/react";
import { Logger } from "@/utils/Logger";
import type { ChatId, ChatState } from "./state";
+import type { ChatAttachment } from "./types";
-export const addMessageToChat = (
- chatState: ChatState,
- chatId: ChatId | null,
- messageId: string,
- role: "user" | "assistant",
- content: string,
- parts?: AIMessage["parts"],
-): ChatState => {
+interface AddMessageToChatParams {
+ chatState: ChatState;
+ chatId: ChatId | null;
+ messageId: string;
+ role: "user" | "assistant";
+ content: string;
+ parts?: AIMessage["parts"];
+ attachments?: ChatAttachment[];
+}
+
+export const addMessageToChat = ({
+ chatState,
+ chatId,
+ messageId,
+ role,
+ content,
+ parts,
+ attachments,
+}: AddMessageToChatParams): ChatState => {
if (!chatId) {
Logger.warn("No active chat");
return chatState;
@@ -42,6 +54,7 @@ export const addMessageToChat = (
content,
timestamp: timestamp,
parts,
+ attachments,
},
],
updatedAt: timestamp,
@@ -53,6 +66,7 @@ export const addMessageToChat = (
...newMessages[messageIndex],
content,
parts,
+ attachments,
};
newChats.set(chat.id, {
...chat,
diff --git a/frontend/src/core/ai/state.ts b/frontend/src/core/ai/state.ts
index 7e6b3cbbcb..c4a8123de3 100644
--- a/frontend/src/core/ai/state.ts
+++ b/frontend/src/core/ai/state.ts
@@ -6,6 +6,7 @@ import { atomWithStorage } from "jotai/utils";
import { adaptForLocalStorage } from "@/utils/storage";
import type { TypedString } from "@/utils/typed";
import type { CellId } from "../cells/ids";
+import type { ChatAttachment } from "./types";
const KEY = "marimo:ai:chatState:v4";
@@ -28,6 +29,7 @@ export interface Message {
content: string;
timestamp: number;
parts?: AIMessage["parts"];
+ attachments?: ChatAttachment[];
}
export interface Chat {
diff --git a/frontend/src/core/ai/types.ts b/frontend/src/core/ai/types.ts
new file mode 100644
index 0000000000..9f4fbb78d6
--- /dev/null
+++ b/frontend/src/core/ai/types.ts
@@ -0,0 +1,17 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+export interface ChatAttachment {
+ /**
+ * The name of the attachment, usually the file name.
+ */
+ name?: string;
+ /**
+ * A string indicating the [media type](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type).
+ * By default, it's extracted from the pathname's extension.
+ */
+ contentType?: string;
+ /**
+ * The URL of the attachment. It can either be a URL to a hosted file or a [Data URL](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs).
+ */
+ url: string;
+}
diff --git a/frontend/src/plugins/impl/MicrophonePlugin.tsx b/frontend/src/plugins/impl/MicrophonePlugin.tsx
index ba0f370c97..87a65a4052 100644
--- a/frontend/src/plugins/impl/MicrophonePlugin.tsx
+++ b/frontend/src/plugins/impl/MicrophonePlugin.tsx
@@ -5,7 +5,7 @@ import { z } from "zod";
import { AudioRecorder } from "@/components/audio/audio-recorder";
import { useAudioRecorder } from "@/hooks/useAudioRecorder";
import type { IPlugin, IPluginProps } from "@/plugins/types";
-import { blobToBase64 } from "@/utils/fileToBase64";
+import { blobToString } from "@/utils/fileToBase64";
import { Labeled } from "./common/labeled";
/**
@@ -33,7 +33,7 @@ const Microphone = ({ setValue, data }: IPluginProps<Value, Data>) => {
const { start, stop, pauseResume, recordingStatus, recordingTime, allowed } =
useAudioRecorder({
onDone: async (file) => {
- const base64 = await blobToBase64(file);
+ const base64 = await blobToString(file, "base64");
setValue(base64);
},
});
diff --git a/frontend/src/plugins/impl/chat/chat-ui.tsx b/frontend/src/plugins/impl/chat/chat-ui.tsx
index baf75dbbb5..c3d0aafac9 100644
--- a/frontend/src/plugins/impl/chat/chat-ui.tsx
+++ b/frontend/src/plugins/impl/chat/chat-ui.tsx
@@ -40,6 +40,7 @@ import {
} from "@/components/ui/popover";
import { Tooltip } from "@/components/ui/tooltip";
import { toast } from "@/components/ui/use-toast";
+import type { ChatAttachment } from "@/core/ai/types";
import { moveToEndOfEditor } from "@/core/codemirror/utils";
import { useAsyncData } from "@/hooks/useAsyncData";
import { renderHTML } from "@/plugins/core/RenderHTML";
@@ -49,12 +50,7 @@ import { Logger } from "@/utils/Logger";
import { Objects } from "@/utils/objects";
import { ErrorBanner } from "../common/error-banner";
import type { PluginFunctions } from "./ChatPlugin";
-import type {
- ChatAttachment,
- ChatConfig,
- ChatMessage,
- ChatRole,
-} from "./types";
+import type { ChatConfig, ChatMessage, ChatRole } from "./types";
interface Props extends PluginFunctions {
prompts: string[];
diff --git a/frontend/src/plugins/impl/chat/types.ts b/frontend/src/plugins/impl/chat/types.ts
index ce0afcd448..2c035bdcc7 100644
--- a/frontend/src/plugins/impl/chat/types.ts
+++ b/frontend/src/plugins/impl/chat/types.ts
@@ -1,4 +1,6 @@
/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { ChatAttachment } from "@/core/ai/types";
export type ChatRole = "system" | "user" | "assistant";
export interface ChatMessage {
@@ -7,12 +9,6 @@ export interface ChatMessage {
attachments?: ChatAttachment[];
}
-export interface ChatAttachment {
- name?: string;
- contentType?: string;
- url: string;
-}
-
export interface SendMessageRequest {
messages: ChatMessage[];
config: {
diff --git a/frontend/src/utils/__tests__/blob.test.ts b/frontend/src/utils/__tests__/blob.test.ts
index 48f46eea96..1e30336dd5 100644
--- a/frontend/src/utils/__tests__/blob.test.ts
+++ b/frontend/src/utils/__tests__/blob.test.ts
@@ -1,6 +1,7 @@
/* Copyright 2024 Marimo. All rights reserved. */
import { beforeEach, describe, expect, test } from "vitest";
import { deserializeBlob, serializeBlob } from "../blob";
+import { blobToString } from "../fileToBase64";
describe("Blob serialization and deserialization", () => {
const testString = "Hello, world!";
@@ -50,3 +51,39 @@ describe("Blob serialization and deserialization", () => {
expect(deserialized.type).toBe(imageBlob.type);
});
});
+
+describe("blobToString", () => {
+ const testString = "Hello, world!";
+ const mimeType = "text/plain";
+
+ test("should convert a Blob to a base64 string", async () => {
+ const blob = new Blob([testString], { type: mimeType });
+ const base64 = await blobToString(blob, "base64");
+ expect(base64).toBe(btoa(testString));
+ });
+
+ test("should convert a File to a base64 string", async () => {
+ const file = new File([testString], "test.txt", { type: mimeType });
+ const base64 = await blobToString(file, "base64");
+ expect(base64).toBe(btoa(testString));
+ });
+
+ test("should convert a Blob to a data URL", async () => {
+ const blob = new Blob([testString], { type: mimeType });
+ const dataUrl = await blobToString(blob, "dataUrl");
+ expect(dataUrl).toBe(`data:${mimeType};base64,${btoa(testString)}`);
+ });
+
+ test("should handle empty Blob", async () => {
+ const blob = new Blob([], { type: mimeType });
+ const base64 = await blobToString(blob, "base64");
+ expect(base64).toBe("");
+ });
+
+ test("should handle binary data", async () => {
+ const binaryData = new Uint8Array([0, 1, 2, 3, 4, 5]);
+ const blob = new Blob([binaryData], { type: "application/octet-stream" });
+ const base64 = await blobToString(blob, "base64");
+ expect(base64).toBe(btoa(String.fromCharCode(...binaryData)));
+ });
+});
diff --git a/frontend/src/utils/fileToBase64.ts b/frontend/src/utils/fileToBase64.ts
index 4fac3f060f..6038f95cca 100644
--- a/frontend/src/utils/fileToBase64.ts
+++ b/frontend/src/utils/fileToBase64.ts
@@ -1,14 +1,29 @@
/* Copyright 2024 Marimo. All rights reserved. */
-export function blobToBase64(blob: Blob): Promise<string> {
+
+/**
+ * Converts a Blob or File to either a base64-encoded string or a data URL.
+ *
+ * @param input - The Blob or File to convert.
+ * @param format - The output format: "base64" for the base64-encoded string, "dataUrl" for the full data URL.
+ * @returns A promise that resolves to the requested string representation.
+ */
+export function blobToString(
+ input: Blob | File,
+ format: "base64" | "dataUrl",
+): Promise<string> {
return new Promise((resolve) => {
const reader = new FileReader();
- reader.readAsDataURL(blob);
+ reader.readAsDataURL(input);
reader.onload = (e) => {
if (e.target?.result) {
const dataURL = e.target.result as string;
- // Get contents from a string of the form: data:*/*;base64,contents
- const b64EncodedContents = dataURL.slice(dataURL.indexOf(",") + 1);
- resolve(b64EncodedContents);
+ if (format === "base64") {
+ // Extract base64 content from data URL. data:*/*;base64,contents
+ const base64Content = dataURL.slice(dataURL.indexOf(",") + 1);
+ resolve(base64Content);
+ } else {
+ resolve(dataURL);
+ }
}
};
});
@@ -22,7 +37,7 @@ export function blobToBase64(blob: Blob): Promise<string> {
export function filesToBase64(files: File[]): Promise<Array<[string, string]>> {
return Promise.all(
files.map((file) =>
- blobToBase64(file).then(
+ blobToString(file, "base64").then(
(contents) => [file.name, contents] as [string, string],
),
),
diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.py
index b01e7149c7..a9c250ae3e 100644
--- a/marimo/_ai/_convert.py
+++ b/marimo/_ai/_convert.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import base64
import json
+from dataclasses import asdict
from typing import TYPE_CHECKING, Any, Literal, Union, cast
from marimo._ai._types import (
@@ -85,15 +86,14 @@ def convert_to_openai_messages(
openai_messages.extend(parts_messages)
continue
- # Handle attachments
parts: list[dict[Any, Any]] = []
if not message.parts or len(message.parts) == 0:
parts.append({"type": "text", "text": message.content})
else:
- parts.extend(
- get_openai_messages_from_parts(message.role, message.parts)
- )
+ parts_dict = [asdict(part) for part in message.parts]
+ parts.extend(parts_dict)
+ # Handle attachments
for attachment in message.attachments:
content_type = attachment.content_type or "text/plain"
diff --git a/tests/_ai/test_chat_convert.py b/tests/_ai/test_chat_convert.py
index b58f5857f3..52ad53f038 100644
--- a/tests/_ai/test_chat_convert.py
+++ b/tests/_ai/test_chat_convert.py
@@ -63,28 +63,97 @@ def test_convert_to_openai_messages(sample_messages: list[ChatMessage]):
assert len(result) == 2
- # Check user message
- assert result[0]["role"] == "user"
- assert len(result[0]["content"]) == 3
- assert result[0]["content"][0] == {
- "type": "text",
- "text": "Hello, I have a question.",
- }
- assert result[0]["content"][1] == {
- "type": "image_url",
- "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},
- }
- assert result[0]["content"][2] == {
- "type": "text",
- "text": "A\n1\n2\n3\n",
- }
+ assert result == [
+ # User message
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Hello, I have a question."},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},
+ },
+ {"type": "text", "text": "A\n1\n2\n3\n"},
+ ],
+ },
+ # Assistant message
+ {
+ "role": "assistant",
+ "content": "Sure, I'd be happy to help. What's your question?",
+ },
+ ]
- # Check assistant message
- assert result[1]["role"] == "assistant"
- assert (
- result[1]["content"]
- == "Sure, I'd be happy to help. What's your question?"
- )
+
+def test_convert_to_openai_messages_with_parts_and_attachments():
+ sample_messages = [
+ ChatMessage(
+ role="user",
+ content="Message with parts and attachments",
+ attachments=[
+ ChatAttachment(
+ name="image.png",
+ content_type="image/png",
+ url=f"data:image/png;base64,{base64.b64encode(b'hello')}",
+ ),
+ ],
+ parts=[
+ TextPart(
+ type="text", text="Message with parts and attachments"
+ ),
+ ReasoningPart(
+ type="reasoning",
+ reasoning="Deep thinking process",
+ details=[
+ ReasoningDetails(
+ type="text", text="Analysis", signature=None
+ )
+ ],
+ ),
+ ToolInvocationPart(
+ type="tool-invocation",
+ tool_invocation=ToolInvocationResult(
+ state="result",
+ result={"answer": 4},
+ tool_call_id="call_123",
+ tool_name="calculator",
+ step=1,
+ args={"expression": "2 + 2"},
+ ),
+ ),
+ ],
+ ),
+ ]
+ result = convert_to_openai_messages(sample_messages)
+ assert result == [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Message with parts and attachments"},
+ {
+ "type": "reasoning",
+ "reasoning": "Deep thinking process",
+ "details": [
+ {"type": "text", "text": "Analysis", "signature": None}
+ ],
+ },
+ {
+ "type": "tool-invocation",
+ "tool_invocation": {
+ "state": "result",
+ "result": {"answer": 4},
+ "tool_call_id": "call_123",
+ "tool_name": "calculator",
+ "step": 1,
+ "args": {"expression": "2 + 2"},
+ },
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,b'aGVsbG8='"},
+ },
+ ],
+ }
+ ]
def test_convert_to_anthropic_messages(sample_messages: list[ChatMessage]):