Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: cell output for `@` context w/ attachments for media output (#6240)

Myles Scolnick03e85b0a5452
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsx
index f7e2cb1e99..a91d215375 100644
--- a/frontend/src/components/chat/chat-panel.tsx
+++ b/frontend/src/components/chat/chat-panel.tsx
@@ -6,6 +6,7 @@ import { storePrompt } from "@marimo-team/codemirror-ai";
import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";
import {
DefaultChatTransport,
+ type FileUIPart,
lastAssistantMessageIsCompleteWithToolCalls,
type ToolUIPart,
} from "ai";
@@ -46,11 +47,13 @@ import {
type ChatId,
chatStateAtom,
} from "@/core/ai/state";
+import type { ChatAttachment } from "@/core/ai/types";
import { aiAtom, aiEnabledAtom } from "@/core/config/config";
import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";
import { FeatureFlagged } from "@/core/config/feature-flag";
import { useRequestClient } from "@/core/network/requests";
import { useRuntimeManager } from "@/core/runtime/config";
+import type { ChatMessage } from "@/plugins/impl/chat/types";
import { ErrorBanner } from "@/plugins/impl/common/error-banner";
import { cn } from "@/utils/cn";
import { timeAgo } from "@/utils/dates";
@@ -58,7 +61,7 @@ import { Logger } from "@/utils/Logger";
import { AIModelDropdown } from "../ai/ai-model-dropdown";
import { useOpenSettingsToTab } from "../app-config/state";
import { PromptInput } from "../editor/ai/add-cell-with-ai";
-import { getAICompletionBody } from "../editor/ai/completion-utils";
+import { getAICompletionBodyWithAttachments } from "../editor/ai/completion-utils";
import { PanelEmptyState } from "../editor/chrome/panels/empty-state";
import { CopyClipboardIcon } from "../icons/copy-icon";
import { Input } from "../ui/input";
@@ -458,7 +461,7 @@ const ChatInput: React.FC<ChatInputProps> = memo(
});
return (
- <div className="relative shrink-0 min-h-[80px] flex flex-col">
+ <div className="relative shrink-0 min-h-[80px] flex flex-col border-t">
<div className={cn("px-2 py-3 flex-1", inputClassName)}>
<PromptInput
inputRef={inputRef}
@@ -539,32 +542,54 @@ const ChatPanelBody = () => {
transport: new DefaultChatTransport({
api: runtimeManager.getAiURL("chat").toString(),
headers: runtimeManager.headers(),
- prepareSendMessagesRequest: (options) => {
- let input = "";
- for (const message of options.messages) {
- const textParts = message.parts.filter((p) => p.type === "text");
- input += textParts.map((p) => p.text).join("\n");
+ prepareSendMessagesRequest: async (options) => {
+ // Map from parts to a single string
+ function toContent(parts: UIMessage["parts"]): string {
+ return parts
+ .map((part) => (part.type === "text" ? part.text : ""))
+ .join("\n");
+ }
+
+ // Map from ai/sdk files to our attachment type
+ function toAttachment(p: FileUIPart): ChatAttachment {
+ return {
+ contentType: p.mediaType,
+ name: p.filename,
+ url: p.url,
+ };
+ }
+
+ const input = toContent(options.messages.flatMap((m) => m.parts));
+ const completionBody = await getAICompletionBodyWithAttachments({
+ input,
+ });
+
+ // Map from UIMessage to our ChatMessage type
+ // If it's the last message, add the attachments from the completion body
+ function mapMessage(
+ m: UIMessage,
+ isLastMessage: boolean,
+ ): ChatMessage & { attachments: ChatAttachment[] } {
+ const attachments = m.parts.flatMap((p) =>
+ p.type === "file" ? toAttachment(p) : [],
+ );
+ if (isLastMessage) {
+ attachments.push(...completionBody.attachments);
+ }
+ return {
+ role: m.role,
+ content: toContent(m.parts),
+ attachments: attachments,
+ };
}
- const completionBody = getAICompletionBody({ input });
return {
body: {
...options,
- ...completionBody,
- messages: options.messages.map((m) => ({
+ ...completionBody.body,
+ messages: options.messages.map((m, idx) => ({
...m,
- attachments: m.parts.flatMap((p) =>
- p.type === "file"
- ? {
- contentType: p.mediaType,
- name: p.filename,
- url: p.url,
- }
- : [],
- ),
- content: m.parts
- ?.map((p) => ("text" in p ? p.text : ""))
- .join("\n"),
+ ...mapMessage(m, idx === options.messages.length - 1),
})),
},
};
diff --git a/frontend/src/components/editor/ai/completion-utils.ts b/frontend/src/components/editor/ai/completion-utils.ts
index 8bbd4c0a93..8c631bc6dd 100644
--- a/frontend/src/components/editor/ai/completion-utils.ts
+++ b/frontend/src/components/editor/ai/completion-utils.ts
@@ -6,23 +6,32 @@ import type {
CompletionSource,
} from "@codemirror/autocomplete";
import { getAIContextRegistry } from "@/core/ai/context/context";
+import type { ChatAttachment } from "@/core/ai/types";
import { getCodes } from "@/core/codemirror/copilot/getCodes";
import type { AiCompletionRequest } from "@/core/network/types";
import { store } from "@/core/state/jotai";
import { Logger } from "@/utils/Logger";
+interface Opts {
+ input: string;
+}
+
+interface AICompletionBodyWithAttachments {
+ body: Omit<AiCompletionRequest, "language" | "prompt" | "code">;
+ attachments: ChatAttachment[];
+}
+
/**
* Gets the request body for the AI completion API.
*/
export function getAICompletionBody({
input,
-}: {
- input: string;
-}): Omit<AiCompletionRequest, "language" | "prompt" | "code"> {
+}: Opts): Omit<AiCompletionRequest, "language" | "prompt" | "code"> {
let contextString = "";
+
// Skip if no '@' in the input
if (input.includes("@")) {
- contextString = extractDatasetsAndVariables(input);
+ contextString = extractTaggedContext(input);
Logger.debug("Included context", contextString);
}
@@ -37,11 +46,49 @@ export function getAICompletionBody({
}
/**
- * Extracts datasets and variables from the input.
+ * Gets the request body and attachments for the AI completion API.
+ */
+export async function getAICompletionBodyWithAttachments({
+ input,
+}: Opts): Promise<AICompletionBodyWithAttachments> {
+ let contextString = "";
+ let attachments: ChatAttachment[] = [];
+
+ // Skip if no '@' in the input
+ if (input.includes("@")) {
+ const registry = getAIContextRegistry(store);
+ const contextIds = registry.parseAllContextIds(input);
+
+ // Get context string
+ contextString = registry.formatContextForAI(contextIds);
+
+ // Get attachments
+ try {
+ attachments = await registry.getAttachmentsForContext(contextIds);
+ Logger.debug("Included attachments", attachments.length);
+ } catch (error) {
+ Logger.error("Error getting attachments:", error);
+ }
+ }
+
+ return {
+ body: {
+ includeOtherCode: getCodes(""),
+ context: {
+ plainText: contextString,
+ schema: [],
+ variables: [],
+ },
+ },
+ attachments,
+ };
+}
+
+/**
+ * Extracts datasets, variables and other context from the input.
* References are with @<name> in the input.
- * Prioritizes datasets over variables if there's a name conflict.
*/
-function extractDatasetsAndVariables(input: string): string {
+function extractTaggedContext(input: string): string {
const registry = getAIContextRegistry(store);
const contextIds = registry.parseAllContextIds(input);
return registry.formatContextForAI(contextIds);
diff --git a/frontend/src/core/ai/context/__tests__/registry.test.ts b/frontend/src/core/ai/context/__tests__/registry.test.ts
index d8cbdbe832..b73bffb3ce 100644
--- a/frontend/src/core/ai/context/__tests__/registry.test.ts
+++ b/frontend/src/core/ai/context/__tests__/registry.test.ts
@@ -2,8 +2,9 @@
import type { Completion } from "@codemirror/autocomplete";
import { createStore } from "jotai";
-import { beforeEach, describe, expect, it } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
import { MockNotebook } from "@/__mocks__/notebook";
+import type { ChatAttachment } from "@/core/ai/types";
import { notebookAtom } from "@/core/cells/cells";
import { CellId as CellIdClass } from "@/core/cells/ids";
import {
@@ -19,10 +20,26 @@ import {
// Mock context item for testing
interface MockContextItem extends AIContextItem {
- type: "mock";
- data: { value: string };
+ type: "mock" | "attachment";
+ data: {
+ value: string;
+ needsAttachment?: boolean;
+ };
}
+// Mock attachment data for testing
+const mockAttachment1: ChatAttachment = {
+ name: "test-image-1.png",
+ contentType: "image/png",
+ url: "data:image/png;base64,mockdata1",
+};
+
+const mockAttachment2: ChatAttachment = {
+ name: "test-image-2.jpg",
+ contentType: "image/jpeg",
+ url: "data:image/jpeg;base64,mockdata2",
+};
+
// Concrete implementation of AIContextProvider for testing
class MockContextProvider extends AIContextProvider<MockContextItem> {
readonly title = "Mock Items";
@@ -76,6 +93,44 @@ class MockContextProvider extends AIContextProvider<MockContextItem> {
}
}
+// Test provider that supports attachments
+class AttachmentContextProvider extends AIContextProvider<MockContextItem> {
+ readonly title = "Attachment Items";
+ readonly mentionPrefix = "@";
+ readonly contextType = "attachment";
+
+ constructor(
+ private items: MockContextItem[] = [],
+ private attachments: ChatAttachment[] = [],
+ ) {
+ super();
+ }
+
+ getItems(): MockContextItem[] {
+ return this.items;
+ }
+
+ formatContext(item: MockContextItem): string {
+ return `Attachment: ${item.name} (${item.data.value})`;
+ }
+
+ formatCompletion(item: MockContextItem): Completion {
+ return this.createBasicCompletion(item);
+ }
+
+ override async getAttachments(
+ items: MockContextItem[],
+ ): Promise<ChatAttachment[]> {
+ // Return attachments for items that need them
+ const itemsNeedingAttachments = items.filter(
+ (item) => item.data.needsAttachment,
+ );
+ return itemsNeedingAttachments
+ .map((_, index) => this.attachments[index % this.attachments.length])
+ .filter(Boolean);
+ }
+}
+
interface FileContextItem extends AIContextItem {
type: "file";
data: { value: string };
@@ -667,4 +722,221 @@ describe("AIContextRegistry", () => {
expect(formattedContext).toContain("Item 1");
});
});
+
+ describe("attachment functionality", () => {
+ let attachmentProvider: AttachmentContextProvider;
+ let attachmentRegistry: AIContextRegistry<MockContextItem>;
+
+ beforeEach(() => {
+ const itemWithAttachment: MockContextItem = {
+ uri: "attachment://item1" as ContextLocatorId,
+ name: "item1",
+ type: "attachment",
+ description: "Item with attachment",
+ data: { needsAttachment: true, value: "test1" },
+ };
+
+ const itemWithoutAttachment: MockContextItem = {
+ uri: "attachment://item2" as ContextLocatorId,
+ name: "item2",
+ type: "attachment",
+ description: "Item without attachment",
+ data: { needsAttachment: false, value: "test2" },
+ };
+
+ attachmentProvider = new AttachmentContextProvider(
+ [itemWithAttachment, itemWithoutAttachment],
+ [mockAttachment1, mockAttachment2],
+ );
+
+ attachmentRegistry = new AIContextRegistry<MockContextItem>().register(
+ attachmentProvider,
+ );
+ });
+
+ describe("getAttachmentsForContext", () => {
+ it("should return empty array for empty context IDs", async () => {
+ const attachments = await attachmentRegistry.getAttachmentsForContext(
+ [],
+ );
+ expect(attachments).toEqual([]);
+ });
+
+ it("should return empty array for non-existent context IDs", async () => {
+ const nonExistentIds = ["attachment://nonexistent" as ContextLocatorId];
+ const attachments =
+ await attachmentRegistry.getAttachmentsForContext(nonExistentIds);
+ expect(attachments).toEqual([]);
+ });
+
+ it("should get attachments from provider that supports them", async () => {
+ const contextIds = ["attachment://item1" as ContextLocatorId];
+ const attachments =
+ await attachmentRegistry.getAttachmentsForContext(contextIds);
+
+ expect(attachments).toHaveLength(1);
+ expect(attachments[0]).toEqual(mockAttachment1);
+ });
+
+ it("should not get attachments for items that don't need them", async () => {
+ const contextIds = ["attachment://item2" as ContextLocatorId];
+ const attachments =
+ await attachmentRegistry.getAttachmentsForContext(contextIds);
+
+ expect(attachments).toHaveLength(0);
+ });
+
+ it("should not get attachments from providers with default implementation", async () => {
+ const simpleRegistry =
+ new AIContextRegistry<MockContextItem>().register(mockProvider);
+
+ const contextIds = ["mock://item1" as ContextLocatorId];
+ const attachments =
+ await simpleRegistry.getAttachmentsForContext(contextIds);
+
+ expect(attachments).toHaveLength(0);
+ });
+
+ it("should handle multiple context IDs from different providers", async () => {
+ const mixedRegistry = new AIContextRegistry<MockContextItem>()
+ .register(attachmentProvider)
+ .register(mockProvider);
+
+ const contextIds = [
+ "attachment://item1" as ContextLocatorId, // should have attachment
+ "attachment://item2" as ContextLocatorId, // should not have attachment
+ "mock://item1" as ContextLocatorId, // provider doesn't support attachments
+ ];
+
+ const attachments =
+ await mixedRegistry.getAttachmentsForContext(contextIds);
+
+ // Only attachment://item1 should contribute an attachment
+ expect(attachments).toHaveLength(1);
+ expect(attachments[0]).toEqual(mockAttachment1);
+ });
+
+ it("should handle multiple items needing attachments", async () => {
+ const multiItem1: MockContextItem = {
+ uri: "attachment://multi1" as ContextLocatorId,
+ name: "multi1",
+ type: "attachment",
+ description: "Multi item 1",
+ data: { needsAttachment: true, value: "multi1" },
+ };
+
+ const multiItem2: MockContextItem = {
+ uri: "attachment://multi2" as ContextLocatorId,
+ name: "multi2",
+ type: "attachment",
+ description: "Multi item 2",
+ data: { needsAttachment: true, value: "multi2" },
+ };
+
+ const multiProvider = new AttachmentContextProvider(
+ [multiItem1, multiItem2],
+ [mockAttachment1, mockAttachment2],
+ );
+
+ const multiRegistry = new AIContextRegistry<MockContextItem>().register(
+ multiProvider,
+ );
+
+ const contextIds = [
+ "attachment://multi1" as ContextLocatorId,
+ "attachment://multi2" as ContextLocatorId,
+ ];
+
+ const attachments =
+ await multiRegistry.getAttachmentsForContext(contextIds);
+
+ expect(attachments).toHaveLength(2);
+ expect(attachments).toContainEqual(mockAttachment1);
+ expect(attachments).toContainEqual(mockAttachment2);
+ });
+
+ it("should handle provider errors gracefully", async () => {
+ const errorProvider = new AttachmentContextProvider([
+ {
+ uri: "attachment://error-item" as ContextLocatorId,
+ name: "error-item",
+ type: "attachment",
+ description: "Item that causes error",
+ data: { needsAttachment: true, value: "error" },
+ },
+ ]);
+
+ // Override getAttachments to throw an error
+ errorProvider.getAttachments = vi
+ .fn()
+ .mockRejectedValue(new Error("Attachment error"));
+
+ const errorRegistry = new AIContextRegistry<MockContextItem>()
+ .register(errorProvider)
+ .register(mockProvider);
+
+ const contextIds = [
+ "attachment://error-item" as ContextLocatorId,
+ "mock://item1" as ContextLocatorId,
+ ];
+
+ // Should not throw and should handle the error gracefully
+ const attachments =
+ await errorRegistry.getAttachmentsForContext(contextIds);
+ expect(attachments).toEqual([]);
+ });
+ });
+
+ describe("provider batching for attachments", () => {
+ it("should batch multiple items to the same provider", async () => {
+ const getAttachmentsSpy = vi.spyOn(
+ attachmentProvider,
+ "getAttachments",
+ );
+
+ const contextIds = [
+ "attachment://item1" as ContextLocatorId,
+ "attachment://item2" as ContextLocatorId,
+ ];
+
+ await attachmentRegistry.getAttachmentsForContext(contextIds);
+
+ // Should call getAttachments once with both items
+ expect(getAttachmentsSpy).toHaveBeenCalledTimes(1);
+ expect(getAttachmentsSpy).toHaveBeenCalledWith(
+ expect.arrayContaining([
+ expect.objectContaining({ uri: "attachment://item1" }),
+ expect.objectContaining({ uri: "attachment://item2" }),
+ ]),
+ );
+ });
+
+ it("should call different providers separately", async () => {
+ const mixedRegistry = new AIContextRegistry<MockContextItem>()
+ .register(attachmentProvider)
+ .register(mockProvider);
+
+ const attachmentSpy = vi.spyOn(attachmentProvider, "getAttachments");
+ const mockSpy = vi.spyOn(mockProvider, "getAttachments");
+
+ const contextIds = [
+ "attachment://item1" as ContextLocatorId,
+ "mock://item1" as ContextLocatorId,
+ ];
+
+ await mixedRegistry.getAttachmentsForContext(contextIds);
+
+ // Should call each provider once
+ expect(attachmentSpy).toHaveBeenCalledTimes(1);
+ expect(mockSpy).toHaveBeenCalledTimes(1);
+
+ expect(attachmentSpy).toHaveBeenCalledWith([
+ expect.objectContaining({ uri: "attachment://item1" }),
+ ]);
+ expect(mockSpy).toHaveBeenCalledWith([
+ expect.objectContaining({ uri: "mock://item1" }),
+ ]);
+ });
+ });
+ });
});
diff --git a/frontend/src/core/ai/context/context.ts b/frontend/src/core/ai/context/context.ts
index 591f0f40d0..c4b7e80f85 100644
--- a/frontend/src/core/ai/context/context.ts
+++ b/frontend/src/core/ai/context/context.ts
@@ -3,6 +3,7 @@
import { allTablesAtom } from "@/core/datasets/data-source-connections";
import type { JotaiStore } from "@/core/state/jotai";
import { variablesAtom } from "@/core/variables/state";
+import { CellOutputContextProvider } from "./providers/cell-output";
import { ErrorContextProvider } from "./providers/error";
import { TableContextProvider } from "./providers/tables";
import { VariableContextProvider } from "./providers/variable";
@@ -14,5 +15,6 @@ export function getAIContextRegistry(store: JotaiStore) {
return new AIContextRegistry()
.register(new TableContextProvider(tablesMap))
.register(new VariableContextProvider(variables, tablesMap))
- .register(new ErrorContextProvider(store));
+ .register(new ErrorContextProvider(store))
+ .register(new CellOutputContextProvider(store));
}
diff --git a/frontend/src/core/ai/context/providers/__tests__/cell-output.test.ts b/frontend/src/core/ai/context/providers/__tests__/cell-output.test.ts
new file mode 100644
index 0000000000..796fdc9559
--- /dev/null
+++ b/frontend/src/core/ai/context/providers/__tests__/cell-output.test.ts
@@ -0,0 +1,374 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { Mocks } from "@/__mocks__/common";
+
+// Mock the external dependencies
+vi.mock("html-to-image", () => ({
+ toPng: vi.fn().mockResolvedValue("data:image/png;base64,mockbase64data"),
+}));
+
+vi.mock("@/utils/Logger", () => ({
+ Logger: Mocks.quietLogger(),
+}));
+
+import type { NotebookState } from "@/core/cells/cells";
+import type { CellId } from "@/core/cells/ids";
+import type { OutputMessage } from "@/core/kernel/messages";
+import type { JotaiStore } from "@/core/state/jotai";
+import { CellOutputContextProvider } from "../cell-output";
+
+// Test helper to create mock store
+function createMockStore(notebook: NotebookState): JotaiStore {
+ const store = {
+ get: vi.fn().mockReturnValue(notebook),
+ } as unknown as JotaiStore;
+ return store;
+}
+
+describe("CellOutputContextProvider", () => {
+ let provider: CellOutputContextProvider;
+ let mockStore: JotaiStore;
+ let mockNotebook: NotebookState;
+
+ beforeEach(() => {
+ // Create a basic mock notebook state
+ mockNotebook = {
+ cellIds: {
+ inOrderIds: ["cell1" as CellId, "cell2" as CellId, "cell3" as CellId],
+ },
+ cellData: {
+ cell1: {
+ id: "cell1" as CellId,
+ name: "My Cell",
+ code: "print('hello world')",
+ },
+ cell2: {
+ id: "cell2" as CellId,
+ name: "",
+ code: "import matplotlib.pyplot as plt\nplt.plot([1,2,3])",
+ },
+ cell3: {
+ id: "cell3" as CellId,
+ name: "Empty Cell",
+ code: "# no output",
+ },
+ },
+ cellRuntime: {
+ cell1: {
+ output: {
+ mimetype: "text/plain",
+ data: "hello world",
+ } as OutputMessage,
+ },
+ cell2: {
+ output: {
+ mimetype: "image/png",
+ data: "base64imagedata",
+ } as OutputMessage,
+ },
+ cell3: {
+ output: null,
+ },
+ },
+ } as unknown as NotebookState;
+
+ mockStore = createMockStore(mockNotebook);
+ provider = new CellOutputContextProvider(mockStore);
+ });
+
+ describe("basic properties", () => {
+ it("should have correct title and context type", () => {
+ expect(provider.title).toBe("Cell Outputs");
+ expect(provider.contextType).toBe("cell-output");
+ expect(provider.mentionPrefix).toBe("@");
+ });
+ });
+
+ describe("getItems", () => {
+ it("should return only cells with outputs", () => {
+ const items = provider.getItems();
+
+ // Should have 2 items (cell1 and cell2, but not cell3 which has no output)
+ expect(items).toHaveLength(2);
+
+ const cellIds = items.map((item) => item.data.cellId);
+ expect(cellIds).toContain("cell1");
+ expect(cellIds).toContain("cell2");
+ expect(cellIds).not.toContain("cell3");
+ });
+
+ it("should correctly identify text vs media outputs", () => {
+ const items = provider.getItems();
+
+ const textItem = items.find((item) => item.data.cellId === "cell1");
+ const mediaItem = items.find((item) => item.data.cellId === "cell2");
+
+ expect(textItem?.data.outputType).toBe("text");
+ expect(mediaItem?.data.outputType).toBe("media");
+ });
+
+ it("should process text content correctly", () => {
+ const items = provider.getItems();
+ const textItem = items.find((item) => item.data.cellId === "cell1");
+
+ expect(textItem?.data.processedContent).toBe("hello world");
+ });
+
+ it("should mark media items for download when no direct URL available", () => {
+ const items = provider.getItems();
+ const mediaItem = items.find((item) => item.data.cellId === "cell2");
+
+ expect(mediaItem?.data.shouldDownloadImage).toBe(true);
+ expect(mediaItem?.data.imageUrl).toBeUndefined();
+ });
+
+ it("should include cell code and names", () => {
+ const items = provider.getItems();
+
+ const item1 = items.find((item) => item.data.cellId === "cell1");
+ const item2 = items.find((item) => item.data.cellId === "cell2");
+
+ expect(item1?.data.cellName).toBe("My Cell");
+ expect(item1?.data.cellCode).toBe("print('hello world')");
+
+ expect(item2?.data.cellName).toBe("cell-1");
+ expect(item2?.data.cellCode).toBe(
+ "import matplotlib.pyplot as plt\nplt.plot([1,2,3])",
+ );
+ });
+ });
+
+ describe("formatContext", () => {
+ it("should format text output context correctly", () => {
+ const items = provider.getItems();
+ const textItem = items.find((item) => item.data.cellId === "cell1");
+
+ if (!textItem) throw new Error("Text item not found");
+
+ const context = provider.formatContext(textItem);
+
+ expect(context).toContain("cell-output");
+ expect(context).toContain("My Cell");
+ expect(context).toContain("Cell Code:");
+ expect(context).toContain("print('hello world')");
+ expect(context).toContain("Output:");
+ expect(context).toContain("hello world");
+ });
+
+ it("should format media output context correctly", () => {
+ const items = provider.getItems();
+ const mediaItem = items.find((item) => item.data.cellId === "cell2");
+
+ if (!mediaItem) throw new Error("Media item not found");
+
+ const context = provider.formatContext(mediaItem);
+
+ expect(context).toContain("cell-output");
+ expect(context).toContain("cell-1");
+ expect(context).toContain("Cell Code:");
+ expect(context).toContain("import matplotlib.pyplot as plt");
+ expect(context).toContain("Media Output: Contains image/png content");
+ });
+ });
+
+ describe("formatCompletion", () => {
+ it("should create proper completion object", () => {
+ const items = provider.getItems();
+ const item = items[0];
+
+ const completion = provider.formatCompletion(item);
+
+ expect(completion.label).toMatch(/@.+/);
+ expect(completion.displayLabel).toBe(item.data.cellName);
+ expect(completion.detail).toContain("output");
+ expect(completion.type).toBe("cell-output");
+ expect(completion.section).toBe("Cell Output");
+ expect(typeof completion.info).toBe("function");
+ });
+ });
+});
+
+// Test utility functions separately
+describe("Cell output utility functions", () => {
+ // We need to extract these functions to test them, or make them exported
+ // For now, let's test through the public interface
+
+ describe("media type detection", () => {
+ let provider: CellOutputContextProvider;
+ let mockStore: JotaiStore;
+
+ beforeEach(() => {
+ const mockNotebook = {
+ cellIds: { inOrderIds: [] },
+ cellData: {},
+ cellRuntime: {},
+ } as unknown as NotebookState;
+
+ mockStore = createMockStore(mockNotebook);
+ provider = new CellOutputContextProvider(mockStore);
+ });
+
+ it("should detect image mimetypes as media", () => {
+ const testCases = [
+ { mimetype: "image/png", data: "test", expected: "media" },
+ { mimetype: "image/jpeg", data: "test", expected: "media" },
+ { mimetype: "image/gif", data: "test", expected: "media" },
+ { mimetype: "image/svg+xml", data: "test", expected: "media" },
+ ];
+
+ for (const testCase of testCases) {
+ mockStore = createMockStore({
+ cellIds: { inOrderIds: ["test" as CellId] },
+ cellData: {
+ test: { id: "test" as CellId, name: "", code: "" },
+ },
+ cellRuntime: {
+ test: {
+ output: {
+ mimetype: testCase.mimetype,
+ data: testCase.data,
+ } as OutputMessage,
+ },
+ },
+ } as unknown as NotebookState);
+
+ provider = new CellOutputContextProvider(mockStore);
+ const items = provider.getItems();
+
+ expect(items[0]?.data.outputType).toBe(testCase.expected);
+ }
+ });
+
+ it("should detect HTML with media tags as media", () => {
+ const htmlWithImage = '<div><img src="test.png" /></div>';
+ const htmlWithCanvas = '<canvas width="100" height="100"></canvas>';
+ const htmlWithSvg = '<svg><circle r="50" /></svg>';
+
+ const testCases = [
+ { data: htmlWithImage, expected: "media" },
+ { data: htmlWithCanvas, expected: "media" },
+ { data: htmlWithSvg, expected: "media" },
+ { data: "<p>Just text</p>", expected: "text" },
+ ];
+
+ for (const testCase of testCases) {
+ mockStore = createMockStore({
+ cellIds: { inOrderIds: ["test" as CellId] },
+ cellData: {
+ test: { id: "test" as CellId, name: "", code: "" },
+ },
+ cellRuntime: {
+ test: {
+ output: {
+ mimetype: "text/html",
+ data: testCase.data,
+ } as OutputMessage,
+ },
+ },
+ } as unknown as NotebookState);
+
+ provider = new CellOutputContextProvider(mockStore);
+ const items = provider.getItems();
+
+ expect(items[0]?.data.outputType).toBe(testCase.expected);
+ }
+ });
+
+ it("should detect text mimetypes correctly", () => {
+ const testCases = [
+ { mimetype: "text/plain", expected: "text" },
+ {
+ mimetype: "text/html",
+ data: "<p>No media tags</p>",
+ expected: "text",
+ },
+ { mimetype: "application/json", expected: "text" },
+ ];
+
+ for (const testCase of testCases) {
+ mockStore = createMockStore({
+ cellIds: { inOrderIds: ["test" as CellId] },
+ cellData: {
+ test: { id: "test" as CellId, name: "", code: "" },
+ },
+ cellRuntime: {
+ test: {
+ output: {
+ mimetype: testCase.mimetype,
+ data: testCase.data || "test data",
+ } as OutputMessage,
+ },
+ },
+ } as unknown as NotebookState);
+
+ provider = new CellOutputContextProvider(mockStore);
+ const items = provider.getItems();
+
+ expect(items[0]?.data.outputType).toBe(testCase.expected);
+ }
+ });
+ });
+
+ describe("HTML content parsing", () => {
+ let provider: CellOutputContextProvider;
+ let mockStore: JotaiStore;
+
+ beforeEach(() => {
+ // Mock DOM methods for HTML parsing
+ const mockDiv = {
+ innerHTML: "",
+ textContent: "",
+ innerText: "",
+ };
+
+ global.document = {
+ createElement: vi.fn().mockReturnValue(mockDiv),
+ } as unknown as Document;
+
+ const mockNotebook = {
+ cellIds: { inOrderIds: [] },
+ cellData: {},
+ cellRuntime: {},
+ } as unknown as NotebookState;
+
+ mockStore = createMockStore(mockNotebook);
+ provider = new CellOutputContextProvider(mockStore);
+ });
+
+ it("should extract text content from HTML", () => {
+ const htmlContent = "<p>Hello <strong>world</strong>!</p>";
+
+ // Mock the DOM parsing result
+ const mockDiv = {
+ innerHTML: "",
+ textContent: "Hello world!",
+ innerText: "Hello world!",
+ };
+
+ (
+ global.document.createElement as ReturnType<typeof vi.fn>
+ ).mockReturnValue(mockDiv);
+
+ mockStore = createMockStore({
+ cellIds: { inOrderIds: ["test" as CellId] },
+ cellData: {
+ test: { id: "test" as CellId, name: "", code: "" },
+ },
+ cellRuntime: {
+ test: {
+ output: {
+ mimetype: "text/html",
+ data: htmlContent,
+ } as OutputMessage,
+ },
+ },
+ } as unknown as NotebookState);
+
+ provider = new CellOutputContextProvider(mockStore);
+ const items = provider.getItems();
+
+ expect(items[0]?.data.processedContent).toBe("Hello world!");
+ });
+ });
+});
diff --git a/frontend/src/core/ai/context/providers/__tests__/error.test.ts b/frontend/src/core/ai/context/providers/__tests__/error.test.ts
index c7b095dcb8..85cb71145d 100644
--- a/frontend/src/core/ai/context/providers/__tests__/error.test.ts
+++ b/frontend/src/core/ai/context/providers/__tests__/error.test.ts
@@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it } from "vitest";
import { MockNotebook } from "@/__mocks__/notebook";
import { notebookAtom } from "@/core/cells/cells";
import { type CellId, CellId as CellIdClass } from "@/core/cells/ids";
+import { Boosts } from "../common";
import { ErrorContextProvider } from "../error";
describe("ErrorContextProvider", () => {
@@ -121,7 +122,7 @@ describe("ErrorContextProvider", () => {
label: "@Errors",
displayLabel: "Errors",
detail: "2 errors",
- boost: 2, // Boosts.ERROR
+ boost: Boosts.ERROR,
type: "error",
apply: "@Errors",
section: "Errors",
@@ -181,7 +182,7 @@ describe("ErrorContextProvider", () => {
expect(completion).toMatchObject({
label: "Error",
displayLabel: "Error",
- boost: 2, // Boosts.ERROR
+ boost: Boosts.ERROR,
});
});
});
diff --git a/frontend/src/core/ai/context/providers/cell-output.ts b/frontend/src/core/ai/context/providers/cell-output.ts
new file mode 100644
index 0000000000..f12b29af0f
--- /dev/null
+++ b/frontend/src/core/ai/context/providers/cell-output.ts
@@ -0,0 +1,347 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { Completion } from "@codemirror/autocomplete";
+import { toPng } from "html-to-image";
+import type { ChatAttachment } from "@/core/ai/types";
+import { notebookAtom } from "@/core/cells/cells";
+import { type CellId, CellOutputId } from "@/core/cells/ids";
+import { displayCellName } from "@/core/cells/names";
+import { isOutputEmpty } from "@/core/cells/outputs";
+import type { OutputMessage } from "@/core/kernel/messages";
+import type { JotaiStore } from "@/core/state/jotai";
+import { Logger } from "@/utils/Logger";
+import { type AIContextItem, AIContextProvider } from "../registry";
+import { contextToXml } from "../utils";
+import { Boosts } from "./common";
+
+export interface CellOutputContextItem extends AIContextItem {
+ type: "cell-output";
+ data: {
+ cellId: CellId;
+ cellName: string;
+ cellCode: string;
+ output: OutputMessage;
+ outputType: "text" | "media";
+ processedContent?: string;
+ imageUrl?: string;
+ shouldDownloadImage?: boolean;
+ };
+}
+
+function isMediaMimetype(
+ mimetype: OutputMessage["mimetype"] | undefined,
+ htmlString: string,
+): boolean {
+ if (!mimetype) return false;
+
+ const mediaPrefixes = ["image/", "video/", "audio/", "application/pdf"];
+ if (mediaPrefixes.some((prefix) => mimetype.startsWith(prefix))) {
+ return true;
+ }
+ const mediaIncludes = ["svg", "vega"];
+ if (mediaIncludes.some((include) => mimetype.includes(include))) {
+ return true;
+ }
+
+ // If it is HTML, we need to check if it contains a media tag
+ if (mimetype === "text/html") {
+ const mediaTags = [
+ "<img",
+ "<video",
+ "<audio",
+ "<iframe",
+ "<canvas",
+ "<svg",
+ "<marimo-ui-element",
+ ];
+ if (mediaTags.some((tag) => htmlString.includes(tag))) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+function parseHtmlContent(htmlString: string): string {
+ try {
+ // Create a temporary DOM element to parse HTML
+ const tempDiv = document.createElement("div");
+ tempDiv.innerHTML = htmlString;
+
+ // Extract text content, removing HTML tags
+ const textContent = tempDiv.textContent || tempDiv.innerText || "";
+
+ // Clean up extra whitespace
+ return textContent.replace(/\s+/g, " ").trim();
+ } catch (error) {
+ Logger.error("Error parsing HTML content:", error);
+ // If parsing fails, return the original string
+ return htmlString;
+ }
+}
+
+export class CellOutputContextProvider extends AIContextProvider<CellOutputContextItem> {
+ readonly title = "Cell Outputs";
+ readonly mentionPrefix = "@";
+ readonly contextType = "cell-output";
+
+ constructor(private store: JotaiStore) {
+ super();
+ }
+
+ getItems(): CellOutputContextItem[] {
+ const notebook = this.store.get(notebookAtom);
+ const items: CellOutputContextItem[] = [];
+
+ for (const [cellIndex, cellId] of notebook.cellIds.inOrderIds.entries()) {
+ const cellRuntime = notebook.cellRuntime[cellId];
+
+ // Filter to only cells with output
+ if (!cellRuntime?.output || isOutputEmpty(cellRuntime.output)) {
+ continue;
+ }
+
+ const cellData = notebook.cellData[cellId];
+ const output = cellRuntime.output;
+ const mimetype = output.mimetype;
+
+ // Determine output type
+ const isMedia = isMediaMimetype(mimetype, String(output.data));
+ const outputType = isMedia ? "media" : "text";
+
+ const cellName = displayCellName(cellData.name, cellIndex);
+
+ let processedContent: string | undefined;
+ let imageUrl: string | undefined;
+ let shouldDownloadImage: boolean = false;
+
+ // Process text content
+ if (outputType === "text" && typeof output.data === "string") {
+ if (mimetype === "text/html") {
+ processedContent = parseHtmlContent(output.data);
+ } else {
+ processedContent = output.data;
+ }
+ }
+
+ // Process media content - for now, we'll just note that it's media
+ if (outputType === "media") {
+ if (
+ typeof output.data === "string" &&
+ output.data.startsWith("data:")
+ ) {
+ imageUrl = output.data; // Data URL
+ } else if (
+ typeof output.data === "string" &&
+ output.data.startsWith("http")
+ ) {
+ imageUrl = output.data; // External URL
+ } else {
+ // Download the image from the DOM element
+ shouldDownloadImage = true;
+ }
+ }
+
+ items.push({
+ uri: this.asURI(cellId),
+ name: cellName,
+ type: this.contextType,
+ description: `Cell output (${mimetype || "unknown"})`,
+ data: {
+ cellId,
+ cellName,
+ cellCode: cellData?.code || "",
+ output,
+ outputType,
+ processedContent,
+ imageUrl,
+ shouldDownloadImage,
+ },
+ });
+ }
+
+ return items;
+ }
+
+ formatCompletion(item: CellOutputContextItem): Completion {
+ const { data } = item;
+ return {
+ label: `@${data.cellName}`,
+ displayLabel: data.cellName,
+ detail: `${data.outputType} output`,
+ boost: Boosts.CELL_OUTPUT,
+ type: this.contextType,
+ section: "Cell Output",
+ apply: `@${data.cellName}`,
+ info: () => {
+ const infoContainer = document.createElement("div");
+ infoContainer.classList.add(
+ "mo-cm-tooltip",
+ "docs-documentation",
+ "min-w-[300px]",
+ "max-w-[500px]",
+ "flex",
+ "flex-col",
+ "gap-2",
+ );
+
+ const headerDiv = document.createElement("div");
+ headerDiv.classList.add("flex", "flex-col", "gap-1");
+
+ const nameDiv = document.createElement("div");
+ nameDiv.classList.add("font-bold", "text-base");
+ nameDiv.textContent = data.cellName;
+ headerDiv.append(nameDiv);
+
+ const descriptionDiv = document.createElement("div");
+ descriptionDiv.classList.add("text-sm", "text-muted-foreground");
+ headerDiv.append(descriptionDiv);
+
+ infoContainer.append(headerDiv);
+
+ // Show cell code preview
+ if (data.cellCode) {
+ const codeHeaderDiv = document.createElement("div");
+ codeHeaderDiv.classList.add(
+ "text-xs",
+ "font-medium",
+ "text-muted-foreground",
+ );
+ codeHeaderDiv.textContent = "Code:";
+ infoContainer.append(codeHeaderDiv);
+
+ const codeDiv = document.createElement("div");
+ codeDiv.classList.add(
+ "text-xs",
+ "font-mono",
+ "bg-muted",
+ "p-2",
+ "rounded",
+ "max-h-20",
+ "overflow-y-auto",
+ );
+ codeDiv.textContent =
+ data.cellCode.slice(0, 200) +
+ (data.cellCode.length > 200 ? "..." : "");
+ infoContainer.append(codeDiv);
+ }
+
+ // Show output preview
+ if (data.processedContent) {
+ const outputHeaderDiv = document.createElement("div");
+ outputHeaderDiv.classList.add(
+ "text-xs",
+ "font-medium",
+ "text-muted-foreground",
+ "mt-2",
+ );
+ outputHeaderDiv.textContent = "Output Preview:";
+ infoContainer.append(outputHeaderDiv);
+
+ const outputDiv = document.createElement("div");
+ outputDiv.classList.add(
+ "text-xs",
+ "bg-muted",
+ "p-2",
+ "rounded",
+ "max-h-24",
+ "overflow-y-auto",
+ "mb-2",
+ );
+ outputDiv.textContent =
+ data.processedContent.slice(0, 300) +
+ (data.processedContent.length > 300 ? "..." : "");
+ infoContainer.append(outputDiv);
+ }
+
+ if (data.outputType === "media") {
+ const mediaDiv = document.createElement("div");
+ mediaDiv.classList.add(
+ "text-xs",
+ "text-muted-foreground",
+ "italic",
+ "mb-2",
+ );
+ mediaDiv.textContent =
+ "Contains media content (image, SVG, or canvas)";
+ infoContainer.append(mediaDiv);
+ }
+
+ return infoContainer;
+ },
+ };
+ }
+
+ formatContext(item: CellOutputContextItem): string {
+ const { data } = item;
+
+ const contextData = {
+ name: data.cellName,
+ cellId: data.cellId,
+ outputType: data.outputType,
+ mimetype: data.output.mimetype,
+ } as const;
+
+ let details = `Cell Code:\n${data.cellCode}\n\n`;
+
+ if (data.outputType === "text" && data.processedContent) {
+ details += `Output:\n${data.processedContent}`;
+ } else if (data.outputType === "media") {
+ details += `Media Output: Contains ${data.output.mimetype} content`;
+ if (data.imageUrl) {
+ details += `\nImage URL: ${data.imageUrl}`;
+ }
+ }
+
+ return contextToXml({
+ type: this.contextType,
+ data: contextData,
+ details,
+ });
+ }
+
+ /** Get attachments for cell output items that have shouldDownloadImage=true */
+ override async getAttachments(
+ items: CellOutputContextItem[],
+ ): Promise<ChatAttachment[]> {
+ // Filter items that need image downloading
+ const itemsNeedingDownload = items.filter(
+ (item) =>
+ item.data.shouldDownloadImage && item.data.outputType === "media",
+ );
+
+ if (itemsNeedingDownload.length === 0) {
+ return [];
+ }
+
+ // Prepare download requests
+ const downloadRequests = itemsNeedingDownload.flatMap((item) => {
+ const outputElement = document.getElementById(
+ CellOutputId.create(item.data.cellId),
+ );
+ if (!outputElement) {
+ Logger.warn(`Output element not found for cell ${item.data.cellId}`);
+ return [];
+ }
+ return {
+ cellId: item.data.cellId,
+ cellName: item.data.cellName,
+ mimetype: item.data.output.mimetype,
+ element: outputElement,
+ };
+ });
+
+ try {
+ return await Promise.all(
+ downloadRequests.map(async (item) => ({
+ name: `${item.cellName}-output-screenshot`,
+ contentType: "image/png",
+ url: await toPng(item.element),
+ })),
+ );
+ } catch (error) {
+ Logger.error("Error downloading cell output images:", error);
+ return [];
+ }
+ }
+}
diff --git a/frontend/src/core/ai/context/providers/common.ts b/frontend/src/core/ai/context/providers/common.ts
index 9976a3e4e7..3c0869b75b 100644
--- a/frontend/src/core/ai/context/providers/common.ts
+++ b/frontend/src/core/ai/context/providers/common.ts
@@ -4,5 +4,6 @@ export const Boosts = {
LOCAL_TABLE: 5,
REMOTE_TABLE: 4,
VARIABLE: 3,
- ERROR: 2,
+ CELL_OUTPUT: 2,
+ ERROR: 1,
} as const;
diff --git a/frontend/src/core/ai/context/registry.ts b/frontend/src/core/ai/context/registry.ts
index 05b20f7307..3280c5fa7b 100644
--- a/frontend/src/core/ai/context/registry.ts
+++ b/frontend/src/core/ai/context/registry.ts
@@ -2,6 +2,10 @@
import type { Completion } from "@codemirror/autocomplete";
import type { Resource } from "@marimo-team/codemirror-mcp";
+import { Memoize } from "typescript-memoize";
+import type { ChatAttachment } from "@/core/ai/types";
+import { Logger } from "@/utils/Logger";
+import { MultiMap } from "@/utils/multi-map";
import type { TypedString } from "@/utils/typed";
/**
@@ -41,6 +45,12 @@ export abstract class AIContextProvider<
/** Format completion */
abstract formatCompletion(item: T): Completion;
+ /** Get attachments for context items (optional, async) */
+ async getAttachments(_items: T[]): Promise<ChatAttachment[]> {
+ // Default implementation returns no attachments
+ return [];
+ }
+
asURI(id: string): ContextLocatorId {
return `${this.contextType}://${id}` as ContextLocatorId;
}
@@ -118,6 +128,7 @@ export class AIContextRegistry<T extends AIContextItem> {
);
}
+ @Memoize()
getAllItems(): T[] {
return [...this.providers].flatMap((provider) => provider.getItems());
}
@@ -177,4 +188,72 @@ export class AIContextRegistry<T extends AIContextItem> {
})
.join("\n\n");
}
+
+ /**
+ * Get attachments for mentioned items
+ */
+ async getAttachmentsForContext(
+ contextIds: ContextLocatorId[],
+ ): Promise<ChatAttachment[]> {
+ const allItems = new Map<ContextLocatorId, T>(
+ this.getAllItems().map((item) => [item.uri as ContextLocatorId, item]),
+ );
+
+ const contextInfo: T[] = [];
+ for (const contextId of contextIds) {
+ const item = allItems.get(contextId);
+ if (item) {
+ contextInfo.push(item);
+ }
+ }
+
+ if (contextInfo.length === 0) {
+ return [];
+ }
+
+ // Group items by provider type to batch attachment requests
+ const itemsByProvider = new MultiMap<string, T>();
+ for (const item of contextInfo) {
+ const providerType = item.type;
+ itemsByProvider.add(providerType, item);
+ }
+
+ // Collect attachments from all providers
+ const attachmentPromises = Array.from(itemsByProvider.entries()).map(
+ async ([providerType, items]) => {
+ const provider = this.getProvider(providerType);
+ if (!provider) {
+ return [];
+ }
+ try {
+ return await provider.getAttachments(items);
+ } catch (error) {
+ Logger.error("Error getting attachments from provider", error);
+ return [];
+ }
+ },
+ );
+
+ const attachmentResults = await Promise.all(attachmentPromises);
+ const results = attachmentResults.flat();
+
+ // Print attachments to the console with a rich image preview
+ if (import.meta.env.DEV) {
+ for (const attachment of results) {
+ // If it's an image, print a rich preview
+ if (
+ /^data:image\/(png|jpeg|jpg|gif|svg\+xml);base64,/.test(
+ attachment.url,
+ )
+ ) {
+ Logger.debug(
+ "%c ",
+ `font-size:1px;padding:140px 180px;background:url('${attachment.url}') no-repeat;background-size:contain;`,
+ );
+ }
+ }
+ }
+
+ return results;
+ }
}
diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.py
index 1d76d7c40a..8ab91e34a8 100644
--- a/marimo/_ai/_convert.py
+++ b/marimo/_ai/_convert.py
@@ -404,7 +404,12 @@ def _extract_text(url: str) -> str:
if url.startswith("data:"):
# extract base64 encoding from url
data = url.split(",")[1]
- return base64.b64decode(data).decode("utf-8")
+ raw = base64.b64decode(data)
+ try:
+ return raw.decode("utf-8")
+ except UnicodeDecodeError:
+ # fallback: try latin1
+ return raw.decode("latin1")
else:
return url