A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: cell output for `@` context w/ attachments for media output (#6240)
Myles Scolnick03e85b0a5452
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsxindex f7e2cb1e99..a91d215375 100644--- a/frontend/src/components/chat/chat-panel.tsx+++ b/frontend/src/components/chat/chat-panel.tsx@@ -6,6 +6,7 @@ import { storePrompt } from "@marimo-team/codemirror-ai";import type { ReactCodeMirrorRef } from "@uiw/react-codemirror";import {DefaultChatTransport,+ type FileUIPart,lastAssistantMessageIsCompleteWithToolCalls,type ToolUIPart,} from "ai";@@ -46,11 +47,13 @@ import {type ChatId,chatStateAtom,} from "@/core/ai/state";+import type { ChatAttachment } from "@/core/ai/types";import { aiAtom, aiEnabledAtom } from "@/core/config/config";import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";import { FeatureFlagged } from "@/core/config/feature-flag";import { useRequestClient } from "@/core/network/requests";import { useRuntimeManager } from "@/core/runtime/config";+import type { ChatMessage } from "@/plugins/impl/chat/types";import { ErrorBanner } from "@/plugins/impl/common/error-banner";import { cn } from "@/utils/cn";import { timeAgo } from "@/utils/dates";@@ -58,7 +61,7 @@ import { Logger } from "@/utils/Logger";import { AIModelDropdown } from "../ai/ai-model-dropdown";import { useOpenSettingsToTab } from "../app-config/state";import { PromptInput } from "../editor/ai/add-cell-with-ai";-import { getAICompletionBody } from "../editor/ai/completion-utils";+import { getAICompletionBodyWithAttachments } from "../editor/ai/completion-utils";import { PanelEmptyState } from "../editor/chrome/panels/empty-state";import { CopyClipboardIcon } from "../icons/copy-icon";import { Input } from "../ui/input";@@ -458,7 +461,7 @@ const ChatInput: React.FC<ChatInputProps> = memo(});return (- <div className="relative shrink-0 min-h-[80px] flex flex-col">+ <div className="relative shrink-0 min-h-[80px] flex flex-col border-t"><div className={cn("px-2 py-3 flex-1", inputClassName)}><PromptInputinputRef={inputRef}@@ -539,32 +542,54 @@ const ChatPanelBody = () => {transport: new DefaultChatTransport({api: runtimeManager.getAiURL("chat").toString(),headers: runtimeManager.headers(),- prepareSendMessagesRequest: (options) => {- let input = "";- for (const message of options.messages) {- const textParts = message.parts.filter((p) => p.type === "text");- input += textParts.map((p) => p.text).join("\n");+ prepareSendMessagesRequest: async (options) => {+ // Map from parts to a single string+ function toContent(parts: UIMessage["parts"]): string {+ return parts+ .map((part) => (part.type === "text" ? part.text : ""))+ .join("\n");+ }++ // Map from ai/sdk files to our attachment type+ function toAttachment(p: FileUIPart): ChatAttachment {+ return {+ contentType: p.mediaType,+ name: p.filename,+ url: p.url,+ };+ }++ const input = toContent(options.messages.flatMap((m) => m.parts));+ const completionBody = await getAICompletionBodyWithAttachments({+ input,+ });++ // Map from UIMessage to our ChatMessage type+ // If it's the last message, add the attachments from the completion body+ function mapMessage(+ m: UIMessage,+ isLastMessage: boolean,+ ): ChatMessage & { attachments: ChatAttachment[] } {+ const attachments = m.parts.flatMap((p) =>+ p.type === "file" ? toAttachment(p) : [],+ );+ if (isLastMessage) {+ attachments.push(...completionBody.attachments);+ }+ return {+ role: m.role,+ content: toContent(m.parts),+ attachments: attachments,+ };}- const completionBody = getAICompletionBody({ input });return {body: {...options,- ...completionBody,- messages: options.messages.map((m) => ({+ ...completionBody.body,+ messages: options.messages.map((m, idx) => ({...m,- attachments: m.parts.flatMap((p) =>- p.type === "file"- ? {- contentType: p.mediaType,- name: p.filename,- url: p.url,- }- : [],- ),- content: m.parts- ?.map((p) => ("text" in p ? p.text : ""))- .join("\n"),+ ...mapMessage(m, idx === options.messages.length - 1),})),},};diff --git a/frontend/src/components/editor/ai/completion-utils.ts b/frontend/src/components/editor/ai/completion-utils.tsindex 8bbd4c0a93..8c631bc6dd 100644--- a/frontend/src/components/editor/ai/completion-utils.ts+++ b/frontend/src/components/editor/ai/completion-utils.ts@@ -6,23 +6,32 @@ import type {CompletionSource,} from "@codemirror/autocomplete";import { getAIContextRegistry } from "@/core/ai/context/context";+import type { ChatAttachment } from "@/core/ai/types";import { getCodes } from "@/core/codemirror/copilot/getCodes";import type { AiCompletionRequest } from "@/core/network/types";import { store } from "@/core/state/jotai";import { Logger } from "@/utils/Logger";+interface Opts {+ input: string;+}++interface AICompletionBodyWithAttachments {+ body: Omit<AiCompletionRequest, "language" | "prompt" | "code">;+ attachments: ChatAttachment[];+}+/*** Gets the request body for the AI completion API.*/export function getAICompletionBody({input,-}: {- input: string;-}): Omit<AiCompletionRequest, "language" | "prompt" | "code"> {+}: Opts): Omit<AiCompletionRequest, "language" | "prompt" | "code"> {let contextString = "";+// Skip if no '@' in the inputif (input.includes("@")) {- contextString = extractDatasetsAndVariables(input);+ contextString = extractTaggedContext(input);Logger.debug("Included context", contextString);}@@ -37,11 +46,49 @@ export function getAICompletionBody({}/**- * Extracts datasets and variables from the input.+ * Gets the request body and attachments for the AI completion API.+ */+export async function getAICompletionBodyWithAttachments({+ input,+}: Opts): Promise<AICompletionBodyWithAttachments> {+ let contextString = "";+ let attachments: ChatAttachment[] = [];++ // Skip if no '@' in the input+ if (input.includes("@")) {+ const registry = getAIContextRegistry(store);+ const contextIds = registry.parseAllContextIds(input);++ // Get context string+ contextString = registry.formatContextForAI(contextIds);++ // Get attachments+ try {+ attachments = await registry.getAttachmentsForContext(contextIds);+ Logger.debug("Included attachments", attachments.length);+ } catch (error) {+ Logger.error("Error getting attachments:", error);+ }+ }++ return {+ body: {+ includeOtherCode: getCodes(""),+ context: {+ plainText: contextString,+ schema: [],+ variables: [],+ },+ },+ attachments,+ };+}++/**+ * Extracts datasets, variables and other context from the input.* References are with @<name> in the input.- * Prioritizes datasets over variables if there's a name conflict.*/-function extractDatasetsAndVariables(input: string): string {+function extractTaggedContext(input: string): string {const registry = getAIContextRegistry(store);const contextIds = registry.parseAllContextIds(input);return registry.formatContextForAI(contextIds);diff --git a/frontend/src/core/ai/context/__tests__/registry.test.ts b/frontend/src/core/ai/context/__tests__/registry.test.tsindex d8cbdbe832..b73bffb3ce 100644--- a/frontend/src/core/ai/context/__tests__/registry.test.ts+++ b/frontend/src/core/ai/context/__tests__/registry.test.ts@@ -2,8 +2,9 @@import type { Completion } from "@codemirror/autocomplete";import { createStore } from "jotai";-import { beforeEach, describe, expect, it } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";import { MockNotebook } from "@/__mocks__/notebook";+import type { ChatAttachment } from "@/core/ai/types";import { notebookAtom } from "@/core/cells/cells";import { CellId as CellIdClass } from "@/core/cells/ids";import {@@ -19,10 +20,26 @@ import {// Mock context item for testinginterface MockContextItem extends AIContextItem {- type: "mock";- data: { value: string };+ type: "mock" | "attachment";+ data: {+ value: string;+ needsAttachment?: boolean;+ };}+// Mock attachment data for testing+const mockAttachment1: ChatAttachment = {+ name: "test-image-1.png",+ contentType: "image/png",+ url: "data:image/png;base64,mockdata1",+};++const mockAttachment2: ChatAttachment = {+ name: "test-image-2.jpg",+ contentType: "image/jpeg",+ url: "data:image/jpeg;base64,mockdata2",+};+// Concrete implementation of AIContextProvider for testingclass MockContextProvider extends AIContextProvider<MockContextItem> {readonly title = "Mock Items";@@ -76,6 +93,44 @@ class MockContextProvider extends AIContextProvider<MockContextItem> {}}+// Test provider that supports attachments+class AttachmentContextProvider extends AIContextProvider<MockContextItem> {+ readonly title = "Attachment Items";+ readonly mentionPrefix = "@";+ readonly contextType = "attachment";++ constructor(+ private items: MockContextItem[] = [],+ private attachments: ChatAttachment[] = [],+ ) {+ super();+ }++ getItems(): MockContextItem[] {+ return this.items;+ }++ formatContext(item: MockContextItem): string {+ return `Attachment: ${item.name} (${item.data.value})`;+ }++ formatCompletion(item: MockContextItem): Completion {+ return this.createBasicCompletion(item);+ }++ override async getAttachments(+ items: MockContextItem[],+ ): Promise<ChatAttachment[]> {+ // Return attachments for items that need them+ const itemsNeedingAttachments = items.filter(+ (item) => item.data.needsAttachment,+ );+ return itemsNeedingAttachments+ .map((_, index) => this.attachments[index % this.attachments.length])+ .filter(Boolean);+ }+}+interface FileContextItem extends AIContextItem {type: "file";data: { value: string };@@ -667,4 +722,221 @@ describe("AIContextRegistry", () => {expect(formattedContext).toContain("Item 1");});});++ describe("attachment functionality", () => {+ let attachmentProvider: AttachmentContextProvider;+ let attachmentRegistry: AIContextRegistry<MockContextItem>;++ beforeEach(() => {+ const itemWithAttachment: MockContextItem = {+ uri: "attachment://item1" as ContextLocatorId,+ name: "item1",+ type: "attachment",+ description: "Item with attachment",+ data: { needsAttachment: true, value: "test1" },+ };++ const itemWithoutAttachment: MockContextItem = {+ uri: "attachment://item2" as ContextLocatorId,+ name: "item2",+ type: "attachment",+ description: "Item without attachment",+ data: { needsAttachment: false, value: "test2" },+ };++ attachmentProvider = new AttachmentContextProvider(+ [itemWithAttachment, itemWithoutAttachment],+ [mockAttachment1, mockAttachment2],+ );++ attachmentRegistry = new AIContextRegistry<MockContextItem>().register(+ attachmentProvider,+ );+ });++ describe("getAttachmentsForContext", () => {+ it("should return empty array for empty context IDs", async () => {+ const attachments = await attachmentRegistry.getAttachmentsForContext(+ [],+ );+ expect(attachments).toEqual([]);+ });++ it("should return empty array for non-existent context IDs", async () => {+ const nonExistentIds = ["attachment://nonexistent" as ContextLocatorId];+ const attachments =+ await attachmentRegistry.getAttachmentsForContext(nonExistentIds);+ expect(attachments).toEqual([]);+ });++ it("should get attachments from provider that supports them", async () => {+ const contextIds = ["attachment://item1" as ContextLocatorId];+ const attachments =+ await attachmentRegistry.getAttachmentsForContext(contextIds);++ expect(attachments).toHaveLength(1);+ expect(attachments[0]).toEqual(mockAttachment1);+ });++ it("should not get attachments for items that don't need them", async () => {+ const contextIds = ["attachment://item2" as ContextLocatorId];+ const attachments =+ await attachmentRegistry.getAttachmentsForContext(contextIds);++ expect(attachments).toHaveLength(0);+ });++ it("should not get attachments from providers with default implementation", async () => {+ const simpleRegistry =+ new AIContextRegistry<MockContextItem>().register(mockProvider);++ const contextIds = ["mock://item1" as ContextLocatorId];+ const attachments =+ await simpleRegistry.getAttachmentsForContext(contextIds);++ expect(attachments).toHaveLength(0);+ });++ it("should handle multiple context IDs from different providers", async () => {+ const mixedRegistry = new AIContextRegistry<MockContextItem>()+ .register(attachmentProvider)+ .register(mockProvider);++ const contextIds = [+ "attachment://item1" as ContextLocatorId, // should have attachment+ "attachment://item2" as ContextLocatorId, // should not have attachment+ "mock://item1" as ContextLocatorId, // provider doesn't support attachments+ ];++ const attachments =+ await mixedRegistry.getAttachmentsForContext(contextIds);++ // Only attachment://item1 should contribute an attachment+ expect(attachments).toHaveLength(1);+ expect(attachments[0]).toEqual(mockAttachment1);+ });++ it("should handle multiple items needing attachments", async () => {+ const multiItem1: MockContextItem = {+ uri: "attachment://multi1" as ContextLocatorId,+ name: "multi1",+ type: "attachment",+ description: "Multi item 1",+ data: { needsAttachment: true, value: "multi1" },+ };++ const multiItem2: MockContextItem = {+ uri: "attachment://multi2" as ContextLocatorId,+ name: "multi2",+ type: "attachment",+ description: "Multi item 2",+ data: { needsAttachment: true, value: "multi2" },+ };++ const multiProvider = new AttachmentContextProvider(+ [multiItem1, multiItem2],+ [mockAttachment1, mockAttachment2],+ );++ const multiRegistry = new AIContextRegistry<MockContextItem>().register(+ multiProvider,+ );++ const contextIds = [+ "attachment://multi1" as ContextLocatorId,+ "attachment://multi2" as ContextLocatorId,+ ];++ const attachments =+ await multiRegistry.getAttachmentsForContext(contextIds);++ expect(attachments).toHaveLength(2);+ expect(attachments).toContainEqual(mockAttachment1);+ expect(attachments).toContainEqual(mockAttachment2);+ });++ it("should handle provider errors gracefully", async () => {+ const errorProvider = new AttachmentContextProvider([+ {+ uri: "attachment://error-item" as ContextLocatorId,+ name: "error-item",+ type: "attachment",+ description: "Item that causes error",+ data: { needsAttachment: true, value: "error" },+ },+ ]);++ // Override getAttachments to throw an error+ errorProvider.getAttachments = vi+ .fn()+ .mockRejectedValue(new Error("Attachment error"));++ const errorRegistry = new AIContextRegistry<MockContextItem>()+ .register(errorProvider)+ .register(mockProvider);++ const contextIds = [+ "attachment://error-item" as ContextLocatorId,+ "mock://item1" as ContextLocatorId,+ ];++ // Should not throw and should handle the error gracefully+ const attachments =+ await errorRegistry.getAttachmentsForContext(contextIds);+ expect(attachments).toEqual([]);+ });+ });++ describe("provider batching for attachments", () => {+ it("should batch multiple items to the same provider", async () => {+ const getAttachmentsSpy = vi.spyOn(+ attachmentProvider,+ "getAttachments",+ );++ const contextIds = [+ "attachment://item1" as ContextLocatorId,+ "attachment://item2" as ContextLocatorId,+ ];++ await attachmentRegistry.getAttachmentsForContext(contextIds);++ // Should call getAttachments once with both items+ expect(getAttachmentsSpy).toHaveBeenCalledTimes(1);+ expect(getAttachmentsSpy).toHaveBeenCalledWith(+ expect.arrayContaining([+ expect.objectContaining({ uri: "attachment://item1" }),+ expect.objectContaining({ uri: "attachment://item2" }),+ ]),+ );+ });++ it("should call different providers separately", async () => {+ const mixedRegistry = new AIContextRegistry<MockContextItem>()+ .register(attachmentProvider)+ .register(mockProvider);++ const attachmentSpy = vi.spyOn(attachmentProvider, "getAttachments");+ const mockSpy = vi.spyOn(mockProvider, "getAttachments");++ const contextIds = [+ "attachment://item1" as ContextLocatorId,+ "mock://item1" as ContextLocatorId,+ ];++ await mixedRegistry.getAttachmentsForContext(contextIds);++ // Should call each provider once+ expect(attachmentSpy).toHaveBeenCalledTimes(1);+ expect(mockSpy).toHaveBeenCalledTimes(1);++ expect(attachmentSpy).toHaveBeenCalledWith([+ expect.objectContaining({ uri: "attachment://item1" }),+ ]);+ expect(mockSpy).toHaveBeenCalledWith([+ expect.objectContaining({ uri: "mock://item1" }),+ ]);+ });+ });+ });});diff --git a/frontend/src/core/ai/context/context.ts b/frontend/src/core/ai/context/context.tsindex 591f0f40d0..c4b7e80f85 100644--- a/frontend/src/core/ai/context/context.ts+++ b/frontend/src/core/ai/context/context.ts@@ -3,6 +3,7 @@import { allTablesAtom } from "@/core/datasets/data-source-connections";import type { JotaiStore } from "@/core/state/jotai";import { variablesAtom } from "@/core/variables/state";+import { CellOutputContextProvider } from "./providers/cell-output";import { ErrorContextProvider } from "./providers/error";import { TableContextProvider } from "./providers/tables";import { VariableContextProvider } from "./providers/variable";@@ -14,5 +15,6 @@ export function getAIContextRegistry(store: JotaiStore) {return new AIContextRegistry().register(new TableContextProvider(tablesMap)).register(new VariableContextProvider(variables, tablesMap))- .register(new ErrorContextProvider(store));+ .register(new ErrorContextProvider(store))+ .register(new CellOutputContextProvider(store));}diff --git a/frontend/src/core/ai/context/providers/__tests__/cell-output.test.ts b/frontend/src/core/ai/context/providers/__tests__/cell-output.test.tsnew file mode 100644index 0000000000..796fdc9559--- /dev/null+++ b/frontend/src/core/ai/context/providers/__tests__/cell-output.test.ts@@ -0,0 +1,374 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { beforeEach, describe, expect, it, vi } from "vitest";+import { Mocks } from "@/__mocks__/common";++// Mock the external dependencies+vi.mock("html-to-image", () => ({+ toPng: vi.fn().mockResolvedValue("data:image/png;base64,mockbase64data"),+}));++vi.mock("@/utils/Logger", () => ({+ Logger: Mocks.quietLogger(),+}));++import type { NotebookState } from "@/core/cells/cells";+import type { CellId } from "@/core/cells/ids";+import type { OutputMessage } from "@/core/kernel/messages";+import type { JotaiStore } from "@/core/state/jotai";+import { CellOutputContextProvider } from "../cell-output";++// Test helper to create mock store+function createMockStore(notebook: NotebookState): JotaiStore {+ const store = {+ get: vi.fn().mockReturnValue(notebook),+ } as unknown as JotaiStore;+ return store;+}++describe("CellOutputContextProvider", () => {+ let provider: CellOutputContextProvider;+ let mockStore: JotaiStore;+ let mockNotebook: NotebookState;++ beforeEach(() => {+ // Create a basic mock notebook state+ mockNotebook = {+ cellIds: {+ inOrderIds: ["cell1" as CellId, "cell2" as CellId, "cell3" as CellId],+ },+ cellData: {+ cell1: {+ id: "cell1" as CellId,+ name: "My Cell",+ code: "print('hello world')",+ },+ cell2: {+ id: "cell2" as CellId,+ name: "",+ code: "import matplotlib.pyplot as plt\nplt.plot([1,2,3])",+ },+ cell3: {+ id: "cell3" as CellId,+ name: "Empty Cell",+ code: "# no output",+ },+ },+ cellRuntime: {+ cell1: {+ output: {+ mimetype: "text/plain",+ data: "hello world",+ } as OutputMessage,+ },+ cell2: {+ output: {+ mimetype: "image/png",+ data: "base64imagedata",+ } as OutputMessage,+ },+ cell3: {+ output: null,+ },+ },+ } as unknown as NotebookState;++ mockStore = createMockStore(mockNotebook);+ provider = new CellOutputContextProvider(mockStore);+ });++ describe("basic properties", () => {+ it("should have correct title and context type", () => {+ expect(provider.title).toBe("Cell Outputs");+ expect(provider.contextType).toBe("cell-output");+ expect(provider.mentionPrefix).toBe("@");+ });+ });++ describe("getItems", () => {+ it("should return only cells with outputs", () => {+ const items = provider.getItems();++ // Should have 2 items (cell1 and cell2, but not cell3 which has no output)+ expect(items).toHaveLength(2);++ const cellIds = items.map((item) => item.data.cellId);+ expect(cellIds).toContain("cell1");+ expect(cellIds).toContain("cell2");+ expect(cellIds).not.toContain("cell3");+ });++ it("should correctly identify text vs media outputs", () => {+ const items = provider.getItems();++ const textItem = items.find((item) => item.data.cellId === "cell1");+ const mediaItem = items.find((item) => item.data.cellId === "cell2");++ expect(textItem?.data.outputType).toBe("text");+ expect(mediaItem?.data.outputType).toBe("media");+ });++ it("should process text content correctly", () => {+ const items = provider.getItems();+ const textItem = items.find((item) => item.data.cellId === "cell1");++ expect(textItem?.data.processedContent).toBe("hello world");+ });++ it("should mark media items for download when no direct URL available", () => {+ const items = provider.getItems();+ const mediaItem = items.find((item) => item.data.cellId === "cell2");++ expect(mediaItem?.data.shouldDownloadImage).toBe(true);+ expect(mediaItem?.data.imageUrl).toBeUndefined();+ });++ it("should include cell code and names", () => {+ const items = provider.getItems();++ const item1 = items.find((item) => item.data.cellId === "cell1");+ const item2 = items.find((item) => item.data.cellId === "cell2");++ expect(item1?.data.cellName).toBe("My Cell");+ expect(item1?.data.cellCode).toBe("print('hello world')");++ expect(item2?.data.cellName).toBe("cell-1");+ expect(item2?.data.cellCode).toBe(+ "import matplotlib.pyplot as plt\nplt.plot([1,2,3])",+ );+ });+ });++ describe("formatContext", () => {+ it("should format text output context correctly", () => {+ const items = provider.getItems();+ const textItem = items.find((item) => item.data.cellId === "cell1");++ if (!textItem) throw new Error("Text item not found");++ const context = provider.formatContext(textItem);++ expect(context).toContain("cell-output");+ expect(context).toContain("My Cell");+ expect(context).toContain("Cell Code:");+ expect(context).toContain("print('hello world')");+ expect(context).toContain("Output:");+ expect(context).toContain("hello world");+ });++ it("should format media output context correctly", () => {+ const items = provider.getItems();+ const mediaItem = items.find((item) => item.data.cellId === "cell2");++ if (!mediaItem) throw new Error("Media item not found");++ const context = provider.formatContext(mediaItem);++ expect(context).toContain("cell-output");+ expect(context).toContain("cell-1");+ expect(context).toContain("Cell Code:");+ expect(context).toContain("import matplotlib.pyplot as plt");+ expect(context).toContain("Media Output: Contains image/png content");+ });+ });++ describe("formatCompletion", () => {+ it("should create proper completion object", () => {+ const items = provider.getItems();+ const item = items[0];++ const completion = provider.formatCompletion(item);++ expect(completion.label).toMatch(/@.+/);+ expect(completion.displayLabel).toBe(item.data.cellName);+ expect(completion.detail).toContain("output");+ expect(completion.type).toBe("cell-output");+ expect(completion.section).toBe("Cell Output");+ expect(typeof completion.info).toBe("function");+ });+ });+});++// Test utility functions separately+describe("Cell output utility functions", () => {+ // We need to extract these functions to test them, or make them exported+ // For now, let's test through the public interface++ describe("media type detection", () => {+ let provider: CellOutputContextProvider;+ let mockStore: JotaiStore;++ beforeEach(() => {+ const mockNotebook = {+ cellIds: { inOrderIds: [] },+ cellData: {},+ cellRuntime: {},+ } as unknown as NotebookState;++ mockStore = createMockStore(mockNotebook);+ provider = new CellOutputContextProvider(mockStore);+ });++ it("should detect image mimetypes as media", () => {+ const testCases = [+ { mimetype: "image/png", data: "test", expected: "media" },+ { mimetype: "image/jpeg", data: "test", expected: "media" },+ { mimetype: "image/gif", data: "test", expected: "media" },+ { mimetype: "image/svg+xml", data: "test", expected: "media" },+ ];++ for (const testCase of testCases) {+ mockStore = createMockStore({+ cellIds: { inOrderIds: ["test" as CellId] },+ cellData: {+ test: { id: "test" as CellId, name: "", code: "" },+ },+ cellRuntime: {+ test: {+ output: {+ mimetype: testCase.mimetype,+ data: testCase.data,+ } as OutputMessage,+ },+ },+ } as unknown as NotebookState);++ provider = new CellOutputContextProvider(mockStore);+ const items = provider.getItems();++ expect(items[0]?.data.outputType).toBe(testCase.expected);+ }+ });++ it("should detect HTML with media tags as media", () => {+ const htmlWithImage = '<div><img src="test.png" /></div>';+ const htmlWithCanvas = '<canvas width="100" height="100"></canvas>';+ const htmlWithSvg = '<svg><circle r="50" /></svg>';++ const testCases = [+ { data: htmlWithImage, expected: "media" },+ { data: htmlWithCanvas, expected: "media" },+ { data: htmlWithSvg, expected: "media" },+ { data: "<p>Just text</p>", expected: "text" },+ ];++ for (const testCase of testCases) {+ mockStore = createMockStore({+ cellIds: { inOrderIds: ["test" as CellId] },+ cellData: {+ test: { id: "test" as CellId, name: "", code: "" },+ },+ cellRuntime: {+ test: {+ output: {+ mimetype: "text/html",+ data: testCase.data,+ } as OutputMessage,+ },+ },+ } as unknown as NotebookState);++ provider = new CellOutputContextProvider(mockStore);+ const items = provider.getItems();++ expect(items[0]?.data.outputType).toBe(testCase.expected);+ }+ });++ it("should detect text mimetypes correctly", () => {+ const testCases = [+ { mimetype: "text/plain", expected: "text" },+ {+ mimetype: "text/html",+ data: "<p>No media tags</p>",+ expected: "text",+ },+ { mimetype: "application/json", expected: "text" },+ ];++ for (const testCase of testCases) {+ mockStore = createMockStore({+ cellIds: { inOrderIds: ["test" as CellId] },+ cellData: {+ test: { id: "test" as CellId, name: "", code: "" },+ },+ cellRuntime: {+ test: {+ output: {+ mimetype: testCase.mimetype,+ data: testCase.data || "test data",+ } as OutputMessage,+ },+ },+ } as unknown as NotebookState);++ provider = new CellOutputContextProvider(mockStore);+ const items = provider.getItems();++ expect(items[0]?.data.outputType).toBe(testCase.expected);+ }+ });+ });++ describe("HTML content parsing", () => {+ let provider: CellOutputContextProvider;+ let mockStore: JotaiStore;++ beforeEach(() => {+ // Mock DOM methods for HTML parsing+ const mockDiv = {+ innerHTML: "",+ textContent: "",+ innerText: "",+ };++ global.document = {+ createElement: vi.fn().mockReturnValue(mockDiv),+ } as unknown as Document;++ const mockNotebook = {+ cellIds: { inOrderIds: [] },+ cellData: {},+ cellRuntime: {},+ } as unknown as NotebookState;++ mockStore = createMockStore(mockNotebook);+ provider = new CellOutputContextProvider(mockStore);+ });++ it("should extract text content from HTML", () => {+ const htmlContent = "<p>Hello <strong>world</strong>!</p>";++ // Mock the DOM parsing result+ const mockDiv = {+ innerHTML: "",+ textContent: "Hello world!",+ innerText: "Hello world!",+ };++ (+ global.document.createElement as ReturnType<typeof vi.fn>+ ).mockReturnValue(mockDiv);++ mockStore = createMockStore({+ cellIds: { inOrderIds: ["test" as CellId] },+ cellData: {+ test: { id: "test" as CellId, name: "", code: "" },+ },+ cellRuntime: {+ test: {+ output: {+ mimetype: "text/html",+ data: htmlContent,+ } as OutputMessage,+ },+ },+ } as unknown as NotebookState);++ provider = new CellOutputContextProvider(mockStore);+ const items = provider.getItems();++ expect(items[0]?.data.processedContent).toBe("Hello world!");+ });+ });+});diff --git a/frontend/src/core/ai/context/providers/__tests__/error.test.ts b/frontend/src/core/ai/context/providers/__tests__/error.test.tsindex c7b095dcb8..85cb71145d 100644--- a/frontend/src/core/ai/context/providers/__tests__/error.test.ts+++ b/frontend/src/core/ai/context/providers/__tests__/error.test.ts@@ -6,6 +6,7 @@ import { beforeEach, describe, expect, it } from "vitest";import { MockNotebook } from "@/__mocks__/notebook";import { notebookAtom } from "@/core/cells/cells";import { type CellId, CellId as CellIdClass } from "@/core/cells/ids";+import { Boosts } from "../common";import { ErrorContextProvider } from "../error";describe("ErrorContextProvider", () => {@@ -121,7 +122,7 @@ describe("ErrorContextProvider", () => {label: "@Errors",displayLabel: "Errors",detail: "2 errors",- boost: 2, // Boosts.ERROR+ boost: Boosts.ERROR,type: "error",apply: "@Errors",section: "Errors",@@ -181,7 +182,7 @@ describe("ErrorContextProvider", () => {expect(completion).toMatchObject({label: "Error",displayLabel: "Error",- boost: 2, // Boosts.ERROR+ boost: Boosts.ERROR,});});});diff --git a/frontend/src/core/ai/context/providers/cell-output.ts b/frontend/src/core/ai/context/providers/cell-output.tsnew file mode 100644index 0000000000..f12b29af0f--- /dev/null+++ b/frontend/src/core/ai/context/providers/cell-output.ts@@ -0,0 +1,347 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { Completion } from "@codemirror/autocomplete";+import { toPng } from "html-to-image";+import type { ChatAttachment } from "@/core/ai/types";+import { notebookAtom } from "@/core/cells/cells";+import { type CellId, CellOutputId } from "@/core/cells/ids";+import { displayCellName } from "@/core/cells/names";+import { isOutputEmpty } from "@/core/cells/outputs";+import type { OutputMessage } from "@/core/kernel/messages";+import type { JotaiStore } from "@/core/state/jotai";+import { Logger } from "@/utils/Logger";+import { type AIContextItem, AIContextProvider } from "../registry";+import { contextToXml } from "../utils";+import { Boosts } from "./common";++export interface CellOutputContextItem extends AIContextItem {+ type: "cell-output";+ data: {+ cellId: CellId;+ cellName: string;+ cellCode: string;+ output: OutputMessage;+ outputType: "text" | "media";+ processedContent?: string;+ imageUrl?: string;+ shouldDownloadImage?: boolean;+ };+}++function isMediaMimetype(+ mimetype: OutputMessage["mimetype"] | undefined,+ htmlString: string,+): boolean {+ if (!mimetype) return false;++ const mediaPrefixes = ["image/", "video/", "audio/", "application/pdf"];+ if (mediaPrefixes.some((prefix) => mimetype.startsWith(prefix))) {+ return true;+ }+ const mediaIncludes = ["svg", "vega"];+ if (mediaIncludes.some((include) => mimetype.includes(include))) {+ return true;+ }++ // If it is HTML, we need to check if it contains a media tag+ if (mimetype === "text/html") {+ const mediaTags = [+ "<img",+ "<video",+ "<audio",+ "<iframe",+ "<canvas",+ "<svg",+ "<marimo-ui-element",+ ];+ if (mediaTags.some((tag) => htmlString.includes(tag))) {+ return true;+ }+ }++ return false;+}++function parseHtmlContent(htmlString: string): string {+ try {+ // Create a temporary DOM element to parse HTML+ const tempDiv = document.createElement("div");+ tempDiv.innerHTML = htmlString;++ // Extract text content, removing HTML tags+ const textContent = tempDiv.textContent || tempDiv.innerText || "";++ // Clean up extra whitespace+ return textContent.replace(/\s+/g, " ").trim();+ } catch (error) {+ Logger.error("Error parsing HTML content:", error);+ // If parsing fails, return the original string+ return htmlString;+ }+}++export class CellOutputContextProvider extends AIContextProvider<CellOutputContextItem> {+ readonly title = "Cell Outputs";+ readonly mentionPrefix = "@";+ readonly contextType = "cell-output";++ constructor(private store: JotaiStore) {+ super();+ }++ getItems(): CellOutputContextItem[] {+ const notebook = this.store.get(notebookAtom);+ const items: CellOutputContextItem[] = [];++ for (const [cellIndex, cellId] of notebook.cellIds.inOrderIds.entries()) {+ const cellRuntime = notebook.cellRuntime[cellId];++ // Filter to only cells with output+ if (!cellRuntime?.output || isOutputEmpty(cellRuntime.output)) {+ continue;+ }++ const cellData = notebook.cellData[cellId];+ const output = cellRuntime.output;+ const mimetype = output.mimetype;++ // Determine output type+ const isMedia = isMediaMimetype(mimetype, String(output.data));+ const outputType = isMedia ? "media" : "text";++ const cellName = displayCellName(cellData.name, cellIndex);++ let processedContent: string | undefined;+ let imageUrl: string | undefined;+ let shouldDownloadImage: boolean = false;++ // Process text content+ if (outputType === "text" && typeof output.data === "string") {+ if (mimetype === "text/html") {+ processedContent = parseHtmlContent(output.data);+ } else {+ processedContent = output.data;+ }+ }++ // Process media content - for now, we'll just note that it's media+ if (outputType === "media") {+ if (+ typeof output.data === "string" &&+ output.data.startsWith("data:")+ ) {+ imageUrl = output.data; // Data URL+ } else if (+ typeof output.data === "string" &&+ output.data.startsWith("http")+ ) {+ imageUrl = output.data; // External URL+ } else {+ // Download the image from the DOM element+ shouldDownloadImage = true;+ }+ }++ items.push({+ uri: this.asURI(cellId),+ name: cellName,+ type: this.contextType,+ description: `Cell output (${mimetype || "unknown"})`,+ data: {+ cellId,+ cellName,+ cellCode: cellData?.code || "",+ output,+ outputType,+ processedContent,+ imageUrl,+ shouldDownloadImage,+ },+ });+ }++ return items;+ }++ formatCompletion(item: CellOutputContextItem): Completion {+ const { data } = item;+ return {+ label: `@${data.cellName}`,+ displayLabel: data.cellName,+ detail: `${data.outputType} output`,+ boost: Boosts.CELL_OUTPUT,+ type: this.contextType,+ section: "Cell Output",+ apply: `@${data.cellName}`,+ info: () => {+ const infoContainer = document.createElement("div");+ infoContainer.classList.add(+ "mo-cm-tooltip",+ "docs-documentation",+ "min-w-[300px]",+ "max-w-[500px]",+ "flex",+ "flex-col",+ "gap-2",+ );++ const headerDiv = document.createElement("div");+ headerDiv.classList.add("flex", "flex-col", "gap-1");++ const nameDiv = document.createElement("div");+ nameDiv.classList.add("font-bold", "text-base");+ nameDiv.textContent = data.cellName;+ headerDiv.append(nameDiv);++ const descriptionDiv = document.createElement("div");+ descriptionDiv.classList.add("text-sm", "text-muted-foreground");+ headerDiv.append(descriptionDiv);++ infoContainer.append(headerDiv);++ // Show cell code preview+ if (data.cellCode) {+ const codeHeaderDiv = document.createElement("div");+ codeHeaderDiv.classList.add(+ "text-xs",+ "font-medium",+ "text-muted-foreground",+ );+ codeHeaderDiv.textContent = "Code:";+ infoContainer.append(codeHeaderDiv);++ const codeDiv = document.createElement("div");+ codeDiv.classList.add(+ "text-xs",+ "font-mono",+ "bg-muted",+ "p-2",+ "rounded",+ "max-h-20",+ "overflow-y-auto",+ );+ codeDiv.textContent =+ data.cellCode.slice(0, 200) ++ (data.cellCode.length > 200 ? "..." : "");+ infoContainer.append(codeDiv);+ }++ // Show output preview+ if (data.processedContent) {+ const outputHeaderDiv = document.createElement("div");+ outputHeaderDiv.classList.add(+ "text-xs",+ "font-medium",+ "text-muted-foreground",+ "mt-2",+ );+ outputHeaderDiv.textContent = "Output Preview:";+ infoContainer.append(outputHeaderDiv);++ const outputDiv = document.createElement("div");+ outputDiv.classList.add(+ "text-xs",+ "bg-muted",+ "p-2",+ "rounded",+ "max-h-24",+ "overflow-y-auto",+ "mb-2",+ );+ outputDiv.textContent =+ data.processedContent.slice(0, 300) ++ (data.processedContent.length > 300 ? "..." : "");+ infoContainer.append(outputDiv);+ }++ if (data.outputType === "media") {+ const mediaDiv = document.createElement("div");+ mediaDiv.classList.add(+ "text-xs",+ "text-muted-foreground",+ "italic",+ "mb-2",+ );+ mediaDiv.textContent =+ "Contains media content (image, SVG, or canvas)";+ infoContainer.append(mediaDiv);+ }++ return infoContainer;+ },+ };+ }++ formatContext(item: CellOutputContextItem): string {+ const { data } = item;++ const contextData = {+ name: data.cellName,+ cellId: data.cellId,+ outputType: data.outputType,+ mimetype: data.output.mimetype,+ } as const;++ let details = `Cell Code:\n${data.cellCode}\n\n`;++ if (data.outputType === "text" && data.processedContent) {+ details += `Output:\n${data.processedContent}`;+ } else if (data.outputType === "media") {+ details += `Media Output: Contains ${data.output.mimetype} content`;+ if (data.imageUrl) {+ details += `\nImage URL: ${data.imageUrl}`;+ }+ }++ return contextToXml({+ type: this.contextType,+ data: contextData,+ details,+ });+ }++ /** Get attachments for cell output items that have shouldDownloadImage=true */+ override async getAttachments(+ items: CellOutputContextItem[],+ ): Promise<ChatAttachment[]> {+ // Filter items that need image downloading+ const itemsNeedingDownload = items.filter(+ (item) =>+ item.data.shouldDownloadImage && item.data.outputType === "media",+ );++ if (itemsNeedingDownload.length === 0) {+ return [];+ }++ // Prepare download requests+ const downloadRequests = itemsNeedingDownload.flatMap((item) => {+ const outputElement = document.getElementById(+ CellOutputId.create(item.data.cellId),+ );+ if (!outputElement) {+ Logger.warn(`Output element not found for cell ${item.data.cellId}`);+ return [];+ }+ return {+ cellId: item.data.cellId,+ cellName: item.data.cellName,+ mimetype: item.data.output.mimetype,+ element: outputElement,+ };+ });++ try {+ return await Promise.all(+ downloadRequests.map(async (item) => ({+ name: `${item.cellName}-output-screenshot`,+ contentType: "image/png",+ url: await toPng(item.element),+ })),+ );+ } catch (error) {+ Logger.error("Error downloading cell output images:", error);+ return [];+ }+ }+}diff --git a/frontend/src/core/ai/context/providers/common.ts b/frontend/src/core/ai/context/providers/common.tsindex 9976a3e4e7..3c0869b75b 100644--- a/frontend/src/core/ai/context/providers/common.ts+++ b/frontend/src/core/ai/context/providers/common.ts@@ -4,5 +4,6 @@ export const Boosts = {LOCAL_TABLE: 5,REMOTE_TABLE: 4,VARIABLE: 3,- ERROR: 2,+ CELL_OUTPUT: 2,+ ERROR: 1,} as const;diff --git a/frontend/src/core/ai/context/registry.ts b/frontend/src/core/ai/context/registry.tsindex 05b20f7307..3280c5fa7b 100644--- a/frontend/src/core/ai/context/registry.ts+++ b/frontend/src/core/ai/context/registry.ts@@ -2,6 +2,10 @@import type { Completion } from "@codemirror/autocomplete";import type { Resource } from "@marimo-team/codemirror-mcp";+import { Memoize } from "typescript-memoize";+import type { ChatAttachment } from "@/core/ai/types";+import { Logger } from "@/utils/Logger";+import { MultiMap } from "@/utils/multi-map";import type { TypedString } from "@/utils/typed";/**@@ -41,6 +45,12 @@ export abstract class AIContextProvider</** Format completion */abstract formatCompletion(item: T): Completion;+ /** Get attachments for context items (optional, async) */+ async getAttachments(_items: T[]): Promise<ChatAttachment[]> {+ // Default implementation returns no attachments+ return [];+ }+asURI(id: string): ContextLocatorId {return `${this.contextType}://${id}` as ContextLocatorId;}@@ -118,6 +128,7 @@ export class AIContextRegistry<T extends AIContextItem> {);}+ @Memoize()getAllItems(): T[] {return [...this.providers].flatMap((provider) => provider.getItems());}@@ -177,4 +188,72 @@ export class AIContextRegistry<T extends AIContextItem> {}).join("\n\n");}++ /**+ * Get attachments for mentioned items+ */+ async getAttachmentsForContext(+ contextIds: ContextLocatorId[],+ ): Promise<ChatAttachment[]> {+ const allItems = new Map<ContextLocatorId, T>(+ this.getAllItems().map((item) => [item.uri as ContextLocatorId, item]),+ );++ const contextInfo: T[] = [];+ for (const contextId of contextIds) {+ const item = allItems.get(contextId);+ if (item) {+ contextInfo.push(item);+ }+ }++ if (contextInfo.length === 0) {+ return [];+ }++ // Group items by provider type to batch attachment requests+ const itemsByProvider = new MultiMap<string, T>();+ for (const item of contextInfo) {+ const providerType = item.type;+ itemsByProvider.add(providerType, item);+ }++ // Collect attachments from all providers+ const attachmentPromises = Array.from(itemsByProvider.entries()).map(+ async ([providerType, items]) => {+ const provider = this.getProvider(providerType);+ if (!provider) {+ return [];+ }+ try {+ return await provider.getAttachments(items);+ } catch (error) {+ Logger.error("Error getting attachments from provider", error);+ return [];+ }+ },+ );++ const attachmentResults = await Promise.all(attachmentPromises);+ const results = attachmentResults.flat();++ // Print attachments to the console with a rich image preview+ if (import.meta.env.DEV) {+ for (const attachment of results) {+ // If it's an image, print a rich preview+ if (+ /^data:image\/(png|jpeg|jpg|gif|svg\+xml);base64,/.test(+ attachment.url,+ )+ ) {+ Logger.debug(+ "%c ",+ `font-size:1px;padding:140px 180px;background:url('${attachment.url}') no-repeat;background-size:contain;`,+ );+ }+ }+ }++ return results;+ }}diff --git a/marimo/_ai/_convert.py b/marimo/_ai/_convert.pyindex 1d76d7c40a..8ab91e34a8 100644--- a/marimo/_ai/_convert.py+++ b/marimo/_ai/_convert.py@@ -404,7 +404,12 @@ def _extract_text(url: str) -> str:if url.startswith("data:"):# extract base64 encoding from urldata = url.split(",")[1]- return base64.b64decode(data).decode("utf-8")+ raw = base64.b64decode(data)+ try:+ return raw.decode("utf-8")+ except UnicodeDecodeError:+ # fallback: try latin1+ return raw.decode("latin1")else:return url