A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
feat: Create frontend tools framework and integrate to backend ai system (#6609)
Joaquin Coromina47bf5702b274
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsxindex 3c380139be..ecd1f4bf97 100644--- a/frontend/src/components/chat/chat-panel.tsx+++ b/frontend/src/components/chat/chat-panel.tsx@@ -44,6 +44,7 @@ import {type ChatId,chatStateAtom,} from "@/core/ai/state";+import { FRONTEND_TOOL_REGISTRY } from "@/core/ai/tools/registry";import { aiAtom, aiEnabledAtom } from "@/core/config/config";import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";import { FeatureFlagged } from "@/core/config/feature-flag";@@ -585,6 +586,7 @@ const ChatPanelBody = () => {return {body: {+ tools: FRONTEND_TOOL_REGISTRY.getToolSchemas(),...options,...completionBody,},diff --git a/frontend/src/components/chat/chat-utils.ts b/frontend/src/components/chat/chat-utils.tsindex 2f343b324d..dc4646a466 100644--- a/frontend/src/components/chat/chat-utils.ts+++ b/frontend/src/components/chat/chat-utils.ts@@ -2,6 +2,7 @@import type { components } from "@marimo-team/marimo-api";import type { FileUIPart, ToolUIPart, UIMessage } from "ai";+import { FRONTEND_TOOL_REGISTRY } from "@/core/ai/tools/registry";import type {InvokeAiToolRequest,InvokeAiToolResponse,@@ -117,15 +118,29 @@ export async function handleToolCall({};}) {try {- const response = await invokeAiTool({- toolName: toolCall.toolName,- arguments: toolCall.input,- });- addToolResult({- tool: toolCall.toolName,- toolCallId: toolCall.toolCallId,- output: response.result || response.error,- });+ if (FRONTEND_TOOL_REGISTRY.has(toolCall.toolName)) {+ // Invoke the frontend tool+ const response = await FRONTEND_TOOL_REGISTRY.invoke(+ toolCall.toolName,+ toolCall.input,+ );+ addToolResult({+ tool: toolCall.toolName,+ toolCallId: toolCall.toolCallId,+ output: response,+ });+ } else {+ // Invoke the backend/mcp tool+ const response = await invokeAiTool({+ toolName: toolCall.toolName,+ arguments: toolCall.input,+ });+ addToolResult({+ tool: toolCall.toolName,+ toolCallId: toolCall.toolCallId,+ output: response.result || response.error,+ });+ }} catch (error) {Logger.error("Tool call failed:", error);addToolResult({diff --git a/frontend/src/core/ai/tools/__tests__/registry.test.ts b/frontend/src/core/ai/tools/__tests__/registry.test.tsnew file mode 100644index 0000000000..8604724235--- /dev/null+++ b/frontend/src/core/ai/tools/__tests__/registry.test.ts@@ -0,0 +1,53 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { describe, expect, it } from "vitest";+import { FrontendToolRegistry } from "../registry";+import { TestFrontendTool } from "../sample-tool";++describe("FrontendToolRegistry", () => {+ it("registers tools via constructor and supports has()", () => {+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);+ expect(registry.has("test_frontend_tool")).toBe(true);+ expect(registry.has("nonexistent_tool" as string)).toBe(false);+ });++ it("invokes a tool with valid args and validates input/output", async () => {+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);+ const ok = await registry.invoke("test_frontend_tool", { name: "Alice" });+ expect(ok).toEqual({ message: "Hello: Alice" });+ });++ it("returns a structured error on invalid args", async () => {+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);+ const err = (await registry.invoke("test_frontend_tool", {})) as Record<+ string,+ unknown+ >;+ expect(err.status).toBe("error");+ expect(err.code).toBe("TOOL_ERROR");+ });++ it("returns tool schemas with expected shape and memoizes the result", () => {+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);++ const schemas1 = registry.getToolSchemas();+ expect(Array.isArray(schemas1)).toBe(true);+ expect(schemas1.length).toBe(1);++ const def = schemas1[0];+ expect(def.name).toBe("test_frontend_tool");+ expect(def.source).toBe("frontend");+ expect(def.mode).toEqual(["ask"]);+ expect(typeof def.description).toBe("string");++ const params = def.parameters as Record<string, unknown>;+ expect(params && typeof params === "object").toBe(true);+ const properties = (params as { properties?: Record<string, unknown> })+ .properties;+ expect(properties && typeof properties === "object").toBe(true);+ expect("name" in (properties ?? {})).toBe(true);++ const schemas2 = registry.getToolSchemas();+ expect(schemas2).toBe(schemas1);+ });+});diff --git a/frontend/src/core/ai/tools/base.ts b/frontend/src/core/ai/tools/base.tsnew file mode 100644index 0000000000..0e2c4c366f--- /dev/null+++ b/frontend/src/core/ai/tools/base.ts@@ -0,0 +1,19 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { z } from "zod";+import type { CopilotMode } from "./registry";++/**+ * Contract for a frontend tool.+ *+ * Implementations can be plain objects or classes. The registry consumes this+ * interface without caring about the underlying implementation.+ */+export interface AiTool<TIn, TOut> {+ name: string;+ description: string;+ schema: z.ZodType<TIn>;+ outputSchema: z.ZodType<TOut>;+ mode: CopilotMode[];+ handler: (args: TIn) => TOut | Promise<TOut>;+}diff --git a/frontend/src/core/ai/tools/registry.ts b/frontend/src/core/ai/tools/registry.tsnew file mode 100644index 0000000000..38dee1137d--- /dev/null+++ b/frontend/src/core/ai/tools/registry.ts@@ -0,0 +1,102 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { components } from "@marimo-team/marimo-api";+import { Memoize } from "typescript-memoize";+import { type ZodObject, z } from "zod";+import type { AiTool } from "./base";+import { TestFrontendTool } from "./sample-tool";++export type AnyZodObject = ZodObject<z.ZodRawShape>;++// eslint-disable-next-line @typescript-eslint/no-explicit-any+type StoredTool = AiTool<any, any>;++/** should be the same as marimo/_config/config.py > CopilotMode */++type ToolDefinition = components["schemas"]["ToolDefinition"];+export type CopilotMode = ToolDefinition["mode"][number];++export interface FrontendToolDefinition extends ToolDefinition {+ source: "frontend";+}++export class FrontendToolRegistry {+ /** All registered tools */+ private tools = new Map<string, StoredTool>();++ constructor(tools: StoredTool[] = []) {+ this.tools = new Map(tools.map((tool) => [tool.name, tool]));+ }++ has(toolName: string) {+ return this.tools.has(toolName);+ }++ private getToolOrThrow(toolName: string): StoredTool {+ const tool = this.tools.get(toolName);+ if (!tool) {+ throw new Error(`Tool ${toolName} not found`);+ }+ return tool;+ }++ async invoke<TName extends string>(+ toolName: TName,+ rawArgs: unknown,+ ): Promise<unknown> {+ const tool = this.getToolOrThrow(toolName);+ const handler = tool.handler;+ const inputSchema = tool.schema;+ const outputSchema = tool.outputSchema;++ try {+ // Parse input args+ const inputResponse = await inputSchema.safeParseAsync(rawArgs);+ if (inputResponse.error) {+ const strError = z.prettifyError(inputResponse.error);+ throw new Error(`Tool ${toolName} returned invalid input: ${strError}`);+ }+ const args = inputResponse.data;++ // Call the handler+ const rawOutput = await handler(args);++ // Parse output+ const response = await outputSchema.safeParseAsync(rawOutput);+ if (response.error) {+ const strError = z.prettifyError(response.error);+ throw new Error(+ `Tool ${toolName} returned invalid output: ${strError}`,+ );+ }+ const output = response.data;+ return output;+ } catch (error) {+ return {+ status: "error",+ code: "TOOL_ERROR",+ message: error instanceof Error ? error.message : String(error),+ suggestedFix: "Try again with valid arguments.",+ meta: {+ args: rawArgs,+ },+ };+ }+ }++ @Memoize()+ getToolSchemas(): FrontendToolDefinition[] {+ return [...this.tools.values()].map((tool) => ({+ name: tool.name,+ description: tool.description,+ parameters: z.toJSONSchema(tool.schema),+ source: "frontend",+ mode: tool.mode,+ }));+ }+}++export const FRONTEND_TOOL_REGISTRY = new FrontendToolRegistry([+ ...(import.meta.env.DEV ? [new TestFrontendTool()] : []),+ // ADD MORE TOOLS HERE+]);diff --git a/frontend/src/core/ai/tools/sample-tool.ts b/frontend/src/core/ai/tools/sample-tool.tsnew file mode 100644index 0000000000..454ff3f7d2--- /dev/null+++ b/frontend/src/core/ai/tools/sample-tool.ts@@ -0,0 +1,36 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { z } from "zod";+import type { AiTool } from "./base";+import type { CopilotMode } from "./registry";++const description = `+Test frontend tool that returns a greeting message.++Args:+- name (string): The name to include in the greeting.++Returns:+- { message: string } — The greeting message, e.g., "Hello: Alice".+`;++interface Input {+ name: string;+}++interface Output {+ message: string;+}++/** A sample frontend tool that returns "hello world" */+export class TestFrontendTool implements AiTool<Input, Output> {+ readonly name = "test_frontend_tool";+ readonly description = description;+ readonly schema = z.object({ name: z.string() });+ readonly outputSchema = z.object({ message: z.string() });+ readonly mode: CopilotMode[] = ["ask"];++ async handler({ name }: Input) {+ return { message: `Hello: ${name}` };+ }+}diff --git a/frontend/tsconfig.json b/frontend/tsconfig.jsonindex 92d6f4e1db..5c9da8a177 100644--- a/frontend/tsconfig.json+++ b/frontend/tsconfig.json@@ -23,7 +23,9 @@/* Style */"allowJs": true /* needed for storybook */,"paths": {- "@/*": ["./src/*"]+ "@/*": ["./src/*"],+ /* Pin zod to root (v4); v3 exists via @modelcontextprotocol/sdk */+ "zod": ["./node_modules/zod"]},"forceConsistentCasingInFileNames": true,"useDefineForClassFields": true,diff --git a/marimo/_cli/development/commands.py b/marimo/_cli/development/commands.pyindex f1ba0891e2..a3a71781a9 100644--- a/marimo/_cli/development/commands.py+++ b/marimo/_cli/development/commands.py@@ -49,6 +49,7 @@ def _generate_server_api_schema() -> dict[str, Any]:from marimo._messaging.mimetypes import KnownMimeTypefrom marimo._plugins.core.web_component import JSONTypefrom marimo._runtime.packages.package_manager import PackageDescription+ from marimo._server.ai.tools.types import ToolDefinitionfrom marimo._server.api.router import build_routesfrom marimo._utils.dataclass_to_openapi import (PythonTypeToOpenAPI,@@ -136,6 +137,7 @@ def _generate_server_api_schema() -> dict[str, Any]:ops.MessageOperation,# aiChatMessage,+ ToolDefinition,]# dataclass components used in requests/responsesdiff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.pyindex 934b1820b3..327471c195 100644--- a/marimo/_server/ai/providers.py+++ b/marimo/_server/ai/providers.py@@ -35,6 +35,7 @@ from marimo._ai._types import ChatMessagefrom marimo._dependencies.dependencies import DependencyManagerfrom marimo._server.ai.config import AnyProviderConfigfrom marimo._server.ai.ids import AiModelId+from marimo._server.ai.tools.types import ToolDefinitionfrom marimo._server.api.status import HTTPStatusTIMEOUT = 30@@ -120,6 +121,7 @@ class CompletionProvider(Generic[ResponseT, StreamT], ABC):messages: list[ChatMessage],system_prompt: str,max_tokens: int,+ additional_tools: list[ToolDefinition],) -> StreamT:"""Create a completion stream."""pass@@ -477,6 +479,7 @@ class OpenAIProvider(messages: list[ChatMessage],system_prompt: str,max_tokens: int,+ additional_tools: list[ToolDefinition],) -> OpenAiStream[ChatCompletionChunk]:client = self.get_client(self.config)tools = self.config.tools@@ -495,7 +498,8 @@ class OpenAIProvider("timeout": TIMEOUT,}if tools:- create_params["tools"] = convert_to_openai_tools(tools)+ all_tools = tools + additional_tools+ create_params["tools"] = convert_to_openai_tools(all_tools)if self._is_reasoning_model(self.model):create_params["reasoning_effort"] = self.DEFAULT_REASONING_EFFORTcreate_params["max_completion_tokens"] = max_tokens@@ -693,6 +697,7 @@ class AnthropicProvider(messages: list[ChatMessage],system_prompt: str,max_tokens: int,+ additional_tools: list[ToolDefinition],) -> AnthropicStream[RawMessageStreamEvent]:client = self.get_client(self.config)tools = self.config.tools@@ -708,7 +713,8 @@ class AnthropicProvider("temperature": self.get_temperature(),}if tools:- create_params["tools"] = convert_to_anthropic_tools(tools)+ all_tools = tools + additional_tools+ create_params["tools"] = convert_to_anthropic_tools(all_tools)if self.is_extended_thinking_model(self.model):create_params["thinking"] = {"type": "enabled",@@ -797,7 +803,10 @@ class GoogleProvider()def get_config(- self, system_prompt: str, max_tokens: int+ self,+ system_prompt: str,+ max_tokens: int,+ additional_tools: list[ToolDefinition],) -> GenerateContentConfig:tools = self.config.toolsconfig = {@@ -806,7 +815,8 @@ class GoogleProvider("max_output_tokens": max_tokens,}if tools:- config["tools"] = convert_to_google_tools(tools)+ all_tools = tools + additional_tools+ config["tools"] = convert_to_google_tools(all_tools)if self.is_thinking_model(self.model):config["thinking_config"] = {"include_thoughts": True,@@ -846,13 +856,16 @@ class GoogleProvider(messages: list[ChatMessage],system_prompt: str,max_tokens: int,+ additional_tools: list[ToolDefinition],) -> AsyncIterator[GenerateContentResponse]:client = self.get_client(self.config)return await client.models.generate_content_stream(model=self.model,contents=convert_to_google_messages(messages),config=self.get_config(- system_prompt=system_prompt, max_tokens=max_tokens+ system_prompt=system_prompt,+ max_tokens=max_tokens,+ additional_tools=additional_tools,),)@@ -952,6 +965,7 @@ class BedrockProvider(messages: list[ChatMessage],system_prompt: str,max_tokens: int,+ additional_tools: list[ToolDefinition],) -> LitellmStream:DependencyManager.litellm.require(why="for AI assistance with Bedrock")DependencyManager.boto3.require(why="for AI assistance with Bedrock")@@ -974,7 +988,8 @@ class BedrockProvider("timeout": TIMEOUT,}if tools:- config["tools"] = convert_to_openai_tools(tools)+ all_tools = tools + additional_tools+ config["tools"] = convert_to_openai_tools(all_tools)return await litellm_completion(**config)diff --git a/marimo/_server/ai/tools/tool_manager.py b/marimo/_server/ai/tools/tool_manager.pyindex d9e0b813e0..2843d08d79 100644--- a/marimo/_server/ai/tools/tool_manager.py+++ b/marimo/_server/ai/tools/tool_manager.py@@ -70,21 +70,13 @@ class ToolManager:LOGGER.debug(f"Registered backend tool: {name}")- def _register_frontend_tool(self, tool: ToolDefinition) -> None:- """Register a frontend tool (definition only - no handler or validator needed)."""- if tool.source != "frontend":- raise ValueError("Tool source must be 'frontend'")-- self._tools[tool.name] = tool- LOGGER.debug(f"Registered frontend tool: {tool.name}")-def _get_all_tools(self) -> list[ToolDefinition]:- """Get all available frontend, backend, and MCP tools."""+ """Get all available backend, and MCP tools."""self._init_backend_tools()- local_tools = list(self._tools.values())+ backend_tools = list(self._tools.values())mcp_tools = self._list_mcp_tools()- all_tools = local_tools + mcp_tools+ all_tools = backend_tools + mcp_toolsreturn all_toolsdef get_tools_for_mode(self, mode: CopilotMode) -> list[ToolDefinition]:@@ -111,10 +103,11 @@ class ToolManager:if tool and tool.source == "backend":return toolelif source == "frontend":- # Check if it's a frontend tool- tool = self._tools.get(name)- if tool and tool.source == "frontend":- return tool+ # ToolManager does not handle frontend tools+ LOGGER.warning(+ f"Tool {name} is a frontend tool and should not be accessed in the backend."+ )+ return Noneelif source == "mcp":# Check MCP toolsmcp_tools = self._list_mcp_tools()@@ -295,7 +288,7 @@ class ToolManager:else:# Unknown tool source- system_error = f"Unknown tool source: {tool.source} for tool {tool_name}. Supported sources are: backend, frontend, mcp."+ system_error = f"Unknown tool source: {tool.source} for tool {tool_name}. Supported sources are: backend and mcp."LOGGER.error(system_error)return ToolCallResult(tool_name=tool_name,@@ -352,9 +345,7 @@ def setup_tool_manager(app: Starlette) -> None:global _TOOL_MANAGERif _TOOL_MANAGER is None:_TOOL_MANAGER = ToolManager(app)- LOGGER.info(- f"ToolManager initialized with {len(_TOOL_MANAGER._tools)} backend/frontend tools"- )+ LOGGER.info("ToolManager initialized")def get_tool_manager() -> ToolManager:diff --git a/marimo/_server/api/endpoints/ai.py b/marimo/_server/api/endpoints/ai.pyindex e8dc56f2f1..298a7a1e6e 100644--- a/marimo/_server/api/endpoints/ai.py+++ b/marimo/_server/api/endpoints/ai.py@@ -164,6 +164,7 @@ async def ai_completion(messages=messages,system_prompt=system_prompt,max_tokens=get_max_tokens(config),+ additional_tools=[],)# Pass back the entire SDK message if the frontend can handle it@@ -234,10 +235,12 @@ async def ai_chat(AnyProviderConfig.for_model(model, ai_config),model=model,)+ additional_tools = body.tools or []response = await provider.stream_completion(messages=messages,system_prompt=system_prompt,max_tokens=max_tokens,+ additional_tools=additional_tools,)return StreamingResponse(@@ -303,6 +306,7 @@ async def ai_inline_completion(messages=messages,system_prompt=system_prompt,max_tokens=INLINE_COMPLETION_MAX_TOKENS,+ additional_tools=[],)content = await provider.collect_stream(response)diff --git a/marimo/_server/models/completion.py b/marimo/_server/models/completion.pyindex 1a945172dc..eaba085faf 100644--- a/marimo/_server/models/completion.py+++ b/marimo/_server/models/completion.py@@ -6,6 +6,7 @@ from typing import Any, Literal, Optional, Unionimport msgspecfrom marimo._ai._types import ChatMessage+from marimo._server.ai.tools.types import ToolDefinitionclass SchemaColumn(msgspec.Struct, rename="camel"):@@ -56,5 +57,6 @@ class ChatRequest(msgspec.Struct, rename="camel"):context: AiCompletionContextinclude_other_code: strmessages: list[ChatMessage]+ tools: Optional[list[ToolDefinition]] = Nonemodel: Optional[str] = Nonevariables: Optional[list[Union[VariableContext, str]]] = Nonediff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex ec7cc076d3..f09c60427e 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -486,6 +486,13 @@ components:- type: string- type: 'null'default: null+ tools:+ anyOf:+ - items:+ $ref: '#/components/schemas/ToolDefinition'+ type: array+ - type: 'null'+ default: nullvariables:anyOf:- items:@@ -3262,6 +3269,34 @@ components:required: []title: SuccessResponsetype: object+ ToolDefinition:+ description: Tool definition compatible with ai-sdk-ui format.+ properties:+ description:+ type: string+ mode:+ items:+ enum:+ - ask+ - manual+ type: array+ name:+ type: string+ parameters:+ type: object+ source:+ enum:+ - backend+ - frontend+ - mcp+ required:+ - name+ - description+ - parameters+ - source+ - mode+ title: ToolDefinition+ type: objectTyLanguageServerConfig:description: 'Configuration options for Ty Language Server.diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex e229e3853a..40bb0f6888 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -2938,6 +2938,8 @@ export interface components {/** @default null */model?: string | null;/** @default null */+ tools?: components["schemas"]["ToolDefinition"][] | null;+ /** @default null */variables?: (string | components["schemas"]["VariableContext"])[] | null;};/** CodeCompletionRequest */@@ -4391,6 +4393,18 @@ export interface components {/** @default true */success?: boolean;};+ /**+ * ToolDefinition+ * @description Tool definition compatible with ai-sdk-ui format.+ */+ ToolDefinition: {+ description: string;+ mode: ("ask" | "manual")[];+ name: string;+ parameters: Record<string, any>;+ /** @enum {unknown} */+ source: "backend" | "frontend" | "mcp";+ };/*** TyLanguageServerConfig* @description Configuration options for Ty Language Server.diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.pyindex 1154a4ebf9..4a593375fb 100644--- a/tests/_server/ai/test_providers.py+++ b/tests/_server/ai/test_providers.py@@ -127,7 +127,7 @@ async def test_openai_provider_max_tokens_parameter(# Call stream_completionmessages = [ChatMessage(role="user", content="test message")]- await provider.stream_completion(messages, "system prompt", 1000)+ await provider.stream_completion(messages, "system prompt", 1000, [])# Verify the correct parameters were passedmock_client.chat.completions.create.assert_called_once()