Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

feat: Create frontend tools framework and integrate to backend ai system (#6609)

Joaquin Coromina47bf5702b274
diff --git a/frontend/src/components/chat/chat-panel.tsx b/frontend/src/components/chat/chat-panel.tsx
index 3c380139be..ecd1f4bf97 100644
--- a/frontend/src/components/chat/chat-panel.tsx
+++ b/frontend/src/components/chat/chat-panel.tsx
@@ -44,6 +44,7 @@ import {
type ChatId,
chatStateAtom,
} from "@/core/ai/state";
+import { FRONTEND_TOOL_REGISTRY } from "@/core/ai/tools/registry";
import { aiAtom, aiEnabledAtom } from "@/core/config/config";
import { DEFAULT_AI_MODEL } from "@/core/config/config-schema";
import { FeatureFlagged } from "@/core/config/feature-flag";
@@ -585,6 +586,7 @@ const ChatPanelBody = () => {
return {
body: {
+ tools: FRONTEND_TOOL_REGISTRY.getToolSchemas(),
...options,
...completionBody,
},
diff --git a/frontend/src/components/chat/chat-utils.ts b/frontend/src/components/chat/chat-utils.ts
index 2f343b324d..dc4646a466 100644
--- a/frontend/src/components/chat/chat-utils.ts
+++ b/frontend/src/components/chat/chat-utils.ts
@@ -2,6 +2,7 @@
import type { components } from "@marimo-team/marimo-api";
import type { FileUIPart, ToolUIPart, UIMessage } from "ai";
+import { FRONTEND_TOOL_REGISTRY } from "@/core/ai/tools/registry";
import type {
InvokeAiToolRequest,
InvokeAiToolResponse,
@@ -117,15 +118,29 @@ export async function handleToolCall({
};
}) {
try {
- const response = await invokeAiTool({
- toolName: toolCall.toolName,
- arguments: toolCall.input,
- });
- addToolResult({
- tool: toolCall.toolName,
- toolCallId: toolCall.toolCallId,
- output: response.result || response.error,
- });
+ if (FRONTEND_TOOL_REGISTRY.has(toolCall.toolName)) {
+ // Invoke the frontend tool
+ const response = await FRONTEND_TOOL_REGISTRY.invoke(
+ toolCall.toolName,
+ toolCall.input,
+ );
+ addToolResult({
+ tool: toolCall.toolName,
+ toolCallId: toolCall.toolCallId,
+ output: response,
+ });
+ } else {
+ // Invoke the backend/mcp tool
+ const response = await invokeAiTool({
+ toolName: toolCall.toolName,
+ arguments: toolCall.input,
+ });
+ addToolResult({
+ tool: toolCall.toolName,
+ toolCallId: toolCall.toolCallId,
+ output: response.result || response.error,
+ });
+ }
} catch (error) {
Logger.error("Tool call failed:", error);
addToolResult({
diff --git a/frontend/src/core/ai/tools/__tests__/registry.test.ts b/frontend/src/core/ai/tools/__tests__/registry.test.ts
new file mode 100644
index 0000000000..8604724235
--- /dev/null
+++ b/frontend/src/core/ai/tools/__tests__/registry.test.ts
@@ -0,0 +1,53 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { describe, expect, it } from "vitest";
+import { FrontendToolRegistry } from "../registry";
+import { TestFrontendTool } from "../sample-tool";
+
+describe("FrontendToolRegistry", () => {
+ it("registers tools via constructor and supports has()", () => {
+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);
+ expect(registry.has("test_frontend_tool")).toBe(true);
+ expect(registry.has("nonexistent_tool" as string)).toBe(false);
+ });
+
+ it("invokes a tool with valid args and validates input/output", async () => {
+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);
+ const ok = await registry.invoke("test_frontend_tool", { name: "Alice" });
+ expect(ok).toEqual({ message: "Hello: Alice" });
+ });
+
+ it("returns a structured error on invalid args", async () => {
+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);
+ const err = (await registry.invoke("test_frontend_tool", {})) as Record<
+ string,
+ unknown
+ >;
+ expect(err.status).toBe("error");
+ expect(err.code).toBe("TOOL_ERROR");
+ });
+
+ it("returns tool schemas with expected shape and memoizes the result", () => {
+ const registry = new FrontendToolRegistry([new TestFrontendTool()]);
+
+ const schemas1 = registry.getToolSchemas();
+ expect(Array.isArray(schemas1)).toBe(true);
+ expect(schemas1.length).toBe(1);
+
+ const def = schemas1[0];
+ expect(def.name).toBe("test_frontend_tool");
+ expect(def.source).toBe("frontend");
+ expect(def.mode).toEqual(["ask"]);
+ expect(typeof def.description).toBe("string");
+
+ const params = def.parameters as Record<string, unknown>;
+ expect(params && typeof params === "object").toBe(true);
+ const properties = (params as { properties?: Record<string, unknown> })
+ .properties;
+ expect(properties && typeof properties === "object").toBe(true);
+ expect("name" in (properties ?? {})).toBe(true);
+
+ const schemas2 = registry.getToolSchemas();
+ expect(schemas2).toBe(schemas1);
+ });
+});
diff --git a/frontend/src/core/ai/tools/base.ts b/frontend/src/core/ai/tools/base.ts
new file mode 100644
index 0000000000..0e2c4c366f
--- /dev/null
+++ b/frontend/src/core/ai/tools/base.ts
@@ -0,0 +1,19 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { z } from "zod";
+import type { CopilotMode } from "./registry";
+
+/**
+ * Contract for a frontend tool.
+ *
+ * Implementations can be plain objects or classes. The registry consumes this
+ * interface without caring about the underlying implementation.
+ */
+export interface AiTool<TIn, TOut> {
+ name: string;
+ description: string;
+ schema: z.ZodType<TIn>;
+ outputSchema: z.ZodType<TOut>;
+ mode: CopilotMode[];
+ handler: (args: TIn) => TOut | Promise<TOut>;
+}
diff --git a/frontend/src/core/ai/tools/registry.ts b/frontend/src/core/ai/tools/registry.ts
new file mode 100644
index 0000000000..38dee1137d
--- /dev/null
+++ b/frontend/src/core/ai/tools/registry.ts
@@ -0,0 +1,102 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { components } from "@marimo-team/marimo-api";
+import { Memoize } from "typescript-memoize";
+import { type ZodObject, z } from "zod";
+import type { AiTool } from "./base";
+import { TestFrontendTool } from "./sample-tool";
+
+export type AnyZodObject = ZodObject<z.ZodRawShape>;
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type StoredTool = AiTool<any, any>;
+
+/** should be the same as marimo/_config/config.py > CopilotMode */
+
+type ToolDefinition = components["schemas"]["ToolDefinition"];
+export type CopilotMode = ToolDefinition["mode"][number];
+
+export interface FrontendToolDefinition extends ToolDefinition {
+ source: "frontend";
+}
+
+export class FrontendToolRegistry {
+ /** All registered tools */
+ private tools = new Map<string, StoredTool>();
+
+ constructor(tools: StoredTool[] = []) {
+ this.tools = new Map(tools.map((tool) => [tool.name, tool]));
+ }
+
+ has(toolName: string) {
+ return this.tools.has(toolName);
+ }
+
+ private getToolOrThrow(toolName: string): StoredTool {
+ const tool = this.tools.get(toolName);
+ if (!tool) {
+ throw new Error(`Tool ${toolName} not found`);
+ }
+ return tool;
+ }
+
+ async invoke<TName extends string>(
+ toolName: TName,
+ rawArgs: unknown,
+ ): Promise<unknown> {
+ const tool = this.getToolOrThrow(toolName);
+ const handler = tool.handler;
+ const inputSchema = tool.schema;
+ const outputSchema = tool.outputSchema;
+
+ try {
+ // Parse input args
+ const inputResponse = await inputSchema.safeParseAsync(rawArgs);
+ if (inputResponse.error) {
+ const strError = z.prettifyError(inputResponse.error);
+ throw new Error(`Tool ${toolName} returned invalid input: ${strError}`);
+ }
+ const args = inputResponse.data;
+
+ // Call the handler
+ const rawOutput = await handler(args);
+
+ // Parse output
+ const response = await outputSchema.safeParseAsync(rawOutput);
+ if (response.error) {
+ const strError = z.prettifyError(response.error);
+ throw new Error(
+ `Tool ${toolName} returned invalid output: ${strError}`,
+ );
+ }
+ const output = response.data;
+ return output;
+ } catch (error) {
+ return {
+ status: "error",
+ code: "TOOL_ERROR",
+ message: error instanceof Error ? error.message : String(error),
+ suggestedFix: "Try again with valid arguments.",
+ meta: {
+ args: rawArgs,
+ },
+ };
+ }
+ }
+
+ @Memoize()
+ getToolSchemas(): FrontendToolDefinition[] {
+ return [...this.tools.values()].map((tool) => ({
+ name: tool.name,
+ description: tool.description,
+ parameters: z.toJSONSchema(tool.schema),
+ source: "frontend",
+ mode: tool.mode,
+ }));
+ }
+}
+
+export const FRONTEND_TOOL_REGISTRY = new FrontendToolRegistry([
+ ...(import.meta.env.DEV ? [new TestFrontendTool()] : []),
+ // ADD MORE TOOLS HERE
+]);
diff --git a/frontend/src/core/ai/tools/sample-tool.ts b/frontend/src/core/ai/tools/sample-tool.ts
new file mode 100644
index 0000000000..454ff3f7d2
--- /dev/null
+++ b/frontend/src/core/ai/tools/sample-tool.ts
@@ -0,0 +1,36 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { z } from "zod";
+import type { AiTool } from "./base";
+import type { CopilotMode } from "./registry";
+
+const description = `
+Test frontend tool that returns a greeting message.
+
+Args:
+- name (string): The name to include in the greeting.
+
+Returns:
+- { message: string } — The greeting message, e.g., "Hello: Alice".
+`;
+
+interface Input {
+ name: string;
+}
+
+interface Output {
+ message: string;
+}
+
+/** A sample frontend tool that returns "hello world" */
+export class TestFrontendTool implements AiTool<Input, Output> {
+ readonly name = "test_frontend_tool";
+ readonly description = description;
+ readonly schema = z.object({ name: z.string() });
+ readonly outputSchema = z.object({ message: z.string() });
+ readonly mode: CopilotMode[] = ["ask"];
+
+ async handler({ name }: Input) {
+ return { message: `Hello: ${name}` };
+ }
+}
diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json
index 92d6f4e1db..5c9da8a177 100644
--- a/frontend/tsconfig.json
+++ b/frontend/tsconfig.json
@@ -23,7 +23,9 @@
/* Style */
"allowJs": true /* needed for storybook */,
"paths": {
- "@/*": ["./src/*"]
+ "@/*": ["./src/*"],
+ /* Pin zod to root (v4); v3 exists via @modelcontextprotocol/sdk */
+ "zod": ["./node_modules/zod"]
},
"forceConsistentCasingInFileNames": true,
"useDefineForClassFields": true,
diff --git a/marimo/_cli/development/commands.py b/marimo/_cli/development/commands.py
index f1ba0891e2..a3a71781a9 100644
--- a/marimo/_cli/development/commands.py
+++ b/marimo/_cli/development/commands.py
@@ -49,6 +49,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
from marimo._messaging.mimetypes import KnownMimeType
from marimo._plugins.core.web_component import JSONType
from marimo._runtime.packages.package_manager import PackageDescription
+ from marimo._server.ai.tools.types import ToolDefinition
from marimo._server.api.router import build_routes
from marimo._utils.dataclass_to_openapi import (
PythonTypeToOpenAPI,
@@ -136,6 +137,7 @@ def _generate_server_api_schema() -> dict[str, Any]:
ops.MessageOperation,
# ai
ChatMessage,
+ ToolDefinition,
]
# dataclass components used in requests/responses
diff --git a/marimo/_server/ai/providers.py b/marimo/_server/ai/providers.py
index 934b1820b3..327471c195 100644
--- a/marimo/_server/ai/providers.py
+++ b/marimo/_server/ai/providers.py
@@ -35,6 +35,7 @@ from marimo._ai._types import ChatMessage
from marimo._dependencies.dependencies import DependencyManager
from marimo._server.ai.config import AnyProviderConfig
from marimo._server.ai.ids import AiModelId
+from marimo._server.ai.tools.types import ToolDefinition
from marimo._server.api.status import HTTPStatus
TIMEOUT = 30
@@ -120,6 +121,7 @@ class CompletionProvider(Generic[ResponseT, StreamT], ABC):
messages: list[ChatMessage],
system_prompt: str,
max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> StreamT:
"""Create a completion stream."""
pass
@@ -477,6 +479,7 @@ class OpenAIProvider(
messages: list[ChatMessage],
system_prompt: str,
max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> OpenAiStream[ChatCompletionChunk]:
client = self.get_client(self.config)
tools = self.config.tools
@@ -495,7 +498,8 @@ class OpenAIProvider(
"timeout": TIMEOUT,
}
if tools:
- create_params["tools"] = convert_to_openai_tools(tools)
+ all_tools = tools + additional_tools
+ create_params["tools"] = convert_to_openai_tools(all_tools)
if self._is_reasoning_model(self.model):
create_params["reasoning_effort"] = self.DEFAULT_REASONING_EFFORT
create_params["max_completion_tokens"] = max_tokens
@@ -693,6 +697,7 @@ class AnthropicProvider(
messages: list[ChatMessage],
system_prompt: str,
max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> AnthropicStream[RawMessageStreamEvent]:
client = self.get_client(self.config)
tools = self.config.tools
@@ -708,7 +713,8 @@ class AnthropicProvider(
"temperature": self.get_temperature(),
}
if tools:
- create_params["tools"] = convert_to_anthropic_tools(tools)
+ all_tools = tools + additional_tools
+ create_params["tools"] = convert_to_anthropic_tools(all_tools)
if self.is_extended_thinking_model(self.model):
create_params["thinking"] = {
"type": "enabled",
@@ -797,7 +803,10 @@ class GoogleProvider(
)
def get_config(
- self, system_prompt: str, max_tokens: int
+ self,
+ system_prompt: str,
+ max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> GenerateContentConfig:
tools = self.config.tools
config = {
@@ -806,7 +815,8 @@ class GoogleProvider(
"max_output_tokens": max_tokens,
}
if tools:
- config["tools"] = convert_to_google_tools(tools)
+ all_tools = tools + additional_tools
+ config["tools"] = convert_to_google_tools(all_tools)
if self.is_thinking_model(self.model):
config["thinking_config"] = {
"include_thoughts": True,
@@ -846,13 +856,16 @@ class GoogleProvider(
messages: list[ChatMessage],
system_prompt: str,
max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> AsyncIterator[GenerateContentResponse]:
client = self.get_client(self.config)
return await client.models.generate_content_stream(
model=self.model,
contents=convert_to_google_messages(messages),
config=self.get_config(
- system_prompt=system_prompt, max_tokens=max_tokens
+ system_prompt=system_prompt,
+ max_tokens=max_tokens,
+ additional_tools=additional_tools,
),
)
@@ -952,6 +965,7 @@ class BedrockProvider(
messages: list[ChatMessage],
system_prompt: str,
max_tokens: int,
+ additional_tools: list[ToolDefinition],
) -> LitellmStream:
DependencyManager.litellm.require(why="for AI assistance with Bedrock")
DependencyManager.boto3.require(why="for AI assistance with Bedrock")
@@ -974,7 +988,8 @@ class BedrockProvider(
"timeout": TIMEOUT,
}
if tools:
- config["tools"] = convert_to_openai_tools(tools)
+ all_tools = tools + additional_tools
+ config["tools"] = convert_to_openai_tools(all_tools)
return await litellm_completion(**config)
diff --git a/marimo/_server/ai/tools/tool_manager.py b/marimo/_server/ai/tools/tool_manager.py
index d9e0b813e0..2843d08d79 100644
--- a/marimo/_server/ai/tools/tool_manager.py
+++ b/marimo/_server/ai/tools/tool_manager.py
@@ -70,21 +70,13 @@ class ToolManager:
LOGGER.debug(f"Registered backend tool: {name}")
- def _register_frontend_tool(self, tool: ToolDefinition) -> None:
- """Register a frontend tool (definition only - no handler or validator needed)."""
- if tool.source != "frontend":
- raise ValueError("Tool source must be 'frontend'")
-
- self._tools[tool.name] = tool
- LOGGER.debug(f"Registered frontend tool: {tool.name}")
-
def _get_all_tools(self) -> list[ToolDefinition]:
- """Get all available frontend, backend, and MCP tools."""
+ """Get all available backend, and MCP tools."""
self._init_backend_tools()
- local_tools = list(self._tools.values())
+ backend_tools = list(self._tools.values())
mcp_tools = self._list_mcp_tools()
- all_tools = local_tools + mcp_tools
+ all_tools = backend_tools + mcp_tools
return all_tools
def get_tools_for_mode(self, mode: CopilotMode) -> list[ToolDefinition]:
@@ -111,10 +103,11 @@ class ToolManager:
if tool and tool.source == "backend":
return tool
elif source == "frontend":
- # Check if it's a frontend tool
- tool = self._tools.get(name)
- if tool and tool.source == "frontend":
- return tool
+ # ToolManager does not handle frontend tools
+ LOGGER.warning(
+ f"Tool {name} is a frontend tool and should not be accessed in the backend."
+ )
+ return None
elif source == "mcp":
# Check MCP tools
mcp_tools = self._list_mcp_tools()
@@ -295,7 +288,7 @@ class ToolManager:
else:
# Unknown tool source
- system_error = f"Unknown tool source: {tool.source} for tool {tool_name}. Supported sources are: backend, frontend, mcp."
+ system_error = f"Unknown tool source: {tool.source} for tool {tool_name}. Supported sources are: backend and mcp."
LOGGER.error(system_error)
return ToolCallResult(
tool_name=tool_name,
@@ -352,9 +345,7 @@ def setup_tool_manager(app: Starlette) -> None:
global _TOOL_MANAGER
if _TOOL_MANAGER is None:
_TOOL_MANAGER = ToolManager(app)
- LOGGER.info(
- f"ToolManager initialized with {len(_TOOL_MANAGER._tools)} backend/frontend tools"
- )
+ LOGGER.info("ToolManager initialized")
def get_tool_manager() -> ToolManager:
diff --git a/marimo/_server/api/endpoints/ai.py b/marimo/_server/api/endpoints/ai.py
index e8dc56f2f1..298a7a1e6e 100644
--- a/marimo/_server/api/endpoints/ai.py
+++ b/marimo/_server/api/endpoints/ai.py
@@ -164,6 +164,7 @@ async def ai_completion(
messages=messages,
system_prompt=system_prompt,
max_tokens=get_max_tokens(config),
+ additional_tools=[],
)
# Pass back the entire SDK message if the frontend can handle it
@@ -234,10 +235,12 @@ async def ai_chat(
AnyProviderConfig.for_model(model, ai_config),
model=model,
)
+ additional_tools = body.tools or []
response = await provider.stream_completion(
messages=messages,
system_prompt=system_prompt,
max_tokens=max_tokens,
+ additional_tools=additional_tools,
)
return StreamingResponse(
@@ -303,6 +306,7 @@ async def ai_inline_completion(
messages=messages,
system_prompt=system_prompt,
max_tokens=INLINE_COMPLETION_MAX_TOKENS,
+ additional_tools=[],
)
content = await provider.collect_stream(response)
diff --git a/marimo/_server/models/completion.py b/marimo/_server/models/completion.py
index 1a945172dc..eaba085faf 100644
--- a/marimo/_server/models/completion.py
+++ b/marimo/_server/models/completion.py
@@ -6,6 +6,7 @@ from typing import Any, Literal, Optional, Union
import msgspec
from marimo._ai._types import ChatMessage
+from marimo._server.ai.tools.types import ToolDefinition
class SchemaColumn(msgspec.Struct, rename="camel"):
@@ -56,5 +57,6 @@ class ChatRequest(msgspec.Struct, rename="camel"):
context: AiCompletionContext
include_other_code: str
messages: list[ChatMessage]
+ tools: Optional[list[ToolDefinition]] = None
model: Optional[str] = None
variables: Optional[list[Union[VariableContext, str]]] = None
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index ec7cc076d3..f09c60427e 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -486,6 +486,13 @@ components:
- type: string
- type: 'null'
default: null
+ tools:
+ anyOf:
+ - items:
+ $ref: '#/components/schemas/ToolDefinition'
+ type: array
+ - type: 'null'
+ default: null
variables:
anyOf:
- items:
@@ -3262,6 +3269,34 @@ components:
required: []
title: SuccessResponse
type: object
+ ToolDefinition:
+ description: Tool definition compatible with ai-sdk-ui format.
+ properties:
+ description:
+ type: string
+ mode:
+ items:
+ enum:
+ - ask
+ - manual
+ type: array
+ name:
+ type: string
+ parameters:
+ type: object
+ source:
+ enum:
+ - backend
+ - frontend
+ - mcp
+ required:
+ - name
+ - description
+ - parameters
+ - source
+ - mode
+ title: ToolDefinition
+ type: object
TyLanguageServerConfig:
description: 'Configuration options for Ty Language Server.
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index e229e3853a..40bb0f6888 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -2938,6 +2938,8 @@ export interface components {
/** @default null */
model?: string | null;
/** @default null */
+ tools?: components["schemas"]["ToolDefinition"][] | null;
+ /** @default null */
variables?: (string | components["schemas"]["VariableContext"])[] | null;
};
/** CodeCompletionRequest */
@@ -4391,6 +4393,18 @@ export interface components {
/** @default true */
success?: boolean;
};
+ /**
+ * ToolDefinition
+ * @description Tool definition compatible with ai-sdk-ui format.
+ */
+ ToolDefinition: {
+ description: string;
+ mode: ("ask" | "manual")[];
+ name: string;
+ parameters: Record<string, any>;
+ /** @enum {unknown} */
+ source: "backend" | "frontend" | "mcp";
+ };
/**
* TyLanguageServerConfig
* @description Configuration options for Ty Language Server.
diff --git a/tests/_server/ai/test_providers.py b/tests/_server/ai/test_providers.py
index 1154a4ebf9..4a593375fb 100644
--- a/tests/_server/ai/test_providers.py
+++ b/tests/_server/ai/test_providers.py
@@ -127,7 +127,7 @@ async def test_openai_provider_max_tokens_parameter(
# Call stream_completion
messages = [ChatMessage(role="user", content="test message")]
- await provider.stream_completion(messages, "system prompt", 1000)
+ await provider.stream_completion(messages, "system prompt", 1000, [])
# Verify the correct parameters were passed
mock_client.chat.completions.create.assert_called_once()