π OpenHands: AI-Driven Development
feat(frontend): add canvas_ui tool so the agent can drive the UI (#420)
Hiep Le5c33c10b18fb
diff --git a/__tests__/api/agent-server-adapter.test.ts b/__tests__/api/agent-server-adapter.test.tsindex 4af700b203..9ed1f9eacb 100644--- a/__tests__/api/agent-server-adapter.test.ts+++ b/__tests__/api/agent-server-adapter.test.ts@@ -107,6 +107,7 @@ describe("buildStartConversationRequest", () => {{ name: "terminal", params: {} },{ name: "file_editor", params: {} },{ name: "task_tracker", params: {} },+ { name: "canvas_ui", params: {} },{ name: "browser_tool_set", params: {} },]);expect(payload.agent.include_default_tools).toEqual([@@ -171,6 +172,7 @@ describe("buildStartConversationRequest", () => {{ name: "terminal", params: {} },{ name: "file_editor", params: {} },{ name: "task_tracker", params: {} },+ { name: "canvas_ui", params: {} },]);});@@ -263,7 +265,11 @@ describe("buildStartConversationRequest", () => {}) as Record<string, unknown>;expect(payload.hook_config).toEqual({ on_start: [] });+ // Canvas-UI tool is auto-injected; user-supplied entries are merged in+ // alongside it. The dedicated canvas_ui describe block below pins the+ // exact merge semantics.expect(payload.tool_module_qualnames).toEqual({+ canvas_ui: "canvas_ui_tool",demo_tool: "pkg.tools.demo",});expect(payload.agent_definitions).toEqual([@@ -318,6 +324,35 @@ describe("buildStartConversationRequest", () => {},});});++ describe("canvas_ui tool injection", () => {+ it("always registers canvas_ui_tool in tool_module_qualnames, even when no user settings supply qualnames", () => {+ const payload = buildStartConversationRequest({+ settings: DEFAULT_SETTINGS,+ }) as { tool_module_qualnames: Record<string, string> };++ expect(payload.tool_module_qualnames).toMatchObject({+ canvas_ui: "canvas_ui_tool",+ });+ });++ it("merges user-supplied tool_module_qualnames alongside canvas_ui_tool without dropping either side", () => {+ const payload = buildStartConversationRequest({+ settings: {+ ...DEFAULT_SETTINGS,+ conversation_settings: {+ ...DEFAULT_SETTINGS.conversation_settings,+ tool_module_qualnames: { my_tool: "my_package.my_tool" },+ },+ },+ }) as { tool_module_qualnames: Record<string, string> };++ expect(payload.tool_module_qualnames).toEqual({+ canvas_ui: "canvas_ui_tool",+ my_tool: "my_package.my_tool",+ });+ });+ });});describe("getDefaultConversationTitle", () => {diff --git a/__tests__/routes/files-tab.test.tsx b/__tests__/routes/files-tab.test.tsxindex 8529e9132e..5b84294cc8 100644--- a/__tests__/routes/files-tab.test.tsx+++ b/__tests__/routes/files-tab.test.tsx@@ -6,6 +6,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest";import { MemoryRouter } from "react-router";import FilesTab from "#/routes/files-tab";+import { useFilesTabStore } from "#/stores/files-tab-store";// Mocks must be declared before the SUT is imported.const useHasAttachedSourceMock = vi.fn();@@ -58,6 +59,14 @@ function renderTab() {describe("FilesTab", () => {beforeEach(() => {+ // `selectedPath` lives in a global Zustand store (useFilesTabStore) and+ // the auto-select effect re-fires when the store is reset between tests,+ // which can race with the Zustand mock's afterEach reset and leave the+ // store polluted with the previous test's path. Resetting here, after+ // the previous test's cleanup() has unmounted any FilesTab, defeats+ // that race so each test starts with a clean selection.+ useFilesTabStore.setState({ selectedPath: null });+useHasAttachedSourceMock.mockReset();useHasGitCommitsMock.mockReset();useWorkspaceFilesMock.mockReset();diff --git a/__tests__/scripts/dev-docker.test.ts b/__tests__/scripts/dev-docker.test.tsindex f4b5fb914f..04bffcdabe 100644--- a/__tests__/scripts/dev-docker.test.ts+++ b/__tests__/scripts/dev-docker.test.ts@@ -1,3 +1,10 @@+// @vitest-environment node+// These tests load `scripts/dev-docker.mjs`, which constructs file:// URLs+// relative to its own location via `new URL("../tools", import.meta.url)`.+// jsdom's URL constructor ignores file:// base URLs (it falls back to its+// document base, e.g. http://localhost:3000/), breaking that resolution;+// the Node environment has the standard WHATWG URL behavior that honors+// the file:// base.import { describe, expect, it } from "vitest";import {diff --git a/__tests__/scripts/dev-extra-backend.test.ts b/__tests__/scripts/dev-extra-backend.test.tsindex fde467706b..f7a0cfa488 100644--- a/__tests__/scripts/dev-extra-backend.test.ts+++ b/__tests__/scripts/dev-extra-backend.test.ts@@ -1,3 +1,10 @@+// @vitest-environment node+// These tests load `scripts/dev-safe.mjs` (via dev-extra-backend), which+// constructs file:// URLs relative to its own location via+// `new URL("../tools", import.meta.url)`. jsdom's URL constructor ignores+// file:// base URLs (it falls back to its document base, e.g.+// http://localhost:3000/), breaking that resolution; the Node environment+// has the standard WHATWG URL behavior that honors the file:// base.import { mkdtempSync, rmSync } from "node:fs";import { tmpdir } from "node:os";import { fileURLToPath } from "node:url";diff --git a/__tests__/scripts/dev-safe.test.ts b/__tests__/scripts/dev-safe.test.tsindex 929851ee27..2ac9101132 100644--- a/__tests__/scripts/dev-safe.test.ts+++ b/__tests__/scripts/dev-safe.test.ts@@ -1,3 +1,10 @@+// @vitest-environment node+// These tests load `scripts/dev-safe.mjs`, which constructs file:// URLs+// relative to its own location via `new URL("../tools", import.meta.url)`.+// jsdom's URL constructor ignores file:// base URLs (it falls back to its+// document base, e.g. http://localhost:3000/), breaking that resolution;+// the Node environment has the standard WHATWG URL behavior that honors+// the file:// base.import net from "node:net";import { spawn } from "node:child_process";import { once } from "node:events";diff --git a/__tests__/scripts/dev-with-automation.test.ts b/__tests__/scripts/dev-with-automation.test.tsindex 2f7eed62ed..727e012a6c 100644--- a/__tests__/scripts/dev-with-automation.test.ts+++ b/__tests__/scripts/dev-with-automation.test.ts@@ -1,3 +1,10 @@+// @vitest-environment node+// These tests load `scripts/dev-with-automation.mjs` and `scripts/dev-safe.mjs`,+// which construct file:// URLs relative to their own location via+// `new URL("../tools", import.meta.url)`. jsdom's URL constructor ignores+// file:// base URLs (it falls back to its document base, e.g.+// http://localhost:3000/), breaking that resolution; the Node environment+// has the standard WHATWG URL behavior that honors the file:// base.import net from "node:net";import { spawn } from "node:child_process";import { once } from "node:events";diff --git a/__tests__/services/canvas-ui.test.ts b/__tests__/services/canvas-ui.test.tsnew file mode 100644index 0000000000..a0d8283922--- /dev/null+++ b/__tests__/services/canvas-ui.test.ts@@ -0,0 +1,118 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++import { handleCanvasUIAction } from "#/services/canvas-ui";+import { useConversationStore } from "#/stores/conversation-store";+import { useFilesTabStore } from "#/stores/files-tab-store";+import type { CanvasUIAction } from "#/types/agent-server/core";+import { isCanvasUIActionEvent } from "#/types/agent-server/type-guards";++// Helper: build a CanvasUIAction without repeating the literal `kind`+// discriminator in every test case.+function action(overrides: Partial<CanvasUIAction>): CanvasUIAction {+ return { kind: "CanvasUIAction", ...overrides } as CanvasUIAction;+}++describe("handleCanvasUIAction", () => {+ beforeEach(() => {+ // Arrange (shared): collapsed right panel, no selected file. Lets us+ // observe both the tab/panel toggling and the path mutation that the+ // dispatcher performs.+ useConversationStore.setState({+ selectedTab: null,+ isRightPanelShown: false,+ hasRightPanelToggled: false,+ });+ useFilesTabStore.setState({ selectedPath: null });+ });++ it("navigate_to_file selects the files tab, reveals the panel, and sets selectedPath", () => {+ handleCanvasUIAction(+ action({ command: "navigate_to_file", path: "docs/intro.html" }),+ );++ const conv = useConversationStore.getState();+ expect(conv.selectedTab).toBe("files");+ expect(conv.isRightPanelShown).toBe(true);+ expect(useFilesTabStore.getState().selectedPath).toBe("docs/intro.html");+ });++ it("show_preview selects the files tab and the requested path", () => {+ handleCanvasUIAction(+ action({ command: "show_preview", path: "report.html" }),+ );++ expect(useConversationStore.getState().selectedTab).toBe("files");+ expect(useFilesTabStore.getState().selectedPath).toBe("report.html");+ });++ it("open_tab switches to a valid tab without touching selectedPath", () => {+ handleCanvasUIAction(action({ command: "open_tab", tab: "terminal" }));++ expect(useConversationStore.getState().selectedTab).toBe("terminal");+ expect(useFilesTabStore.getState().selectedPath).toBeNull();+ });++ it("open_tab ignores unknown tab values and surfaces a warning for debuggability", () => {+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});++ try {+ handleCanvasUIAction(action({ command: "open_tab", tab: "not_a_tab" }));++ expect(useConversationStore.getState().selectedTab).toBeNull();+ expect(useConversationStore.getState().isRightPanelShown).toBe(false);+ expect(warnSpy).toHaveBeenCalledWith(+ expect.stringContaining("not_a_tab"),+ );+ } finally {+ warnSpy.mockRestore();+ }+ });++ it("navigate_to_file leaves selectedPath alone when no path is supplied", () => {+ useFilesTabStore.setState({ selectedPath: "previous.txt" });++ handleCanvasUIAction(action({ command: "navigate_to_file", path: null }));++ expect(useFilesTabStore.getState().selectedPath).toBe("previous.txt");+ expect(useConversationStore.getState().selectedTab).toBe("files");+ });+});++describe("isCanvasUIActionEvent", () => {+ function makeActionEvent(overrides: Record<string, unknown> = {}) {+ return {+ id: "evt-1",+ timestamp: "2026-05-13T00:00:00Z",+ source: "agent",+ action: { kind: "CanvasUIAction" },+ tool_name: "canvas_ui",+ tool_call_id: "call-1",+ ...overrides,+ };+ }++ it("returns true for an ActionEvent whose tool_name is canvas_ui", () => {+ expect(isCanvasUIActionEvent(makeActionEvent() as never)).toBe(true);+ });++ it("returns false when tool_name belongs to a different tool", () => {+ expect(+ isCanvasUIActionEvent(+ makeActionEvent({ tool_name: "execute_bash" }) as never,+ ),+ ).toBe(false);+ });++ it("returns false for a non-action event (no action field)", () => {+ const observationEvent = {+ id: "evt-2",+ timestamp: "2026-05-13T00:00:00Z",+ source: "environment",+ observation: { kind: "ExecuteBashObservation" },+ tool_name: "canvas_ui",+ tool_call_id: "call-1",+ };++ expect(isCanvasUIActionEvent(observationEvent as never)).toBe(false);+ });+});diff --git a/scripts/dev-docker.mjs b/scripts/dev-docker.mjsindex ff6e5224aa..d2a5a14969 100644--- a/scripts/dev-docker.mjs+++ b/scripts/dev-docker.mjs@@ -71,6 +71,15 @@ import { buildFrontend } from "./static-build.mjs";// Path inside the container where OH_AGENT_SERVER_LOCAL_PATH is bind-mounted.const CONTAINER_LOCAL_SDK_DIR = "/agent-server-src";+// Host directory containing Agent-Canvas-specific Python tools (e.g. the+// canvas_ui tool). Mounted read-only into the container and added to+// sys.path via OH_EXTRA_PYTHON_PATH so the agent-server can import the+// modules listed in `tool_module_qualnames`.+const CONTAINER_CANVAS_TOOLS_DIR = "/canvas-tools";+const HOST_CANVAS_TOOLS_DIR = fileURLToPath(+ new URL("../tools", import.meta.url),+);+// Docker image for the agent-server.const AGENT_SERVER_REPO = "ghcr.io/openhands/agent-server";// Default tag used when OH_AGENT_SERVER_GIT_REF is not set.@@ -270,6 +279,14 @@ function startAgentServerDocker(config) {const dockerArgs = ["run", "--rm", "--name", CONTAINER_NAME, "--init"];dockerArgs.push(...getDockerUserArgs(userSpec));dockerArgs.push("-v", `${process.env.PROJECT_PATH}:/projects`);+ // Read-only mount of the Agent-Canvas tools directory. Coupled with+ // OH_EXTRA_PYTHON_PATH below so the agent-server can import+ // canvas_ui_tool when the conversation request lists it under+ // tool_module_qualnames.+ dockerArgs.push(+ "-v",+ `${HOST_CANVAS_TOOLS_DIR}:${CONTAINER_CANVAS_TOOLS_DIR}:ro`,+ );// Bind-mount the local software-agent-sdk checkout if requested. Mounted// rw so editable installs can write their .dist-info into each package@@ -318,6 +335,9 @@ function startAgentServerDocker(config) {// Required so the secret-seeding PUT /api/settings/secrets call from// the host can authenticate against the agent-server in the container.OH_SESSION_API_KEYS_0: config.sessionApiKey,+ // Make the mounted canvas-tools directory importable so the agent-server+ // can resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool).+ OH_EXTRA_PYTHON_PATH: CONTAINER_CANVAS_TOOLS_DIR,};for (const [k, v] of Object.entries(containerEnv)) {dockerArgs.push("-e", `${k}=${v}`);@@ -381,12 +401,14 @@ if (isMainModule) {export {AGENT_SERVER_REPO,+ CONTAINER_CANVAS_TOOLS_DIR,CONTAINER_HOME_DIR,CONTAINER_LOCAL_SDK_DIR,CONTAINER_NAME,CONTAINER_OPENHANDS_DIR,CONTAINER_WORKSPACES_DIR,DEFAULT_AGENT_SERVER_TAG,+ HOST_CANVAS_TOOLS_DIR,checkDockerPrereqs,getDockerHomeTmpfsArgs,getDockerUserArgs,diff --git a/scripts/dev-safe.mjs b/scripts/dev-safe.mjsindex b456648af3..3e423b7c82 100644--- a/scripts/dev-safe.mjs+++ b/scripts/dev-safe.mjs@@ -14,7 +14,7 @@ import { homedir, tmpdir } from "node:os";import path from "node:path";import process from "node:process";import { setTimeout as delay } from "node:timers/promises";-import { pathToFileURL } from "node:url";+import { fileURLToPath, pathToFileURL } from "node:url";import {getProcessTreeSpawnOptions,@@ -523,6 +523,7 @@ export async function buildSafeDevConfigAsync(* @property {string} workingDir* @property {string} secretKey* @property {string} sessionApiKey+ * @property {string} canvasToolsDir*//**@@ -562,6 +563,12 @@ function buildConfigFromPorts(ports, cwd, env) {env.VITE_SESSION_API_KEY ||getOrCreatePersistedSessionApiKey(persistedKeyPath);+ // Host directory containing Agent-Canvas-specific Python tools (e.g. the+ // canvas_ui tool). Added to OH_EXTRA_PYTHON_PATH below so the agent-server+ // can import the modules listed in `tool_module_qualnames`. Lives at+ // <repo-root>/tools relative to this script.+ const canvasToolsDir = fileURLToPath(new URL("../tools", import.meta.url));+return {cwd,backendPort,@@ -576,6 +583,7 @@ function buildConfigFromPorts(ports, cwd, env) {workingDir: env.VITE_WORKING_DIR || workspacesPath,secretKey,sessionApiKey,+ canvasToolsDir,};}@@ -597,6 +605,9 @@ export function buildAgentServerEnv(config) {OH_SECRET_KEY: config.secretKey,// Use OH_SESSION_API_KEYS_0 for agent-server V1 config formatOH_SESSION_API_KEYS_0: config.sessionApiKey,+ // Make the host tools/ directory importable so the agent-server can+ // resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool).+ OH_EXTRA_PYTHON_PATH: config.canvasToolsDir,};}diff --git a/src/api/agent-server-adapter.ts b/src/api/agent-server-adapter.tsindex b768b8cc16..dfe59df089 100644--- a/src/api/agent-server-adapter.ts+++ b/src/api/agent-server-adapter.ts@@ -42,7 +42,18 @@ export interface DirectConversationInfo {} | null;}-const DEFAULT_TOOL_NAMES = ["terminal", "file_editor", "task_tracker"];+// Module qualname for the Canvas-UI tool. The agent-server imports this via+// tool_module_qualnames; the host directory is exposed via OH_EXTRA_PYTHON_PATH+// (see scripts/dev-docker.mjs and scripts/dev-safe.mjs).+const CANVAS_UI_TOOL_NAME = "canvas_ui";+const CANVAS_UI_TOOL_MODULE = "canvas_ui_tool";++const DEFAULT_TOOL_NAMES = [+ "terminal",+ "file_editor",+ "task_tracker",+ CANVAS_UI_TOOL_NAME,+];const BROWSER_TOOL_SET_NAME = "browser_tool_set";const DEFAULT_BUILT_IN_TOOL_NAMES = ["FinishTool", "ThinkTool"];const SWITCH_LLM_TOOL_NAME = "SwitchLLMTool";@@ -611,9 +622,16 @@ export function buildStartConversationRequest(payload.hook_config = conversationSettings.hook_config;}- if (conversationSettings.tool_module_qualnames) {- payload.tool_module_qualnames = conversationSettings.tool_module_qualnames;- }+ // Always include the canvas_ui tool module so the agent-server imports it+ // and registers the tool. User-supplied entries from conversationSettings+ // take precedence on key conflict (the canvas_ui key is ours and shouldn't+ // collide in practice).+ payload.tool_module_qualnames = {+ [CANVAS_UI_TOOL_NAME]: CANVAS_UI_TOOL_MODULE,+ ...((conversationSettings.tool_module_qualnames as+ | Record<string, string>+ | undefined) ?? {}),+ };if (conversationSettings.agent_definitions) {payload.agent_definitions = conversationSettings.agent_definitions;diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsxindex 0bac2745ec..dd28175999 100644--- a/src/contexts/conversation-websocket-context.tsx+++ b/src/contexts/conversation-websocket-context.tsx@@ -33,7 +33,9 @@ import {isPlanningFileEditorObservationEvent,isBrowserObservationEvent,isBrowserNavigateActionEvent,+ isCanvasUIActionEvent,} from "#/types/agent-server/type-guards";+import { handleCanvasUIAction } from "#/services/canvas-ui";import { ConversationStateUpdateEventStats } from "#/types/agent-server/core/events/conversation-state-event";import type {ConversationErrorEvent,@@ -494,6 +496,14 @@ export function ConversationWebSocketProvider({if (isBrowserNavigateActionEvent(event)) {useBrowserStore.getState().setUrl(event.action.url);}++ // Handle canvas_ui custom-tool ActionEvents - drive the frontend+ // (navigate to a file, switch tabs, show a preview). The tool+ // executes server-side as a no-op; the actual UI change happens+ // here on the client.+ if (isCanvasUIActionEvent(event)) {+ handleCanvasUIAction(event.action);+ }}} catch (error) {console.warn("Failed to parse WebSocket message as JSON:", error);diff --git a/src/routes/files-tab.tsx b/src/routes/files-tab.tsxindex b6c36aaad3..f19f698ddc 100644--- a/src/routes/files-tab.tsx+++ b/src/routes/files-tab.tsx@@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next";import { useQueryClient } from "@tanstack/react-query";import { I18nKey } from "#/i18n/declaration";+import { useFilesTabStore } from "#/stores/files-tab-store";import { useWorkspaceFiles } from "#/hooks/query/use-workspace-files";import { useWorkspaceFileContent } from "#/hooks/query/use-workspace-file-content";import { useHasAttachedSource } from "#/hooks/use-has-attached-source";@@ -64,7 +65,8 @@ function FilesTab() {const filesQuery = useWorkspaceFiles();const paths = useMemo(() => filesQuery.data ?? [], [filesQuery.data]);- const [selectedPath, setSelectedPath] = useState<string | null>(null);+ const selectedPath = useFilesTabStore((s) => s.selectedPath);+ const setSelectedPath = useFilesTabStore((s) => s.setSelectedPath);// Pre-fetch the selected file's content here too so the toolbar's// "open in new window" link can reach for its `staticUrl`. react-querydiff --git a/src/services/canvas-ui.ts b/src/services/canvas-ui.tsnew file mode 100644index 0000000000..cf95f54fb2--- /dev/null+++ b/src/services/canvas-ui.ts@@ -0,0 +1,55 @@+import {+ ConversationTab,+ useConversationStore,+} from "#/stores/conversation-store";+import { useFilesTabStore } from "#/stores/files-tab-store";+import type { CanvasUIAction } from "#/types/agent-server/core";++const VALID_TABS: ReadonlySet<ConversationTab> = new Set<ConversationTab>([+ "files",+ "browser",+ "vscode",+ "terminal",+ "planner",+ "tasklist",+]);++// Mirrors src/hooks/use-select-conversation-tab.ts so a non-React caller (the+// WebSocket dispatch) gets the same "reveal the right panel if collapsed"+// behavior as in-app tab switches.+function navigateToTab(tab: ConversationTab) {+ const store = useConversationStore.getState();+ store.setSelectedTab(tab);+ if (!store.isRightPanelShown) {+ store.setHasRightPanelToggled(true);+ store.setIsRightPanelShown(true);+ }+}++function isValidTab(value: string): value is ConversationTab {+ return VALID_TABS.has(value as ConversationTab);+}++export function handleCanvasUIAction(action: CanvasUIAction): void {+ switch (action.command) {+ case "navigate_to_file":+ case "show_preview":+ navigateToTab("files");+ if (action.path) {+ useFilesTabStore.getState().setSelectedPath(action.path);+ }+ return;+ case "open_tab":+ if (action.tab && isValidTab(action.tab)) {+ navigateToTab(action.tab);+ } else if (action.tab) {+ // Surface unknown tab names so they're diagnosable from the browser+ // console rather than failing silently. Valid tabs are listed in+ // VALID_TABS above and mirror ConversationTab.+ console.warn(+ `[canvas_ui] Ignoring open_tab with unknown tab: ${action.tab}`,+ );+ }+ return;+ }+}diff --git a/src/stores/files-tab-store.ts b/src/stores/files-tab-store.tsnew file mode 100644index 0000000000..59a32e754b--- /dev/null+++ b/src/stores/files-tab-store.ts@@ -0,0 +1,13 @@+import { create } from "zustand";++interface FilesTabState {+ selectedPath: string | null;+ setSelectedPath: (path: string | null) => void;+}++// Hoisted out of files-tab.tsx local state so non-React callers (e.g. the+// canvas_ui tool dispatcher in the WebSocket context) can drive selection.+export const useFilesTabStore = create<FilesTabState>((set) => ({+ selectedPath: null,+ setSelectedPath: (selectedPath) => set({ selectedPath }),+}));diff --git a/src/types/agent-server/core/base/action.ts b/src/types/agent-server/core/base/action.tsindex 3b41b100fb..70f035ac5e 100644--- a/src/types/agent-server/core/base/action.ts+++ b/src/types/agent-server/core/base/action.ts@@ -277,6 +277,17 @@ export interface InvokeSkillAction extends ActionBase<"InvokeSkillAction"> {name: string;}+/**+ * Frontend-injected custom tool. Emitted over the existing WebSocket as a+ * regular ActionEvent; intercepted client-side by handleCanvasUIAction.+ * The Python definition lives in tools/canvas_ui_tool.py.+ */+export interface CanvasUIAction extends ActionBase<"CanvasUIAction"> {+ command: "navigate_to_file" | "open_tab" | "show_preview";+ path?: string | null;+ tab?: string | null;+}+export type Action =| MCPToolAction| FinishAction@@ -299,4 +310,5 @@ export type Action =| BrowserCloseTabAction| GlobAction| GrepAction- | InvokeSkillAction;+ | InvokeSkillAction+ | CanvasUIAction;diff --git a/src/types/agent-server/core/base/base.ts b/src/types/agent-server/core/base/base.tsindex ea24c5bbb1..2159fbadc1 100644--- a/src/types/agent-server/core/base/base.ts+++ b/src/types/agent-server/core/base/base.ts@@ -20,7 +20,11 @@ type ActionOnlyType =| "BrowserGoBack"| "BrowserListTabs"| "BrowserSwitchTab"- | "BrowserCloseTab";+ | "BrowserCloseTab"+ // Frontend-injected custom tool. Not part of the upstream SDK Action+ // union but emitted as a regular ActionEvent over the WebSocket. See+ // tools/canvas_ui_tool.py and src/services/canvas-ui.ts.+ | "CanvasUI";type ObservationOnlyType = "Browser";diff --git a/src/types/agent-server/type-guards.ts b/src/types/agent-server/type-guards.tsindex 5a9cbdcfc8..b549538fc3 100644--- a/src/types/agent-server/type-guards.ts+++ b/src/types/agent-server/type-guards.ts@@ -9,6 +9,7 @@ import {TerminalObservation,BrowserObservation,BrowserNavigateAction,+ CanvasUIAction,} from "./core";import { AgentErrorEvent } from "./core/events/observation-event";import { MessageEvent } from "./core/events/message-event";@@ -153,6 +154,19 @@ export const isBrowserNavigateActionEvent = (): event is ActionEvent<BrowserNavigateAction> =>isActionEvent(event) && event.action.kind === "BrowserNavigateAction";+/**+ * Type guard for the canvas_ui custom tool's ActionEvent.+ *+ * The tool is injected via tool_module_qualnames (see canvas_ui_tool.py and+ * agent-server-adapter.ts). We discriminate on tool_name (which we control+ * via register_tool("canvas_ui", ...)). The predicate narrows the event so+ * the call site can read `event.action.command` etc. without further casts.+ */+export const isCanvasUIActionEvent = (+ event: OpenHandsEvent,+): event is ActionEvent<CanvasUIAction> =>+ isActionEvent(event) && event.tool_name === "canvas_ui";+/*** Type guard function to check if an event is a system prompt event*/diff --git a/tools/canvas_ui_tool.py b/tools/canvas_ui_tool.pynew file mode 100644index 0000000000..c402b45df6--- /dev/null+++ b/tools/canvas_ui_tool.py@@ -0,0 +1,129 @@+"""Canvas UI control tool.++Shipped with Agent Canvas. Mounted into the agent-server container at+``/canvas-tools`` and loaded via ``tool_module_qualnames`` so the agent can+direct the frontend (navigate to a file, switch tabs, show a preview).++The server-side executor is a no-op that returns an acknowledgment. The actual+UI effect happens client-side: the frontend watches the WebSocket stream for+``ActionEvent``s with ``tool_name == "canvas_ui"`` and dispatches the command.+"""++from collections.abc import Sequence+from typing import Literal++from pydantic import Field++from openhands.sdk import Action, Observation, ToolDefinition+from openhands.sdk.tool import (+ ToolAnnotations,+ ToolExecutor,+ register_tool,+)+++CanvasCommand = Literal["navigate_to_file", "open_tab", "show_preview"]+CanvasTab = Literal[+ "files", "browser", "vscode", "terminal", "planner", "tasklist"+]+++class CanvasUIAction(Action):+ """Direct the Agent Canvas frontend to perform a UI action."""++ command: CanvasCommand = Field(description="UI command to dispatch.")+ path: str | None = Field(+ default=None,+ description=(+ "Workspace-relative file path. Required for navigate_to_file and "+ "show_preview; ignored otherwise."+ ),+ )+ tab: CanvasTab | None = Field(+ default=None,+ description=(+ "Tab to open. Required for open_tab; ignored otherwise. One of "+ "files, browser, vscode, terminal, planner, tasklist."+ ),+ )+++class CanvasUIObservation(Observation):+ """Acknowledgment that the UI command was dispatched to the frontend."""+++class CanvasUIExecutor(ToolExecutor[CanvasUIAction, CanvasUIObservation]):+ def __call__(+ self,+ action: CanvasUIAction,+ conversation=None, # noqa: ARG002+ ) -> CanvasUIObservation:+ return CanvasUIObservation.from_text(+ f"UI command '{action.command}' dispatched to the Agent Canvas frontend."+ )+++_CANVAS_UI_DESCRIPTION = """The user is interacting with you inside Agent Canvas β a web UI with a chat panel on the left and a tabbed right-side panel (files, terminal, browser, vscode, planner, tasklist). This tool lets you drive that right-side panel so the user sees what you just produced.++They will NOT see the files you wrote, the terminal output, or the browser+unless you call this tool to switch the right-side panel to the relevant+tab. Call this every time you finish work that produces something the user+should look at β don't rely on them noticing on their own.++When to call (pick the most specific option that matches your last action):++* You wrote or modified a single file (ANY language, ANY size β including+ small scripts like a hello-world bash file) β+ command="navigate_to_file", path=<workspace-relative path of that file>++* You generated an HTML page, image, SVG, PDF, markdown report, or other+ previewable artifact β+ command="show_preview", path=<that file>++* You finished editing multiple files in one logical step β+ command="open_tab", tab="files"+ (The Files tab automatically renders a diff view when the workspace has+ uncommitted git changes, which covers the "highlight changes" case.)++* You ran a long-running terminal command, or one whose output the user+ should inspect β+ command="open_tab", tab="terminal"++* You browsed to a URL the user should see β+ command="open_tab", tab="browser"++Call this BEFORE writing your chat-message summary of the change, so the+artifact is visible while the user reads what you did. One canvas_ui call+per logical step is enough β don't repeat it for the same file or tab in+the same turn."""+++class CanvasUITool(ToolDefinition[CanvasUIAction, CanvasUIObservation]):+ """Tool for controlling the Agent Canvas UI from the agent."""++ @classmethod+ def create(+ cls,+ conv_state=None, # noqa: ARG003+ **params, # noqa: ARG003+ ) -> Sequence["CanvasUITool"]:+ return [+ cls(+ description=_CANVAS_UI_DESCRIPTION,+ action_type=CanvasUIAction,+ observation_type=CanvasUIObservation,+ executor=CanvasUIExecutor(),+ annotations=ToolAnnotations(+ readOnlyHint=True,+ destructiveHint=False,+ idempotentHint=True,+ openWorldHint=False,+ ),+ )+ ]+++# Auto-register at import time. The agent-server imports this module via+# tool_module_qualnames; this call wires the tool into the registry so it can+# be referenced by name in conversation tool lists.+register_tool("canvas_ui", CanvasUITool)