Skip to content

πŸ™Œ OpenHands: AI-Driven Development

feat(frontend): add canvas_ui tool so the agent can drive the UI (#420)

Hiep Le5c33c10b18fb
diff --git a/__tests__/api/agent-server-adapter.test.ts b/__tests__/api/agent-server-adapter.test.ts
index 4af700b203..9ed1f9eacb 100644
--- a/__tests__/api/agent-server-adapter.test.ts
+++ b/__tests__/api/agent-server-adapter.test.ts
@@ -107,6 +107,7 @@ describe("buildStartConversationRequest", () => {
{ name: "terminal", params: {} },
{ name: "file_editor", params: {} },
{ name: "task_tracker", params: {} },
+ { name: "canvas_ui", params: {} },
{ name: "browser_tool_set", params: {} },
]);
expect(payload.agent.include_default_tools).toEqual([
@@ -171,6 +172,7 @@ describe("buildStartConversationRequest", () => {
{ name: "terminal", params: {} },
{ name: "file_editor", params: {} },
{ name: "task_tracker", params: {} },
+ { name: "canvas_ui", params: {} },
]);
});
@@ -263,7 +265,11 @@ describe("buildStartConversationRequest", () => {
}) as Record<string, unknown>;
expect(payload.hook_config).toEqual({ on_start: [] });
+ // Canvas-UI tool is auto-injected; user-supplied entries are merged in
+ // alongside it. The dedicated canvas_ui describe block below pins the
+ // exact merge semantics.
expect(payload.tool_module_qualnames).toEqual({
+ canvas_ui: "canvas_ui_tool",
demo_tool: "pkg.tools.demo",
});
expect(payload.agent_definitions).toEqual([
@@ -318,6 +324,35 @@ describe("buildStartConversationRequest", () => {
},
});
});
+
+ describe("canvas_ui tool injection", () => {
+ it("always registers canvas_ui_tool in tool_module_qualnames, even when no user settings supply qualnames", () => {
+ const payload = buildStartConversationRequest({
+ settings: DEFAULT_SETTINGS,
+ }) as { tool_module_qualnames: Record<string, string> };
+
+ expect(payload.tool_module_qualnames).toMatchObject({
+ canvas_ui: "canvas_ui_tool",
+ });
+ });
+
+ it("merges user-supplied tool_module_qualnames alongside canvas_ui_tool without dropping either side", () => {
+ const payload = buildStartConversationRequest({
+ settings: {
+ ...DEFAULT_SETTINGS,
+ conversation_settings: {
+ ...DEFAULT_SETTINGS.conversation_settings,
+ tool_module_qualnames: { my_tool: "my_package.my_tool" },
+ },
+ },
+ }) as { tool_module_qualnames: Record<string, string> };
+
+ expect(payload.tool_module_qualnames).toEqual({
+ canvas_ui: "canvas_ui_tool",
+ my_tool: "my_package.my_tool",
+ });
+ });
+ });
});
describe("getDefaultConversationTitle", () => {
diff --git a/__tests__/routes/files-tab.test.tsx b/__tests__/routes/files-tab.test.tsx
index 8529e9132e..5b84294cc8 100644
--- a/__tests__/routes/files-tab.test.tsx
+++ b/__tests__/routes/files-tab.test.tsx
@@ -6,6 +6,7 @@ import { describe, it, expect, vi, beforeEach } from "vitest";
import { MemoryRouter } from "react-router";
import FilesTab from "#/routes/files-tab";
+import { useFilesTabStore } from "#/stores/files-tab-store";
// Mocks must be declared before the SUT is imported.
const useHasAttachedSourceMock = vi.fn();
@@ -58,6 +59,14 @@ function renderTab() {
describe("FilesTab", () => {
beforeEach(() => {
+ // `selectedPath` lives in a global Zustand store (useFilesTabStore) and
+ // the auto-select effect re-fires when the store is reset between tests,
+ // which can race with the Zustand mock's afterEach reset and leave the
+ // store polluted with the previous test's path. Resetting here, after
+ // the previous test's cleanup() has unmounted any FilesTab, defeats
+ // that race so each test starts with a clean selection.
+ useFilesTabStore.setState({ selectedPath: null });
+
useHasAttachedSourceMock.mockReset();
useHasGitCommitsMock.mockReset();
useWorkspaceFilesMock.mockReset();
diff --git a/__tests__/scripts/dev-docker.test.ts b/__tests__/scripts/dev-docker.test.ts
index f4b5fb914f..04bffcdabe 100644
--- a/__tests__/scripts/dev-docker.test.ts
+++ b/__tests__/scripts/dev-docker.test.ts
@@ -1,3 +1,10 @@
+// @vitest-environment node
+// These tests load `scripts/dev-docker.mjs`, which constructs file:// URLs
+// relative to its own location via `new URL("../tools", import.meta.url)`.
+// jsdom's URL constructor ignores file:// base URLs (it falls back to its
+// document base, e.g. http://localhost:3000/), breaking that resolution;
+// the Node environment has the standard WHATWG URL behavior that honors
+// the file:// base.
import { describe, expect, it } from "vitest";
import {
diff --git a/__tests__/scripts/dev-extra-backend.test.ts b/__tests__/scripts/dev-extra-backend.test.ts
index fde467706b..f7a0cfa488 100644
--- a/__tests__/scripts/dev-extra-backend.test.ts
+++ b/__tests__/scripts/dev-extra-backend.test.ts
@@ -1,3 +1,10 @@
+// @vitest-environment node
+// These tests load `scripts/dev-safe.mjs` (via dev-extra-backend), which
+// constructs file:// URLs relative to its own location via
+// `new URL("../tools", import.meta.url)`. jsdom's URL constructor ignores
+// file:// base URLs (it falls back to its document base, e.g.
+// http://localhost:3000/), breaking that resolution; the Node environment
+// has the standard WHATWG URL behavior that honors the file:// base.
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { fileURLToPath } from "node:url";
diff --git a/__tests__/scripts/dev-safe.test.ts b/__tests__/scripts/dev-safe.test.ts
index 929851ee27..2ac9101132 100644
--- a/__tests__/scripts/dev-safe.test.ts
+++ b/__tests__/scripts/dev-safe.test.ts
@@ -1,3 +1,10 @@
+// @vitest-environment node
+// These tests load `scripts/dev-safe.mjs`, which constructs file:// URLs
+// relative to its own location via `new URL("../tools", import.meta.url)`.
+// jsdom's URL constructor ignores file:// base URLs (it falls back to its
+// document base, e.g. http://localhost:3000/), breaking that resolution;
+// the Node environment has the standard WHATWG URL behavior that honors
+// the file:// base.
import net from "node:net";
import { spawn } from "node:child_process";
import { once } from "node:events";
diff --git a/__tests__/scripts/dev-with-automation.test.ts b/__tests__/scripts/dev-with-automation.test.ts
index 2f7eed62ed..727e012a6c 100644
--- a/__tests__/scripts/dev-with-automation.test.ts
+++ b/__tests__/scripts/dev-with-automation.test.ts
@@ -1,3 +1,10 @@
+// @vitest-environment node
+// These tests load `scripts/dev-with-automation.mjs` and `scripts/dev-safe.mjs`,
+// which construct file:// URLs relative to their own location via
+// `new URL("../tools", import.meta.url)`. jsdom's URL constructor ignores
+// file:// base URLs (it falls back to its document base, e.g.
+// http://localhost:3000/), breaking that resolution; the Node environment
+// has the standard WHATWG URL behavior that honors the file:// base.
import net from "node:net";
import { spawn } from "node:child_process";
import { once } from "node:events";
diff --git a/__tests__/services/canvas-ui.test.ts b/__tests__/services/canvas-ui.test.ts
new file mode 100644
index 0000000000..a0d8283922
--- /dev/null
+++ b/__tests__/services/canvas-ui.test.ts
@@ -0,0 +1,118 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+import { handleCanvasUIAction } from "#/services/canvas-ui";
+import { useConversationStore } from "#/stores/conversation-store";
+import { useFilesTabStore } from "#/stores/files-tab-store";
+import type { CanvasUIAction } from "#/types/agent-server/core";
+import { isCanvasUIActionEvent } from "#/types/agent-server/type-guards";
+
+// Helper: build a CanvasUIAction without repeating the literal `kind`
+// discriminator in every test case.
+function action(overrides: Partial<CanvasUIAction>): CanvasUIAction {
+ return { kind: "CanvasUIAction", ...overrides } as CanvasUIAction;
+}
+
+describe("handleCanvasUIAction", () => {
+ beforeEach(() => {
+ // Arrange (shared): collapsed right panel, no selected file. Lets us
+ // observe both the tab/panel toggling and the path mutation that the
+ // dispatcher performs.
+ useConversationStore.setState({
+ selectedTab: null,
+ isRightPanelShown: false,
+ hasRightPanelToggled: false,
+ });
+ useFilesTabStore.setState({ selectedPath: null });
+ });
+
+ it("navigate_to_file selects the files tab, reveals the panel, and sets selectedPath", () => {
+ handleCanvasUIAction(
+ action({ command: "navigate_to_file", path: "docs/intro.html" }),
+ );
+
+ const conv = useConversationStore.getState();
+ expect(conv.selectedTab).toBe("files");
+ expect(conv.isRightPanelShown).toBe(true);
+ expect(useFilesTabStore.getState().selectedPath).toBe("docs/intro.html");
+ });
+
+ it("show_preview selects the files tab and the requested path", () => {
+ handleCanvasUIAction(
+ action({ command: "show_preview", path: "report.html" }),
+ );
+
+ expect(useConversationStore.getState().selectedTab).toBe("files");
+ expect(useFilesTabStore.getState().selectedPath).toBe("report.html");
+ });
+
+ it("open_tab switches to a valid tab without touching selectedPath", () => {
+ handleCanvasUIAction(action({ command: "open_tab", tab: "terminal" }));
+
+ expect(useConversationStore.getState().selectedTab).toBe("terminal");
+ expect(useFilesTabStore.getState().selectedPath).toBeNull();
+ });
+
+ it("open_tab ignores unknown tab values and surfaces a warning for debuggability", () => {
+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+
+ try {
+ handleCanvasUIAction(action({ command: "open_tab", tab: "not_a_tab" }));
+
+ expect(useConversationStore.getState().selectedTab).toBeNull();
+ expect(useConversationStore.getState().isRightPanelShown).toBe(false);
+ expect(warnSpy).toHaveBeenCalledWith(
+ expect.stringContaining("not_a_tab"),
+ );
+ } finally {
+ warnSpy.mockRestore();
+ }
+ });
+
+ it("navigate_to_file leaves selectedPath alone when no path is supplied", () => {
+ useFilesTabStore.setState({ selectedPath: "previous.txt" });
+
+ handleCanvasUIAction(action({ command: "navigate_to_file", path: null }));
+
+ expect(useFilesTabStore.getState().selectedPath).toBe("previous.txt");
+ expect(useConversationStore.getState().selectedTab).toBe("files");
+ });
+});
+
+describe("isCanvasUIActionEvent", () => {
+ function makeActionEvent(overrides: Record<string, unknown> = {}) {
+ return {
+ id: "evt-1",
+ timestamp: "2026-05-13T00:00:00Z",
+ source: "agent",
+ action: { kind: "CanvasUIAction" },
+ tool_name: "canvas_ui",
+ tool_call_id: "call-1",
+ ...overrides,
+ };
+ }
+
+ it("returns true for an ActionEvent whose tool_name is canvas_ui", () => {
+ expect(isCanvasUIActionEvent(makeActionEvent() as never)).toBe(true);
+ });
+
+ it("returns false when tool_name belongs to a different tool", () => {
+ expect(
+ isCanvasUIActionEvent(
+ makeActionEvent({ tool_name: "execute_bash" }) as never,
+ ),
+ ).toBe(false);
+ });
+
+ it("returns false for a non-action event (no action field)", () => {
+ const observationEvent = {
+ id: "evt-2",
+ timestamp: "2026-05-13T00:00:00Z",
+ source: "environment",
+ observation: { kind: "ExecuteBashObservation" },
+ tool_name: "canvas_ui",
+ tool_call_id: "call-1",
+ };
+
+ expect(isCanvasUIActionEvent(observationEvent as never)).toBe(false);
+ });
+});
diff --git a/scripts/dev-docker.mjs b/scripts/dev-docker.mjs
index ff6e5224aa..d2a5a14969 100644
--- a/scripts/dev-docker.mjs
+++ b/scripts/dev-docker.mjs
@@ -71,6 +71,15 @@ import { buildFrontend } from "./static-build.mjs";
// Path inside the container where OH_AGENT_SERVER_LOCAL_PATH is bind-mounted.
const CONTAINER_LOCAL_SDK_DIR = "/agent-server-src";
+// Host directory containing Agent-Canvas-specific Python tools (e.g. the
+// canvas_ui tool). Mounted read-only into the container and added to
+// sys.path via OH_EXTRA_PYTHON_PATH so the agent-server can import the
+// modules listed in `tool_module_qualnames`.
+const CONTAINER_CANVAS_TOOLS_DIR = "/canvas-tools";
+const HOST_CANVAS_TOOLS_DIR = fileURLToPath(
+ new URL("../tools", import.meta.url),
+);
+
// Docker image for the agent-server.
const AGENT_SERVER_REPO = "ghcr.io/openhands/agent-server";
// Default tag used when OH_AGENT_SERVER_GIT_REF is not set.
@@ -270,6 +279,14 @@ function startAgentServerDocker(config) {
const dockerArgs = ["run", "--rm", "--name", CONTAINER_NAME, "--init"];
dockerArgs.push(...getDockerUserArgs(userSpec));
dockerArgs.push("-v", `${process.env.PROJECT_PATH}:/projects`);
+ // Read-only mount of the Agent-Canvas tools directory. Coupled with
+ // OH_EXTRA_PYTHON_PATH below so the agent-server can import
+ // canvas_ui_tool when the conversation request lists it under
+ // tool_module_qualnames.
+ dockerArgs.push(
+ "-v",
+ `${HOST_CANVAS_TOOLS_DIR}:${CONTAINER_CANVAS_TOOLS_DIR}:ro`,
+ );
// Bind-mount the local software-agent-sdk checkout if requested. Mounted
// rw so editable installs can write their .dist-info into each package
@@ -318,6 +335,9 @@ function startAgentServerDocker(config) {
// Required so the secret-seeding PUT /api/settings/secrets call from
// the host can authenticate against the agent-server in the container.
OH_SESSION_API_KEYS_0: config.sessionApiKey,
+ // Make the mounted canvas-tools directory importable so the agent-server
+ // can resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool).
+ OH_EXTRA_PYTHON_PATH: CONTAINER_CANVAS_TOOLS_DIR,
};
for (const [k, v] of Object.entries(containerEnv)) {
dockerArgs.push("-e", `${k}=${v}`);
@@ -381,12 +401,14 @@ if (isMainModule) {
export {
AGENT_SERVER_REPO,
+ CONTAINER_CANVAS_TOOLS_DIR,
CONTAINER_HOME_DIR,
CONTAINER_LOCAL_SDK_DIR,
CONTAINER_NAME,
CONTAINER_OPENHANDS_DIR,
CONTAINER_WORKSPACES_DIR,
DEFAULT_AGENT_SERVER_TAG,
+ HOST_CANVAS_TOOLS_DIR,
checkDockerPrereqs,
getDockerHomeTmpfsArgs,
getDockerUserArgs,
diff --git a/scripts/dev-safe.mjs b/scripts/dev-safe.mjs
index b456648af3..3e423b7c82 100644
--- a/scripts/dev-safe.mjs
+++ b/scripts/dev-safe.mjs
@@ -14,7 +14,7 @@ import { homedir, tmpdir } from "node:os";
import path from "node:path";
import process from "node:process";
import { setTimeout as delay } from "node:timers/promises";
-import { pathToFileURL } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
import {
getProcessTreeSpawnOptions,
@@ -523,6 +523,7 @@ export async function buildSafeDevConfigAsync(
* @property {string} workingDir
* @property {string} secretKey
* @property {string} sessionApiKey
+ * @property {string} canvasToolsDir
*/
/**
@@ -562,6 +563,12 @@ function buildConfigFromPorts(ports, cwd, env) {
env.VITE_SESSION_API_KEY ||
getOrCreatePersistedSessionApiKey(persistedKeyPath);
+ // Host directory containing Agent-Canvas-specific Python tools (e.g. the
+ // canvas_ui tool). Added to OH_EXTRA_PYTHON_PATH below so the agent-server
+ // can import the modules listed in `tool_module_qualnames`. Lives at
+ // <repo-root>/tools relative to this script.
+ const canvasToolsDir = fileURLToPath(new URL("../tools", import.meta.url));
+
return {
cwd,
backendPort,
@@ -576,6 +583,7 @@ function buildConfigFromPorts(ports, cwd, env) {
workingDir: env.VITE_WORKING_DIR || workspacesPath,
secretKey,
sessionApiKey,
+ canvasToolsDir,
};
}
@@ -597,6 +605,9 @@ export function buildAgentServerEnv(config) {
OH_SECRET_KEY: config.secretKey,
// Use OH_SESSION_API_KEYS_0 for agent-server V1 config format
OH_SESSION_API_KEYS_0: config.sessionApiKey,
+ // Make the host tools/ directory importable so the agent-server can
+ // resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool).
+ OH_EXTRA_PYTHON_PATH: config.canvasToolsDir,
};
}
diff --git a/src/api/agent-server-adapter.ts b/src/api/agent-server-adapter.ts
index b768b8cc16..dfe59df089 100644
--- a/src/api/agent-server-adapter.ts
+++ b/src/api/agent-server-adapter.ts
@@ -42,7 +42,18 @@ export interface DirectConversationInfo {
} | null;
}
-const DEFAULT_TOOL_NAMES = ["terminal", "file_editor", "task_tracker"];
+// Module qualname for the Canvas-UI tool. The agent-server imports this via
+// tool_module_qualnames; the host directory is exposed via OH_EXTRA_PYTHON_PATH
+// (see scripts/dev-docker.mjs and scripts/dev-safe.mjs).
+const CANVAS_UI_TOOL_NAME = "canvas_ui";
+const CANVAS_UI_TOOL_MODULE = "canvas_ui_tool";
+
+const DEFAULT_TOOL_NAMES = [
+ "terminal",
+ "file_editor",
+ "task_tracker",
+ CANVAS_UI_TOOL_NAME,
+];
const BROWSER_TOOL_SET_NAME = "browser_tool_set";
const DEFAULT_BUILT_IN_TOOL_NAMES = ["FinishTool", "ThinkTool"];
const SWITCH_LLM_TOOL_NAME = "SwitchLLMTool";
@@ -611,9 +622,16 @@ export function buildStartConversationRequest(
payload.hook_config = conversationSettings.hook_config;
}
- if (conversationSettings.tool_module_qualnames) {
- payload.tool_module_qualnames = conversationSettings.tool_module_qualnames;
- }
+ // Always include the canvas_ui tool module so the agent-server imports it
+ // and registers the tool. User-supplied entries from conversationSettings
+ // take precedence on key conflict (the canvas_ui key is ours and shouldn't
+ // collide in practice).
+ payload.tool_module_qualnames = {
+ [CANVAS_UI_TOOL_NAME]: CANVAS_UI_TOOL_MODULE,
+ ...((conversationSettings.tool_module_qualnames as
+ | Record<string, string>
+ | undefined) ?? {}),
+ };
if (conversationSettings.agent_definitions) {
payload.agent_definitions = conversationSettings.agent_definitions;
diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsx
index 0bac2745ec..dd28175999 100644
--- a/src/contexts/conversation-websocket-context.tsx
+++ b/src/contexts/conversation-websocket-context.tsx
@@ -33,7 +33,9 @@ import {
isPlanningFileEditorObservationEvent,
isBrowserObservationEvent,
isBrowserNavigateActionEvent,
+ isCanvasUIActionEvent,
} from "#/types/agent-server/type-guards";
+import { handleCanvasUIAction } from "#/services/canvas-ui";
import { ConversationStateUpdateEventStats } from "#/types/agent-server/core/events/conversation-state-event";
import type {
ConversationErrorEvent,
@@ -494,6 +496,14 @@ export function ConversationWebSocketProvider({
if (isBrowserNavigateActionEvent(event)) {
useBrowserStore.getState().setUrl(event.action.url);
}
+
+ // Handle canvas_ui custom-tool ActionEvents - drive the frontend
+ // (navigate to a file, switch tabs, show a preview). The tool
+ // executes server-side as a no-op; the actual UI change happens
+ // here on the client.
+ if (isCanvasUIActionEvent(event)) {
+ handleCanvasUIAction(event.action);
+ }
}
} catch (error) {
console.warn("Failed to parse WebSocket message as JSON:", error);
diff --git a/src/routes/files-tab.tsx b/src/routes/files-tab.tsx
index b6c36aaad3..f19f698ddc 100644
--- a/src/routes/files-tab.tsx
+++ b/src/routes/files-tab.tsx
@@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next";
import { useQueryClient } from "@tanstack/react-query";
import { I18nKey } from "#/i18n/declaration";
+import { useFilesTabStore } from "#/stores/files-tab-store";
import { useWorkspaceFiles } from "#/hooks/query/use-workspace-files";
import { useWorkspaceFileContent } from "#/hooks/query/use-workspace-file-content";
import { useHasAttachedSource } from "#/hooks/use-has-attached-source";
@@ -64,7 +65,8 @@ function FilesTab() {
const filesQuery = useWorkspaceFiles();
const paths = useMemo(() => filesQuery.data ?? [], [filesQuery.data]);
- const [selectedPath, setSelectedPath] = useState<string | null>(null);
+ const selectedPath = useFilesTabStore((s) => s.selectedPath);
+ const setSelectedPath = useFilesTabStore((s) => s.setSelectedPath);
// Pre-fetch the selected file's content here too so the toolbar's
// "open in new window" link can reach for its `staticUrl`. react-query
diff --git a/src/services/canvas-ui.ts b/src/services/canvas-ui.ts
new file mode 100644
index 0000000000..cf95f54fb2
--- /dev/null
+++ b/src/services/canvas-ui.ts
@@ -0,0 +1,55 @@
+import {
+ ConversationTab,
+ useConversationStore,
+} from "#/stores/conversation-store";
+import { useFilesTabStore } from "#/stores/files-tab-store";
+import type { CanvasUIAction } from "#/types/agent-server/core";
+
+const VALID_TABS: ReadonlySet<ConversationTab> = new Set<ConversationTab>([
+ "files",
+ "browser",
+ "vscode",
+ "terminal",
+ "planner",
+ "tasklist",
+]);
+
+// Mirrors src/hooks/use-select-conversation-tab.ts so a non-React caller (the
+// WebSocket dispatch) gets the same "reveal the right panel if collapsed"
+// behavior as in-app tab switches.
+function navigateToTab(tab: ConversationTab) {
+ const store = useConversationStore.getState();
+ store.setSelectedTab(tab);
+ if (!store.isRightPanelShown) {
+ store.setHasRightPanelToggled(true);
+ store.setIsRightPanelShown(true);
+ }
+}
+
+function isValidTab(value: string): value is ConversationTab {
+ return VALID_TABS.has(value as ConversationTab);
+}
+
+export function handleCanvasUIAction(action: CanvasUIAction): void {
+ switch (action.command) {
+ case "navigate_to_file":
+ case "show_preview":
+ navigateToTab("files");
+ if (action.path) {
+ useFilesTabStore.getState().setSelectedPath(action.path);
+ }
+ return;
+ case "open_tab":
+ if (action.tab && isValidTab(action.tab)) {
+ navigateToTab(action.tab);
+ } else if (action.tab) {
+ // Surface unknown tab names so they're diagnosable from the browser
+ // console rather than failing silently. Valid tabs are listed in
+ // VALID_TABS above and mirror ConversationTab.
+ console.warn(
+ `[canvas_ui] Ignoring open_tab with unknown tab: ${action.tab}`,
+ );
+ }
+ return;
+ }
+}
diff --git a/src/stores/files-tab-store.ts b/src/stores/files-tab-store.ts
new file mode 100644
index 0000000000..59a32e754b
--- /dev/null
+++ b/src/stores/files-tab-store.ts
@@ -0,0 +1,13 @@
+import { create } from "zustand";
+
+interface FilesTabState {
+ selectedPath: string | null;
+ setSelectedPath: (path: string | null) => void;
+}
+
+// Hoisted out of files-tab.tsx local state so non-React callers (e.g. the
+// canvas_ui tool dispatcher in the WebSocket context) can drive selection.
+export const useFilesTabStore = create<FilesTabState>((set) => ({
+ selectedPath: null,
+ setSelectedPath: (selectedPath) => set({ selectedPath }),
+}));
diff --git a/src/types/agent-server/core/base/action.ts b/src/types/agent-server/core/base/action.ts
index 3b41b100fb..70f035ac5e 100644
--- a/src/types/agent-server/core/base/action.ts
+++ b/src/types/agent-server/core/base/action.ts
@@ -277,6 +277,17 @@ export interface InvokeSkillAction extends ActionBase<"InvokeSkillAction"> {
name: string;
}
+/**
+ * Frontend-injected custom tool. Emitted over the existing WebSocket as a
+ * regular ActionEvent; intercepted client-side by handleCanvasUIAction.
+ * The Python definition lives in tools/canvas_ui_tool.py.
+ */
+export interface CanvasUIAction extends ActionBase<"CanvasUIAction"> {
+ command: "navigate_to_file" | "open_tab" | "show_preview";
+ path?: string | null;
+ tab?: string | null;
+}
+
export type Action =
| MCPToolAction
| FinishAction
@@ -299,4 +310,5 @@ export type Action =
| BrowserCloseTabAction
| GlobAction
| GrepAction
- | InvokeSkillAction;
+ | InvokeSkillAction
+ | CanvasUIAction;
diff --git a/src/types/agent-server/core/base/base.ts b/src/types/agent-server/core/base/base.ts
index ea24c5bbb1..2159fbadc1 100644
--- a/src/types/agent-server/core/base/base.ts
+++ b/src/types/agent-server/core/base/base.ts
@@ -20,7 +20,11 @@ type ActionOnlyType =
| "BrowserGoBack"
| "BrowserListTabs"
| "BrowserSwitchTab"
- | "BrowserCloseTab";
+ | "BrowserCloseTab"
+ // Frontend-injected custom tool. Not part of the upstream SDK Action
+ // union but emitted as a regular ActionEvent over the WebSocket. See
+ // tools/canvas_ui_tool.py and src/services/canvas-ui.ts.
+ | "CanvasUI";
type ObservationOnlyType = "Browser";
diff --git a/src/types/agent-server/type-guards.ts b/src/types/agent-server/type-guards.ts
index 5a9cbdcfc8..b549538fc3 100644
--- a/src/types/agent-server/type-guards.ts
+++ b/src/types/agent-server/type-guards.ts
@@ -9,6 +9,7 @@ import {
TerminalObservation,
BrowserObservation,
BrowserNavigateAction,
+ CanvasUIAction,
} from "./core";
import { AgentErrorEvent } from "./core/events/observation-event";
import { MessageEvent } from "./core/events/message-event";
@@ -153,6 +154,19 @@ export const isBrowserNavigateActionEvent = (
): event is ActionEvent<BrowserNavigateAction> =>
isActionEvent(event) && event.action.kind === "BrowserNavigateAction";
+/**
+ * Type guard for the canvas_ui custom tool's ActionEvent.
+ *
+ * The tool is injected via tool_module_qualnames (see canvas_ui_tool.py and
+ * agent-server-adapter.ts). We discriminate on tool_name (which we control
+ * via register_tool("canvas_ui", ...)). The predicate narrows the event so
+ * the call site can read `event.action.command` etc. without further casts.
+ */
+export const isCanvasUIActionEvent = (
+ event: OpenHandsEvent,
+): event is ActionEvent<CanvasUIAction> =>
+ isActionEvent(event) && event.tool_name === "canvas_ui";
+
/**
* Type guard function to check if an event is a system prompt event
*/
diff --git a/tools/canvas_ui_tool.py b/tools/canvas_ui_tool.py
new file mode 100644
index 0000000000..c402b45df6
--- /dev/null
+++ b/tools/canvas_ui_tool.py
@@ -0,0 +1,129 @@
+"""Canvas UI control tool.
+
+Shipped with Agent Canvas. Mounted into the agent-server container at
+``/canvas-tools`` and loaded via ``tool_module_qualnames`` so the agent can
+direct the frontend (navigate to a file, switch tabs, show a preview).
+
+The server-side executor is a no-op that returns an acknowledgment. The actual
+UI effect happens client-side: the frontend watches the WebSocket stream for
+``ActionEvent``s with ``tool_name == "canvas_ui"`` and dispatches the command.
+"""
+
+from collections.abc import Sequence
+from typing import Literal
+
+from pydantic import Field
+
+from openhands.sdk import Action, Observation, ToolDefinition
+from openhands.sdk.tool import (
+ ToolAnnotations,
+ ToolExecutor,
+ register_tool,
+)
+
+
+CanvasCommand = Literal["navigate_to_file", "open_tab", "show_preview"]
+CanvasTab = Literal[
+ "files", "browser", "vscode", "terminal", "planner", "tasklist"
+]
+
+
+class CanvasUIAction(Action):
+ """Direct the Agent Canvas frontend to perform a UI action."""
+
+ command: CanvasCommand = Field(description="UI command to dispatch.")
+ path: str | None = Field(
+ default=None,
+ description=(
+ "Workspace-relative file path. Required for navigate_to_file and "
+ "show_preview; ignored otherwise."
+ ),
+ )
+ tab: CanvasTab | None = Field(
+ default=None,
+ description=(
+ "Tab to open. Required for open_tab; ignored otherwise. One of "
+ "files, browser, vscode, terminal, planner, tasklist."
+ ),
+ )
+
+
+class CanvasUIObservation(Observation):
+ """Acknowledgment that the UI command was dispatched to the frontend."""
+
+
+class CanvasUIExecutor(ToolExecutor[CanvasUIAction, CanvasUIObservation]):
+ def __call__(
+ self,
+ action: CanvasUIAction,
+ conversation=None, # noqa: ARG002
+ ) -> CanvasUIObservation:
+ return CanvasUIObservation.from_text(
+ f"UI command '{action.command}' dispatched to the Agent Canvas frontend."
+ )
+
+
+_CANVAS_UI_DESCRIPTION = """The user is interacting with you inside Agent Canvas β€” a web UI with a chat panel on the left and a tabbed right-side panel (files, terminal, browser, vscode, planner, tasklist). This tool lets you drive that right-side panel so the user sees what you just produced.
+
+They will NOT see the files you wrote, the terminal output, or the browser
+unless you call this tool to switch the right-side panel to the relevant
+tab. Call this every time you finish work that produces something the user
+should look at β€” don't rely on them noticing on their own.
+
+When to call (pick the most specific option that matches your last action):
+
+* You wrote or modified a single file (ANY language, ANY size β€” including
+ small scripts like a hello-world bash file) β†’
+ command="navigate_to_file", path=<workspace-relative path of that file>
+
+* You generated an HTML page, image, SVG, PDF, markdown report, or other
+ previewable artifact β†’
+ command="show_preview", path=<that file>
+
+* You finished editing multiple files in one logical step β†’
+ command="open_tab", tab="files"
+ (The Files tab automatically renders a diff view when the workspace has
+ uncommitted git changes, which covers the "highlight changes" case.)
+
+* You ran a long-running terminal command, or one whose output the user
+ should inspect β†’
+ command="open_tab", tab="terminal"
+
+* You browsed to a URL the user should see β†’
+ command="open_tab", tab="browser"
+
+Call this BEFORE writing your chat-message summary of the change, so the
+artifact is visible while the user reads what you did. One canvas_ui call
+per logical step is enough β€” don't repeat it for the same file or tab in
+the same turn."""
+
+
+class CanvasUITool(ToolDefinition[CanvasUIAction, CanvasUIObservation]):
+ """Tool for controlling the Agent Canvas UI from the agent."""
+
+ @classmethod
+ def create(
+ cls,
+ conv_state=None, # noqa: ARG003
+ **params, # noqa: ARG003
+ ) -> Sequence["CanvasUITool"]:
+ return [
+ cls(
+ description=_CANVAS_UI_DESCRIPTION,
+ action_type=CanvasUIAction,
+ observation_type=CanvasUIObservation,
+ executor=CanvasUIExecutor(),
+ annotations=ToolAnnotations(
+ readOnlyHint=True,
+ destructiveHint=False,
+ idempotentHint=True,
+ openWorldHint=False,
+ ),
+ )
+ ]
+
+
+# Auto-register at import time. The agent-server imports this module via
+# tool_module_qualnames; this call wires the tool into the registry so it can
+# be referenced by name in conversation tool lists.
+register_tool("canvas_ui", CanvasUITool)