Skip to content

πŸ™Œ OpenHands: AI-Driven Development

refactor: define canvas UI as an SDK client tool (#1797)

🐾 smolpaws2b7ceea667b4
diff --git a/__tests__/api/agent-server-adapter.test.ts b/__tests__/api/agent-server-adapter.test.ts
index 17f699d4f1..fa3d767dc6 100644
--- a/__tests__/api/agent-server-adapter.test.ts
+++ b/__tests__/api/agent-server-adapter.test.ts
@@ -1,4 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui";
import {
ACP_SERVER_TAG_KEY,
@@ -117,7 +118,6 @@ describe("buildStartConversationRequest", () => {
{ name: "terminal", params: {} },
{ name: "file_editor", params: {} },
{ name: "task_tracker", params: {} },
- { name: "canvas_ui", params: {} },
{ name: "browser_tool_set", params: {} },
{ name: "task_tool_set", params: {} },
]);
@@ -412,11 +412,7 @@ describe("buildStartConversationRequest", () => {
}) as Record<string, unknown>;
expect(payload.hook_config).toEqual({ on_start: [] });
- // Canvas-UI tool is auto-injected; user-supplied entries are merged in
- // alongside it. The dedicated canvas_ui describe block below pins the
- // exact merge semantics.
expect(payload.tool_module_qualnames).toEqual({
- canvas_ui: "canvas_ui_tool",
demo_tool: "pkg.tools.demo",
});
expect(payload.agent_definitions).toEqual([
@@ -595,40 +591,88 @@ describe("buildStartConversationRequest", () => {
});
});
- describe("canvas_ui tool injection", () => {
- it("registers canvas_ui_tool in tool_module_qualnames when the backend advertises canvas_ui", () => {
+ describe("canvas_ui client tool injection", () => {
+ it("sends canvas_ui as a client-defined JSON tool", () => {
const payload = buildStartConversationRequest({
settings: DEFAULT_SETTINGS,
- }) as { tool_module_qualnames: Record<string, string> };
+ });
- expect(payload.tool_module_qualnames).toMatchObject({
- canvas_ui: "canvas_ui_tool",
+ expect(payload.client_tools).toHaveLength(1);
+ expect(payload.client_tools[0]).toMatchObject({
+ name: CANVAS_UI_CLIENT_TOOL_NAME,
+ parameters: {
+ type: "object",
+ properties: {
+ command: {
+ enum: ["navigate_to_file", "open_tab", "show_preview"],
+ },
+ },
+ required: ["command"],
+ },
+ annotations: {
+ readOnlyHint: true,
+ destructiveHint: false,
+ idempotentHint: true,
+ openWorldHint: false,
+ },
});
+ expect(
+ payload.agent_settings?.tools?.map((tool) => tool.name) ?? [],
+ ).not.toContain("canvas_ui");
+ expect(payload.tool_module_qualnames).toBeUndefined();
});
- it("omits canvas_ui and its module qualname when the backend does not advertise canvas_ui", () => {
- mockIsAgentServerToolAvailable.mockImplementation(
- (toolName: string) => toolName !== "canvas_ui",
- );
+ it("omits the client tool for an inline ACP agent", () => {
+ const payload = buildStartConversationRequest({
+ settings: {
+ ...DEFAULT_SETTINGS,
+ agent_settings: {
+ ...DEFAULT_SETTINGS.agent_settings,
+ agent_kind: "acp",
+ acp_server: "custom",
+ acp_command: ["custom-acp"],
+ },
+ },
+ });
+
+ expect(payload.client_tools).toEqual([]);
+ });
+ it("omits the client tool for an ACP profile when global settings are stale", () => {
const payload = buildStartConversationRequest({
settings: DEFAULT_SETTINGS,
- }) as {
- agent_settings: { tools: Array<{ name: string }> };
- tool_module_qualnames?: Record<string, string>;
- };
+ agentProfileId: "profile-acp",
+ agentProfileKind: "acp",
+ });
- expect(
- payload.agent_settings.tools.map((tool) => tool.name),
- ).not.toContain("canvas_ui");
- expect(payload.tool_module_qualnames).toBeUndefined();
+ expect(payload.client_tools).toEqual([]);
});
- it("drops a user-supplied canvas_ui module qualname when the backend does not advertise canvas_ui", () => {
- mockIsAgentServerToolAvailable.mockImplementation(
- (toolName: string) => toolName !== "canvas_ui",
- );
+ it("sends the client tool for an OpenHands profile", () => {
+ const payload = buildStartConversationRequest({
+ settings: DEFAULT_SETTINGS,
+ agentProfileId: "profile-openhands",
+ agentProfileKind: "openhands",
+ });
+
+ expect(payload.client_tools.map((tool) => tool.name)).toEqual([
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ ]);
+ });
+
+ it("sends the client tool when resuming a conversation", () => {
+ const payload = buildStartConversationRequest({
+ settings: DEFAULT_SETTINGS,
+ conversationId: "legacy-conversation-id",
+ }) as { conversation_id: string; client_tools: Array<{ name: string }> };
+
+ expect(payload.conversation_id).toBe("legacy-conversation-id");
+ expect(payload.client_tools.map((tool) => tool.name)).toEqual([
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ ]);
+ });
+ it("drops conflicting user-supplied Canvas module qualnames", () => {
const payload = buildStartConversationRequest({
settings: {
...DEFAULT_SETTINGS,
@@ -636,6 +680,7 @@ describe("buildStartConversationRequest", () => {
...DEFAULT_SETTINGS.conversation_settings,
tool_module_qualnames: {
canvas_ui: "custom_canvas_ui_tool",
+ [CANVAS_UI_CLIENT_TOOL_NAME]: "custom_canvas_ui_control_tool",
my_tool: "my_package.my_tool",
},
},
@@ -647,7 +692,7 @@ describe("buildStartConversationRequest", () => {
});
});
- it("merges user-supplied tool_module_qualnames alongside canvas_ui_tool without dropping either side", () => {
+ it("preserves non-Canvas custom tool modules", () => {
const payload = buildStartConversationRequest({
settings: {
...DEFAULT_SETTINGS,
@@ -659,7 +704,6 @@ describe("buildStartConversationRequest", () => {
}) as { tool_module_qualnames: Record<string, string> };
expect(payload.tool_module_qualnames).toEqual({
- canvas_ui: "canvas_ui_tool",
my_tool: "my_package.my_tool",
});
});
diff --git a/__tests__/components/conversation-events/get-event-content.test.tsx b/__tests__/components/conversation-events/get-event-content.test.tsx
index f2b9c9472b..44eefa10ae 100644
--- a/__tests__/components/conversation-events/get-event-content.test.tsx
+++ b/__tests__/components/conversation-events/get-event-content.test.tsx
@@ -1,7 +1,15 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
+import {
+ CANVAS_UI_CLIENT_ACTION_KIND,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+} from "#/constants/canvas-ui";
import { getEventContent } from "#/components/conversation-events/chat";
-import { ActionEvent, ObservationEvent, SecurityRisk } from "#/types/agent-server/core";
+import {
+ ActionEvent,
+ ObservationEvent,
+ SecurityRisk,
+} from "#/types/agent-server/core";
const terminalActionEvent: ActionEvent = {
id: "action-1",
@@ -326,4 +334,62 @@ describe("getEventContent", () => {
"UI command 'open_tab' dispatched to the Agent Canvas frontend.",
);
});
+
+ it("renders client-defined Canvas UI events like legacy events", () => {
+ const canvasUIAction: ActionEvent = {
+ id: "action-canvas-client",
+ timestamp: new Date().toISOString(),
+ source: "agent",
+ thought: [],
+ thinking_blocks: [],
+ action: {
+ kind: CANVAS_UI_CLIENT_ACTION_KIND,
+ command: "open_tab",
+ path: null,
+ tab: "files",
+ },
+ tool_name: CANVAS_UI_CLIENT_TOOL_NAME,
+ tool_call_id: "tool-canvas-client",
+ tool_call: {
+ id: "tool-canvas-client",
+ type: "function",
+ function: {
+ name: CANVAS_UI_CLIENT_TOOL_NAME,
+ arguments: '{"command":"open_tab","tab":"files"}',
+ },
+ },
+ llm_response_id: "response-canvas-client",
+ security_risk: SecurityRisk.LOW,
+ summary: "",
+ };
+ const canvasUIObservation: ObservationEvent = {
+ id: "obs-canvas-client",
+ timestamp: new Date().toISOString(),
+ source: "environment",
+ tool_name: CANVAS_UI_CLIENT_TOOL_NAME,
+ tool_call_id: "tool-canvas-client",
+ action_id: "action-canvas-client",
+ observation: {
+ kind: "ClientToolObservation",
+ content: [{ type: "text", text: "Tool call dispatched to client." }],
+ is_error: false,
+ },
+ };
+
+ const actionContent = getEventContent(canvasUIAction);
+ render(<span>{actionContent.title}</span>);
+ expect(screen.getByText("CANVASUI")).toBeInTheDocument();
+
+ const { title, details } = getEventContent(
+ canvasUIObservation,
+ canvasUIAction,
+ );
+ render(<span>{title}</span>);
+ expect(
+ screen.getByText("OBSERVATION_MESSAGE$CANVAS_UI"),
+ ).toBeInTheDocument();
+ expect(details).toBe(
+ "UI command 'open_tab' dispatched to the Agent Canvas frontend.",
+ );
+ });
});
diff --git a/__tests__/hooks/mutation/use-create-conversation.test.tsx b/__tests__/hooks/mutation/use-create-conversation.test.tsx
index 47db81bb56..0b4e2cb9a4 100644
--- a/__tests__/hooks/mutation/use-create-conversation.test.tsx
+++ b/__tests__/hooks/mutation/use-create-conversation.test.tsx
@@ -382,8 +382,8 @@ describe("useCreateConversation", () => {
// The active profile IS the well-known default β†’ it's the enriched baseline
// (mirrors agent_settings), not a deliberate profile pick, so the launch
// stays on the agent_settings path (no profile tail) even though its
- // llm_profile_ref resolves. Keeps <RUNTIME_SERVICES>/canvas_ui/project
- // skills, which the profile-resolution path drops.
+ // llm_profile_ref resolves. Keeps <RUNTIME_SERVICES> and project skills,
+ // which the profile-resolution path drops.
listAgentProfilesMock.mockResolvedValue({
profiles: [
{
@@ -461,6 +461,7 @@ describe("useCreateConversation", () => {
const call = createConversationSpy.mock.lastCall;
expect(call?.[9]).toBe("profile-acp-default");
+ expect(call?.[10]).toBe("acp");
});
it("launches the seeded `default` profile from its resolved id on cloud (no agent_settings fallback exists there) (#1571)", async () => {
diff --git a/__tests__/services/canvas-ui.test.ts b/__tests__/services/canvas-ui.test.ts
index 71b084fe64..9eb1193fb6 100644
--- a/__tests__/services/canvas-ui.test.ts
+++ b/__tests__/services/canvas-ui.test.ts
@@ -1,5 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ CANVAS_UI_CLIENT_ACTION_KIND,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ LEGACY_CANVAS_UI_TOOL_NAME,
+} from "#/constants/canvas-ui";
import { handleCanvasUIAction } from "#/services/canvas-ui";
import { useConversationStore } from "#/stores/conversation-store";
import { useFilesTabStore } from "#/stores/files-tab-store";
@@ -98,14 +103,24 @@ describe("isCanvasUIActionEvent", () => {
timestamp: "2026-05-13T00:00:00Z",
source: "agent",
action: { kind: "CanvasUIAction" },
- tool_name: "canvas_ui",
+ tool_name: LEGACY_CANVAS_UI_TOOL_NAME,
tool_call_id: "call-1",
...overrides,
};
}
- it("returns true for an ActionEvent whose tool_name is canvas_ui", () => {
- expect(isCanvasUIActionEvent(makeActionEvent() as never)).toBe(true);
+ it.each([
+ ["CanvasUIAction", LEGACY_CANVAS_UI_TOOL_NAME],
+ [CANVAS_UI_CLIENT_ACTION_KIND, CANVAS_UI_CLIENT_TOOL_NAME],
+ ])("returns true for a %s ActionEvent from %s", (kind, toolName) => {
+ expect(
+ isCanvasUIActionEvent(
+ makeActionEvent({
+ action: { kind, command: "open_tab" },
+ tool_name: toolName,
+ }) as never,
+ ),
+ ).toBe(true);
});
it("returns false when tool_name belongs to a different tool", () => {
@@ -122,7 +137,7 @@ describe("isCanvasUIActionEvent", () => {
timestamp: "2026-05-13T00:00:00Z",
source: "environment",
observation: { kind: "ExecuteBashObservation" },
- tool_name: "canvas_ui",
+ tool_name: LEGACY_CANVAS_UI_TOOL_NAME,
tool_call_id: "call-1",
};
diff --git a/__tests__/tools/canvas-ui-tool.test.ts b/__tests__/tools/canvas-ui-tool.test.ts
index e4c9e146a5..acae157204 100644
--- a/__tests__/tools/canvas-ui-tool.test.ts
+++ b/__tests__/tools/canvas-ui-tool.test.ts
@@ -3,18 +3,44 @@ import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
+import {
+ CANVAS_UI_CLIENT_ACTION_KIND,
+ CANVAS_UI_CLIENT_TOOL,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ LEGACY_CANVAS_UI_TOOL_NAME,
+} from "#/api/canvas-ui-client-tool";
+
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
-const toolSource = readFileSync(resolve(repoRoot, "tools/canvas_ui_tool.py"), "utf8");
+const legacyToolSource = readFileSync(
+ resolve(repoRoot, "tools/canvas_ui_tool.py"),
+ "utf8",
+);
-describe("canvas_ui browser guidance", () => {
+describe("canvas_ui client tool", () => {
it("tells the agent to capture a browser screenshot before opening the browser tab", () => {
const captureInstruction = "browser_get_state(include_screenshot=true)";
const openBrowserInstruction = 'command="open_tab", tab="browser"';
- expect(toolSource).toContain(captureInstruction);
- expect(toolSource).toContain(openBrowserInstruction);
- expect(toolSource.indexOf(captureInstruction)).toBeLessThan(
- toolSource.indexOf(openBrowserInstruction),
+ expect(CANVAS_UI_CLIENT_TOOL.description).toContain(captureInstruction);
+ expect(CANVAS_UI_CLIENT_TOOL.description).toContain(openBrowserInstruction);
+ expect(
+ CANVAS_UI_CLIENT_TOOL.description.indexOf(captureInstruction),
+ ).toBeLessThan(
+ CANVAS_UI_CLIENT_TOOL.description.indexOf(openBrowserInstruction),
+ );
+ });
+
+ it("exports the semantic tool name and generated action kind", () => {
+ expect(CANVAS_UI_CLIENT_TOOL_NAME).toBe("canvas_ui_control");
+ expect(CANVAS_UI_CLIENT_ACTION_KIND).toBe("ClientAction_canvas_ui_control");
+ expect(CANVAS_UI_CLIENT_TOOL.name).toBe(CANVAS_UI_CLIENT_TOOL_NAME);
+ expect(CANVAS_UI_CLIENT_TOOL.name).not.toBe(LEGACY_CANVAS_UI_TOOL_NAME);
+ });
+
+ it("retains the Python registration shim for persisted conversations", () => {
+ expect(legacyToolSource).toContain("Legacy conversation compatibility");
+ expect(legacyToolSource).toContain(
+ 'register_tool("canvas_ui", CanvasUITool)',
);
});
});
diff --git a/docker/Dockerfile b/docker/Dockerfile
index bcfbed22b2..4535f23d0c 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -126,8 +126,9 @@ COPY --from=frontend-build /build/node_modules/totalist /opt/agent-canvas/node_m
# emit the agent's <RUNTIME_SERVICES> block; same builder the dev stack uses).
COPY scripts/runtime-services-info.mjs /opt/agent-canvas/runtime-services-info.mjs
-# Copy custom tools (e.g. canvas_ui_tool.py) so the agent-server can import
-# them via tool_module_qualnames. OH_EXTRA_PYTHON_PATH is set in entrypoint.sh.
+# Persisted conversations created before the client_tools migration still
+# import canvas_ui_tool by qualname. Keep the compatibility module available;
+# new conversations define canvas_ui_control through a JSON client tool.
COPY tools/ /opt/agent-canvas/tools/
# Copy generated defaults.env (from config/defaults.json via config-gen stage)
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 27f9b22ad6..1e9d303807 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -124,9 +124,8 @@ export AGENT_SERVER_URL="${AGENT_SERVER_URL:-http://127.0.0.1:${AGENT_SERVER_POR
# for locally-generated session keys.
export AUTOMATION_AGENT_SERVER_URL="${AUTOMATION_AGENT_SERVER_URL:-http://127.0.0.1:${AGENT_SERVER_PORT}}"
-# Make custom tools (e.g. canvas_ui_tool.py) importable by the agent-server
-# via tool_module_qualnames. Matches what scripts/dev-safe.mjs does with
-# OH_EXTRA_PYTHON_PATH: config.canvasToolsDir.
+# Keep the legacy canvas_ui_tool module importable when the agent-server restores
+# conversations whose persisted metadata still references its module qualname.
export OH_EXTRA_PYTHON_PATH="${OH_EXTRA_PYTHON_PATH:-/opt/agent-canvas/tools}"
# Track child PIDs so we can clean up on exit.
diff --git a/docs/ACP_AGENTS.md b/docs/ACP_AGENTS.md
index ab2be745e6..2e61a1ed52 100644
--- a/docs/ACP_AGENTS.md
+++ b/docs/ACP_AGENTS.md
@@ -144,9 +144,10 @@ point Canvas at it). In short:
```bash
# 1. Agent Server in a container (CORS allows localhost, so the browser talks
-# to it directly). The image pre-installs the ACP CLI wrappers. The
-# canvas_ui tool is mounted so the agent-server can import the module Canvas
-# references in every start request.
+# to it directly). The image pre-installs the ACP CLI wrappers. New
+# canvas_ui_control calls use client_tools; the Python mount keeps
+# pre-migration conversations loadable when persisted metadata imports
+# canvas_ui_tool.
# Minimum image: 1.28.0-python (first compatible ACP provider/model protocol
# surface for current Canvas). Override SHA with a newer build.
docker run -d --name oh-acp -p 8010:8000 -v acp-data:/workspace \
diff --git a/examples/acp-docker/docker-compose.yml b/examples/acp-docker/docker-compose.yml
index 8e14a7d2b6..1a653c1c5f 100644
--- a/examples/acp-docker/docker-compose.yml
+++ b/examples/acp-docker/docker-compose.yml
@@ -30,10 +30,9 @@ services:
# host:container β€” Canvas points VITE_BACKEND_BASE_URL at http://localhost:8010.
- "8010:8000"
environment:
- # Canvas's start request always references the bundled ``canvas_ui`` tool
- # via ``tool_module_qualnames``; the agent-server imports that module from
- # OH_EXTRA_PYTHON_PATH at conversation creation. Without the mount below +
- # this var, conversation creation fails to import ``canvas_ui_tool``.
+ # New conversations define canvas_ui_control through client_tools. Keep the
+ # old Python module importable so conversations persisted before that migration
+ # can restore CanvasUIAction / CanvasUIObservation events after a restart.
- OH_EXTRA_PYTHON_PATH=/canvas-tools
# Optional cipher key. ACP conversations work without it (Canvas sends ACP
# provider credentials as loopback LookupSecrets resolved from the
@@ -66,8 +65,8 @@ services:
# Persist conversations AND the credential files the SDK materialises
# (Codex auth.json under CODEX_HOME, Gemini ADC/SA JSON) across restarts.
- acp-data:/workspace
- # Canvas-specific Python tools (the canvas_ui tool) the agent-server loads
- # via OH_EXTRA_PYTHON_PATH. Path is relative to this compose file.
+ # Legacy canvas_ui module used only when restoring pre-client_tools state.
+ # Path is relative to this compose file.
- ../../tools:/canvas-tools:ro
restart: unless-stopped
diff --git a/scripts/dev-safe.mjs b/scripts/dev-safe.mjs
index 71378eb82e..59247d86e8 100644
--- a/scripts/dev-safe.mjs
+++ b/scripts/dev-safe.mjs
@@ -636,10 +636,9 @@ function buildConfigFromPorts(ports, cwd, env) {
env.LOCAL_BACKEND_API_KEY ||
getOrCreatePersistedApiKeyFile(persistedKeyPath);
- // Host directory containing Agent-Canvas-specific Python tools (e.g. the
- // canvas_ui tool). Added to OH_EXTRA_PYTHON_PATH below so the agent-server
- // can import the modules listed in `tool_module_qualnames`. Lives at
- // <repo-root>/tools relative to this script.
+ // Host directory containing the legacy canvas_ui Python module. Persisted
+ // conversations created before the client_tools migration still reference
+ // its module qualname, so the agent-server can import it when resuming them.
const canvasToolsDir = fileURLToPath(new URL("../tools", import.meta.url));
return {
@@ -719,8 +718,8 @@ export function buildAgentServerEnv(config) {
// a follow-up change to the automation preset reads
// OH_SESSION_API_KEYS_0 directly (which is already in env).
AGENT_SERVER_URL: config.backendBaseUrl,
- // Make the host tools/ directory importable so the agent-server can
- // resolve modules listed in tool_module_qualnames (e.g. canvas_ui_tool).
+ // Let the agent-server resolve canvas_ui_tool when old persisted metadata
+ // requests that compatibility module during startup.
OH_EXTRA_PYTHON_PATH: config.canvasToolsDir,
};
}
diff --git a/src/api/agent-server-adapter.test.ts b/src/api/agent-server-adapter.test.ts
index 7248b9058d..05a549b3db 100644
--- a/src/api/agent-server-adapter.test.ts
+++ b/src/api/agent-server-adapter.test.ts
@@ -1,4 +1,5 @@
import { describe, expect, it } from "vitest";
+import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui";
import { DEFAULT_SETTINGS } from "#/services/settings";
import type { Settings } from "#/types/settings";
import { buildStartConversationRequest } from "./agent-server-adapter";
@@ -120,10 +121,14 @@ describe("buildStartConversationRequest β€” agentProfileId path", () => {
const payload = buildStartConversationRequest({
settings,
agentProfileId: "profile-xyz",
+ agentProfileKind: "openhands",
});
expect(payload.agent_profile_id).toBe("profile-xyz");
expect(payload.agent_settings).toBeUndefined();
+ expect(payload.client_tools.map((tool) => tool.name)).toEqual([
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ ]);
});
it("suppresses the ACP server tag when launching from a profile", () => {
diff --git a/src/api/agent-server-adapter.ts b/src/api/agent-server-adapter.ts
index ed07c28a56..38cb1e15d8 100644
--- a/src/api/agent-server-adapter.ts
+++ b/src/api/agent-server-adapter.ts
@@ -2,7 +2,7 @@ import { ACP_SETTINGS_KEYS } from "@openhands/typescript-client";
import { SKILLS_CATALOG } from "@openhands/extensions/skills";
import { DEFAULT_SETTINGS } from "#/services/settings";
import { ExecutionStatus } from "#/types/agent-server/core";
-import { Settings, SettingsValue } from "#/types/settings";
+import { AgentKind, Settings, SettingsValue } from "#/types/settings";
import {
getAcpPreferredDefaultModel,
getAcpProvider,
@@ -28,6 +28,12 @@ import {
OPENAI_SUBSCRIPTION_VENDOR,
isSubscriptionLlmConfig,
} from "#/constants/llm-subscription";
+import {
+ CANVAS_UI_CLIENT_TOOL,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ LEGACY_CANVAS_UI_TOOL_NAME,
+ type ClientToolSpec,
+} from "./canvas-ui-client-tool";
export interface DirectConversationInfo {
id: string;
@@ -88,18 +94,7 @@ export interface DirectConversationInfo {
} | null;
}
-// Module qualname for the Canvas-UI tool. The agent-server imports this via
-// tool_module_qualnames; the host directory is exposed via OH_EXTRA_PYTHON_PATH
-// (see scripts/dev-safe.mjs).
-const CANVAS_UI_TOOL_NAME = "canvas_ui";
-const CANVAS_UI_TOOL_MODULE = "canvas_ui_tool";
-
-const DEFAULT_TOOL_NAMES = [
- "terminal",
- "file_editor",
- "task_tracker",
- CANVAS_UI_TOOL_NAME,
-];
+const DEFAULT_TOOL_NAMES = ["terminal", "file_editor", "task_tracker"];
const BROWSER_TOOL_SET_NAME = "browser_tool_set";
const TASK_TOOL_SET_NAME = "task_tool_set";
@@ -518,10 +513,6 @@ function isToolRecord(
}
function shouldIncludeTool(name: string, agentSettings: SettingsRecord) {
- if (name === CANVAS_UI_TOOL_NAME) {
- return isAgentServerToolAvailable(name);
- }
-
if (name === BROWSER_TOOL_SET_NAME) {
return browserToolsEnabled() && isAgentServerToolAvailable(name);
}
@@ -889,6 +880,7 @@ type StartConversationPayload = Record<string, unknown> & {
conversation_id?: string;
secrets?: Record<string, LookupSecret>;
tags?: Record<string, string>;
+ client_tools: ClientToolSpec[];
tool_module_qualnames?: Record<string, string>;
};
@@ -907,6 +899,7 @@ export interface StartConversationOptions {
// When set, the conversation launches from this AgentProfile (resolved
// server-side) instead of an inline ``agent_settings`` dump (#3727).
agentProfileId?: string;
+ agentProfileKind?: AgentKind;
}
export function buildStartConversationRequest(
@@ -917,6 +910,11 @@ export function buildStartConversationRequest(
: options.settings;
const acpMode = isAcpAgent(sourceAgentSettings);
+ const launchAgentKind = options.agentProfileId
+ ? options.agentProfileKind
+ : acpMode
+ ? "acp"
+ : "openhands";
const agentSettings = buildConfiguredAgentSettings(sourceAgentSettings);
const acpServerTag = acpMode
? getAcpServerTag(sourceAgentSettings)
@@ -947,14 +945,15 @@ export function buildStartConversationRequest(
// server/SDK's responsibility to restore on the profile path β€” tracked in
// software-agent-sdk#3967 (profile resolution must attach the default
// toolset + public skills, else a profile-launched OpenHands agent has only
- // Finish/Think). The two genuinely canvas-only enrichments β€” the
- // ``canvas_ui`` tool and the dev ``RUNTIME_SERVICES`` system-message-suffix
- // (``buildAgentContext``) β€” have no server-side representation and are
- // intentionally not carried on the profile path.
+ // Finish/Think). The dev ``RUNTIME_SERVICES`` system-message suffix remains
+ // agent-settings-only; the Canvas UI tool is a top-level client tool and
+ // therefore works on both inline-agent and profile launch paths.
...(options.agentProfileId
? { agent_profile_id: options.agentProfileId }
: { agent_settings: agentSettings }),
workspace: conversationSettings.workspace,
+ client_tools:
+ launchAgentKind === "openhands" ? [CANVAS_UI_CLIENT_TOOL] : [],
confirmation_policy:
getConversationConfirmationPolicy(conversationSettings),
max_iterations:
@@ -1008,20 +1007,13 @@ export function buildStartConversationRequest(
payload.hook_config = conversationSettings.hook_config;
}
- const toolModuleQualnames: Record<string, string> = {};
- const canvasUiAvailable = isAgentServerToolAvailable(CANVAS_UI_TOOL_NAME);
- if (canvasUiAvailable) {
- toolModuleQualnames[CANVAS_UI_TOOL_NAME] = CANVAS_UI_TOOL_MODULE;
- }
- Object.assign(
- toolModuleQualnames,
- (conversationSettings.tool_module_qualnames as
+ const toolModuleQualnames = {
+ ...((conversationSettings.tool_module_qualnames as
| Record<string, string>
- | undefined) ?? {},
- );
- if (!canvasUiAvailable) {
- delete toolModuleQualnames[CANVAS_UI_TOOL_NAME];
- }
+ | undefined) ?? {}),
+ };
+ delete toolModuleQualnames[LEGACY_CANVAS_UI_TOOL_NAME];
+ delete toolModuleQualnames[CANVAS_UI_CLIENT_TOOL_NAME];
if (Object.keys(toolModuleQualnames).length > 0) {
payload.tool_module_qualnames = toolModuleQualnames;
}
@@ -1090,6 +1082,7 @@ export async function buildStartConversationRequestWithEncryptedSettings(options
workingDir?: string;
worktree?: boolean;
agentProfileId?: string;
+ agentProfileKind?: AgentKind;
}): Promise<Record<string, unknown>> {
const { SecretsService } = await import("./secrets-service");
diff --git a/src/api/canvas-ui-client-tool.ts b/src/api/canvas-ui-client-tool.ts
new file mode 100644
index 0000000000..168c83eab4
--- /dev/null
+++ b/src/api/canvas-ui-client-tool.ts
@@ -0,0 +1,91 @@
+import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui";
+
+export {
+ CANVAS_UI_CLIENT_ACTION_KIND,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ LEGACY_CANVAS_UI_TOOL_NAME,
+} from "#/constants/canvas-ui";
+
+export interface ClientToolSpec {
+ name: string;
+ description: string;
+ parameters: Record<string, unknown>;
+ annotations?: {
+ title?: string | null;
+ readOnlyHint: boolean;
+ destructiveHint: boolean;
+ idempotentHint: boolean;
+ openWorldHint: boolean;
+ };
+}
+
+const CANVAS_UI_DESCRIPTION = `The user is interacting with you inside Agent Canvas β€” a web UI with a chat panel on the left and a tabbed right-side panel (files, terminal, browser, vscode, planner, tasklist). This tool lets you drive that right-side panel so the user sees what you just produced.
+
+They will NOT see the files you wrote, the terminal output, or the browser
+unless you call this tool to switch the right-side panel to the relevant
+tab. Call this every time you finish work that produces something the user
+should look at β€” don't rely on them noticing on their own.
+
+When to call (pick the most specific option that matches your last action):
+
+* You wrote or modified a single file (ANY language, ANY size β€” including
+ small scripts like a hello-world bash file) β†’
+ command="navigate_to_file", path=<workspace-relative path of that file>
+
+* You generated an HTML page, image, SVG, PDF, markdown report, or other
+ previewable artifact β†’
+ command="show_preview", path=<that file>
+
+* You finished editing multiple files in one logical step β†’
+ command="open_tab", tab="files"
+ (The Files tab automatically renders a diff view when the workspace has
+ uncommitted git changes, which covers the "highlight changes" case.)
+
+* You ran a long-running terminal command, or one whose output the user
+ should inspect β†’
+ command="open_tab", tab="terminal"
+
+* You browsed to a URL the user should see β†’
+ First call browser_get_state(include_screenshot=true) after your final
+ browser interaction so Agent Canvas has a screenshot to display, then call
+ command="open_tab", tab="browser"
+ (browser_navigate alone only updates the URL; without browser_get_state,
+ the Browser tab will open without a screenshot.)
+
+Call this BEFORE writing your chat-message summary of the change, so the
+artifact is visible while the user reads what you did. One canvas_ui_control
+call per logical step is enough β€” don't repeat it for the same file or tab in
+the same turn.`;
+
+export const CANVAS_UI_CLIENT_TOOL: ClientToolSpec = {
+ name: CANVAS_UI_CLIENT_TOOL_NAME,
+ description: CANVAS_UI_DESCRIPTION,
+ parameters: {
+ type: "object",
+ additionalProperties: false,
+ properties: {
+ command: {
+ type: "string",
+ enum: ["navigate_to_file", "open_tab", "show_preview"],
+ description: "UI command to dispatch.",
+ },
+ path: {
+ type: "string",
+ description:
+ "Workspace-relative file path. Required for navigate_to_file and show_preview; ignored otherwise.",
+ },
+ tab: {
+ type: "string",
+ enum: ["files", "browser", "vscode", "terminal", "planner", "tasklist"],
+ description: "Tab to open. Required for open_tab; ignored otherwise.",
+ },
+ },
+ required: ["command"],
+ },
+ annotations: {
+ readOnlyHint: true,
+ destructiveHint: false,
+ idempotentHint: true,
+ openWorldHint: false,
+ },
+};
diff --git a/src/api/conversation-service/agent-server-conversation-service.api.ts b/src/api/conversation-service/agent-server-conversation-service.api.ts
index f330a8bcff..fe7d6c991a 100644
--- a/src/api/conversation-service/agent-server-conversation-service.api.ts
+++ b/src/api/conversation-service/agent-server-conversation-service.api.ts
@@ -10,7 +10,7 @@ import {
VSCodeClient,
} from "@openhands/typescript-client/clients";
import { v4 as uuidv4 } from "uuid";
-import { Provider } from "#/types/settings";
+import { AgentKind, Provider } from "#/types/settings";
import type { ConversationRuntimeContext } from "#/api/conversation-file-upload.api";
import { buildHttpBaseUrl } from "#/utils/websocket-url";
import {
@@ -376,6 +376,7 @@ class AgentServerConversationService {
// cloud app-server (OpenHands #15060): local threads it through the
// encrypted-settings builder; cloud sends it as a flat request field.
agentProfileId?: string,
+ agentProfileKind?: AgentKind,
): Promise<AppConversationStartTask> {
if (getActiveBackend().backend.kind === "cloud") {
// Cloud path mirrors OpenHands' frontend: build a flat
@@ -429,6 +430,7 @@ class AgentServerConversationService {
workingDir,
worktree: resolvedWorkspaceMode === "new_worktree",
agentProfileId,
+ agentProfileKind,
});
const data = await new ConversationClient(
diff --git a/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx b/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx
index abc66215bf..e5f96f2c1d 100644
--- a/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx
+++ b/src/components/conversation-events/chat/event-content-helpers/get-event-content.tsx
@@ -1,5 +1,9 @@
import { Trans } from "react-i18next";
import React from "react";
+import {
+ CANVAS_UI_CLIENT_ACTION_KIND,
+ CANVAS_UI_CLIENT_TOOL_NAME,
+} from "#/constants/canvas-ui";
import {
OpenHandsEvent,
ObservationEvent,
@@ -9,6 +13,7 @@ import {
isActionEvent,
isObservationEvent,
isACPToolCallEvent,
+ isCanvasUIActionEvent,
} from "#/types/agent-server/type-guards";
import { MonoComponent } from "../../../features/chat/mono-component";
import { PathComponent } from "../../../features/chat/path-component";
@@ -236,6 +241,9 @@ const getActionEventTitle = (event: OpenHandsEvent): React.ReactNode => {
case "BrowserCloseTabAction":
actionKey = "ACTION_MESSAGE$BROWSE";
break;
+ case "CanvasUIAction":
+ case CANVAS_UI_CLIENT_ACTION_KIND:
+ return "CANVASUI";
default:
// For unknown actions, use the type name
return String(actionType).replace("Action", "").toUpperCase();
@@ -324,6 +332,12 @@ const getObservationEventTitle = (
case "CanvasUIObservation":
observationKey = "OBSERVATION_MESSAGE$CANVAS_UI";
break;
+ case "ClientToolObservation":
+ if (event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME) {
+ observationKey = "OBSERVATION_MESSAGE$CANVAS_UI";
+ break;
+ }
+ return observationType.replace("Observation", "").toUpperCase();
case "SwitchLLMObservation":
observationKey = event.observation.is_error
? "MODEL$SWITCH_FAILED"
@@ -376,6 +390,22 @@ const getObservationEventTitle = (
return observationType;
};
+const getCanvasUIClientObservationContent = (
+ event: ObservationEvent,
+ correspondingAction?: ActionEvent,
+): string | null => {
+ if (
+ event.observation.kind !== "ClientToolObservation" ||
+ event.tool_name !== CANVAS_UI_CLIENT_TOOL_NAME ||
+ !correspondingAction ||
+ !isCanvasUIActionEvent(correspondingAction)
+ ) {
+ return null;
+ }
+
+ return `UI command '${correspondingAction.action.command}' dispatched to the Agent Canvas frontend.`;
+};
+
export const getEventContent = (
event: OpenHandsEvent | SkillReadyEvent,
correspondingAction?: ActionEvent,
@@ -409,6 +439,7 @@ export const getEventContent = (
);
} else {
details =
+ getCanvasUIClientObservationContent(event, correspondingAction) ??
resolveVisualizerBody(event, correspondingAction) ??
getObservationContent(event);
}
diff --git a/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts b/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts
index e209bb6642..45ec656e43 100644
--- a/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts
+++ b/src/components/conversation-events/chat/event-content-helpers/get-observation-content.ts
@@ -1,3 +1,4 @@
+import { CANVAS_UI_CLIENT_TOOL_NAME } from "#/constants/canvas-ui";
import { ObservationEvent } from "#/types/agent-server/core";
import { getObservationResult } from "./get-observation-result";
import { getDefaultEventContent, MAX_CONTENT_LENGTH } from "./shared";
@@ -417,6 +418,13 @@ export const getObservationContent = (event: ObservationEvent): string => {
event as ObservationEvent<CanvasUIObservation>,
);
+ case "ClientToolObservation":
+ return event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME
+ ? getCanvasUIObservationContent(
+ event as ObservationEvent<CanvasUIObservation>,
+ )
+ : getDefaultEventContent(event);
+
case "SwitchLLMObservation":
return getSwitchLLMObservationContent(
event as ObservationEvent<SwitchLLMObservation>,
diff --git a/src/constants/canvas-ui.ts b/src/constants/canvas-ui.ts
new file mode 100644
index 0000000000..4981cef02a
--- /dev/null
+++ b/src/constants/canvas-ui.ts
@@ -0,0 +1,10 @@
+export const LEGACY_CANVAS_UI_TOOL_NAME = "canvas_ui";
+export const CANVAS_UI_CLIENT_TOOL_NAME = "canvas_ui_control";
+
+/**
+ * Action discriminator generated by the SDK for the client-defined Canvas UI
+ * tool (`ClientAction_<tool-name>`). Keep the SDK naming convention contained
+ * here instead of repeating its generated class name throughout the frontend.
+ */
+export const CANVAS_UI_CLIENT_ACTION_KIND =
+ `ClientAction_${CANVAS_UI_CLIENT_TOOL_NAME}` as const;
diff --git a/src/contexts/conversation-websocket-context.tsx b/src/contexts/conversation-websocket-context.tsx
index 44da40624f..14940e9354 100644
--- a/src/contexts/conversation-websocket-context.tsx
+++ b/src/contexts/conversation-websocket-context.tsx
@@ -637,10 +637,9 @@ export function ConversationWebSocketProvider({
invalidateConversationQueries(queryClient, conversationId);
}
- // Handle canvas_ui custom-tool ActionEvents - drive the frontend
- // (navigate to a file, switch tabs, show a preview). The tool
- // executes server-side as a no-op; the actual UI change happens
- // here on the client.
+ // Handle canvas_ui ActionEvents from both the legacy Python tool and
+ // the client-defined JSON tool. The server acknowledges immediately;
+ // the actual UI change happens here on the client.
if (isCanvasUIActionEvent(event)) {
handleCanvasUIAction(event.action, conversationId ?? null);
}
diff --git a/src/hooks/mutation/use-create-conversation.ts b/src/hooks/mutation/use-create-conversation.ts
index 7632e4d37b..5296818d25 100644
--- a/src/hooks/mutation/use-create-conversation.ts
+++ b/src/hooks/mutation/use-create-conversation.ts
@@ -2,7 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";
import AgentServerConversationService from "#/api/conversation-service/agent-server-conversation-service.api";
import { PluginSpec } from "#/api/conversation-service/agent-server-conversation-service.types";
import { SuggestedTask } from "#/utils/types";
-import { Provider } from "#/types/settings";
+import { AgentKind, Provider } from "#/types/settings";
import { useTracking } from "#/hooks/use-tracking";
import { useLlmProfiles } from "#/hooks/query/use-llm-profiles";
import { useAgentProfiles } from "#/hooks/query/use-agent-profiles";
@@ -149,17 +149,17 @@ export const useCreateConversation = () => {
// deliberate profile pick β€” it mirrors global agent_settings. Launch it
// via agent_settings so the canvas-only enrichments the profile-resolution
// path drops survive for the common home-launch: the <RUNTIME_SERVICES>
- // system-message suffix, the canvas_ui tool, and project-skill loading
- // (buildAgentContext). Named profiles are deliberate custom configs and
- // still use the profile path (accepting that enrichment boundary).
+ // system-message suffix and project-skill loading (buildAgentContext).
+ // Named profiles are deliberate custom configs and still use the profile
+ // path (accepting that enrichment boundary).
// Trade-off: per-profile fields set on `default` itself don't apply on
// home-launch β€” custom per-profile config belongs in a named profile.
//
// Scoped to OpenHands: an ACP `default` must keep the profile path.
// Activation is pointer-only, so global agent_settings is stale (often
// still OpenHands) when an ACP profile is active β€” launching it via
- // agent_settings would start the wrong agent. ACP also carries no
- // <RUNTIME_SERVICES>/canvas_ui enrichment, so there's nothing to preserve.
+ // agent_settings would start the wrong agent. ACP carries no
+ // <RUNTIME_SERVICES> enrichment, so there's nothing to preserve.
//
// Scoped to local: cloud never writes agent_settings, so it always
// resolves `default` server-side via agent_profile_id (validated below).
@@ -198,15 +198,20 @@ export const useCreateConversation = () => {
}
}
- // Only extend the call with the [sandboxId, agentProfileId] tail when
- // launching from a profile, so a plain create stays byte-identical to
- // the legacy agent_settings path (#3727). sandboxId is unused here.
- // TODO(#1587): createConversation has grown to 10 positional params;
+ // Only extend the call with the profile tail when launching from a
+ // profile, so a plain create stays byte-identical to the legacy
+ // agent_settings path (#3727). sandboxId is unused here.
+ // TODO(#1587): createConversation has grown to 11 positional params;
// refactor it to an options object so this position-skipping tail isn't
// needed.
- const profileArgs: [undefined, string] | [] = effectiveAgentProfileId
- ? [undefined, effectiveAgentProfileId]
- : [];
+ const profileArgs: [undefined, string, AgentKind | undefined] | [] =
+ effectiveAgentProfileId
+ ? [
+ undefined,
+ effectiveAgentProfileId,
+ resolvedAgentProfile?.agent_kind,
+ ]
+ : [];
const conversation =
await AgentServerConversationService.createConversation(
diff --git a/src/types/agent-server/core/base/action.ts b/src/types/agent-server/core/base/action.ts
index 2595c1cb64..a316bf584c 100644
--- a/src/types/agent-server/core/base/action.ts
+++ b/src/types/agent-server/core/base/action.ts
@@ -1,3 +1,4 @@
+import { CANVAS_UI_CLIENT_ACTION_KIND } from "#/constants/canvas-ui";
import { ActionBase } from "./base";
import { TaskItem } from "./common";
@@ -308,11 +309,13 @@ export interface SwitchLLMAction extends ActionBase<"SwitchLLMAction"> {
}
/**
- * Frontend-injected custom tool. Emitted over the existing WebSocket as a
- * regular ActionEvent; intercepted client-side by handleCanvasUIAction.
- * The Python definition lives in tools/canvas_ui_tool.py.
+ * Canvas UI action emitted over the existing WebSocket and intercepted by
+ * handleCanvasUIAction. Legacy conversations use CanvasUIAction; new
+ * conversations use the SDK-generated client action kind.
*/
-export interface CanvasUIAction extends ActionBase<"CanvasUIAction"> {
+export interface CanvasUIAction extends ActionBase<
+ "CanvasUIAction" | typeof CANVAS_UI_CLIENT_ACTION_KIND
+> {
command: "navigate_to_file" | "open_tab" | "show_preview";
path?: string | null;
tab?: string | null;
diff --git a/src/types/agent-server/core/base/base.ts b/src/types/agent-server/core/base/base.ts
index 2f1700a58e..891e105851 100644
--- a/src/types/agent-server/core/base/base.ts
+++ b/src/types/agent-server/core/base/base.ts
@@ -1,3 +1,5 @@
+import { CANVAS_UI_CLIENT_ACTION_KIND } from "#/constants/canvas-ui";
+
type EventType =
| "MCPTool"
| "Finish"
@@ -22,9 +24,8 @@ type ActionOnlyType =
| "BrowserListTabs"
| "BrowserSwitchTab"
| "BrowserCloseTab"
- // Frontend-injected custom tool. Not part of the upstream SDK Action
- // union but emitted as a regular ActionEvent over the WebSocket. See
- // tools/canvas_ui_tool.py and src/services/canvas-ui.ts.
+ // Legacy Python-defined Canvas tool kind. The client-tool kind is added
+ // separately below because the SDK generates its full discriminator.
| "CanvasUI";
type ObservationOnlyType = "Browser";
@@ -35,7 +36,8 @@ type ActionEventType =
| "GlobAction"
| "GrepAction"
// The `task` tool delegating work to a spawned subagent.
- | "TaskAction";
+ | "TaskAction"
+ | typeof CANVAS_UI_CLIENT_ACTION_KIND;
type ObservationEventType =
| `${ObservationOnlyType}Observation`
| `${EventType}Observation`
@@ -44,8 +46,9 @@ type ObservationEventType =
| "GrepObservation"
// Result of the `task` tool, which delegates work to a spawned subagent.
| "TaskObservation"
- // Acknowledgement emitted after a `canvas_ui` command is dispatched.
- | "CanvasUIObservation";
+ // Legacy and client-defined acknowledgements for Canvas UI dispatches.
+ | "CanvasUIObservation"
+ | "ClientToolObservation";
export interface ActionBase<T extends ActionEventType = ActionEventType> {
kind: T;
diff --git a/src/types/agent-server/core/base/observation.ts b/src/types/agent-server/core/base/observation.ts
index 6fc6d5127e..9a6fd719aa 100644
--- a/src/types/agent-server/core/base/observation.ts
+++ b/src/types/agent-server/core/base/observation.ts
@@ -325,7 +325,9 @@ export interface TaskObservation extends ObservationBase<"TaskObservation"> {
status: string;
}
-export interface CanvasUIObservation extends ObservationBase<"CanvasUIObservation"> {
+export interface CanvasUIObservation extends ObservationBase<
+ "CanvasUIObservation" | "ClientToolObservation"
+> {
/**
* Acknowledgement text returned after the canvas UI command is dispatched.
*/
diff --git a/src/types/agent-server/type-guards.ts b/src/types/agent-server/type-guards.ts
index 485ba2358f..542a9da400 100644
--- a/src/types/agent-server/type-guards.ts
+++ b/src/types/agent-server/type-guards.ts
@@ -1,3 +1,8 @@
+import {
+ CANVAS_UI_CLIENT_TOOL_NAME,
+ LEGACY_CANVAS_UI_TOOL_NAME,
+} from "#/constants/canvas-ui";
+
import {
OpenHandsEvent,
ObservationEvent,
@@ -167,17 +172,17 @@ export const isBrowserNavigateActionEvent = (
isActionEvent(event) && event.action.kind === "BrowserNavigateAction";
/**
- * Type guard for the canvas_ui custom tool's ActionEvent.
+ * Type guard for Canvas UI tool ActionEvents.
*
- * The tool is injected via tool_module_qualnames (see canvas_ui_tool.py and
- * agent-server-adapter.ts). We discriminate on tool_name (which we control
- * via register_tool("canvas_ui", ...)). The predicate narrows the event so
- * the call site can read `event.action.command` etc. without further casts.
+ * Discriminating on tool_name supports legacy CanvasUIAction events and the
+ * SDK-generated action kind without leaking that generated name here.
*/
export const isCanvasUIActionEvent = (
event: OpenHandsEvent,
): event is ActionEvent<CanvasUIAction> =>
- isActionEvent(event) && event.tool_name === "canvas_ui";
+ isActionEvent(event) &&
+ (event.tool_name === LEGACY_CANVAS_UI_TOOL_NAME ||
+ event.tool_name === CANVAS_UI_CLIENT_TOOL_NAME);
/**
* Type guard function to check if an event is a system prompt event
diff --git a/tests/e2e/live-acp/README.md b/tests/e2e/live-acp/README.md
index 68016aeadd..f4b6e02b0a 100644
--- a/tests/e2e/live-acp/README.md
+++ b/tests/e2e/live-acp/README.md
@@ -18,12 +18,12 @@ and needs a running container + real host credentials).
## Run it
```bash
-# 1. Agent-server container with the canvas_ui tool mounted (as the dev stack does).
-# Minimum 1.25.0-python (software-agent-sdk#3510); override for a newer build.
+# 1. Agent-server container. v1.28.0 adds the client_tools API used by Canvas.
+# The Python mount keeps pre-migration conversation state loadable.
docker run -d --name oh-acp -p 8010:8000 \
-v oh-acp-data:/workspace \
-v "$(pwd)/tools:/canvas-tools:ro" -e OH_EXTRA_PYTHON_PATH=/canvas-tools \
- ghcr.io/openhands/agent-server:1.25.0-python
+ ghcr.io/openhands/agent-server:1.28.0-python
# 2. Run the e2e (all providers, or a subset).
npx vite-node -c tests/e2e/live-acp/vite-node.config.mts \
diff --git a/tests/e2e/live-acp/acp-docker-e2e.mts b/tests/e2e/live-acp/acp-docker-e2e.mts
index f2858119e0..0ed722313d 100644
--- a/tests/e2e/live-acp/acp-docker-e2e.mts
+++ b/tests/e2e/live-acp/acp-docker-e2e.mts
@@ -10,17 +10,17 @@
* LookupSecrets Canvas emits resolve back from the store and authenticate the
* CLI end-to-end (including the SDK's acp_file_secrets materialisation).
*
- * Requires an agent-server with software-agent-sdk#3510 (first in v1.25.0): the
- * ACP credentials ride as loopback LookupSecrets, and only #3510 resolves them
- * off the event loop β€” an older image deadlocks ("Failed to start ACP server:
- * timed out").
+ * Requires agent-server v1.28.0: it includes both software-agent-sdk#3510 for
+ * off-loop LookupSecret resolution and the client_tools API used by
+ * canvas_ui_control. Older images either deadlock resolving ACP credentials or
+ * omit the Canvas UI tool.
*
* Excluded from `npm test` (lives under tests/). Run it by hand against a
* running container:
*
* docker run -d --name oh-acp -p 8010:8000 -v oh-acp-data:/workspace \
* -v "$(pwd)/tools:/canvas-tools:ro" -e OH_EXTRA_PYTHON_PATH=/canvas-tools \
- * ghcr.io/openhands/agent-server:1.25.0-python
+ * ghcr.io/openhands/agent-server:1.28.0-python
* npx vite-node -c tests/e2e/live-acp/vite-node.config.mts \
* tests/e2e/live-acp/acp-docker-e2e.mts -- codex claude gemini
*
diff --git a/tests/e2e/live/utils/agent-server-conversation.ts b/tests/e2e/live/utils/agent-server-conversation.ts
index 9001abe27e..dc8d1fc662 100644
--- a/tests/e2e/live/utils/agent-server-conversation.ts
+++ b/tests/e2e/live/utils/agent-server-conversation.ts
@@ -5,6 +5,7 @@ import {
type Locator,
type Page,
} from "@playwright/test";
+import { CANVAS_UI_CLIENT_TOOL } from "../../../../src/api/canvas-ui-client-tool";
export const BACKEND_URL =
process.env.LIVE_E2E_BACKEND_URL ?? "http://127.0.0.1:18100";
@@ -46,7 +47,6 @@ const DEFAULT_AGENT_TOOLS = [
{ name: "terminal", params: {} },
{ name: "file_editor", params: {} },
{ name: "task_tracker", params: {} },
- { name: "canvas_ui", params: {} },
];
export const sessionApiKey = firstNonEmpty(
process.env.LIVE_E2E_SESSION_API_KEY,
@@ -249,9 +249,7 @@ export async function createLiveConversation(
verification: buildLiveVerificationSettings(options),
tools: DEFAULT_AGENT_TOOLS,
},
- tool_module_qualnames: {
- canvas_ui: "canvas_ui_tool",
- },
+ client_tools: [CANVAS_UI_CLIENT_TOOL],
},
});
diff --git a/tools/canvas_ui_tool.py b/tools/canvas_ui_tool.py
index 851997f981..1fe91b1842 100644
--- a/tools/canvas_ui_tool.py
+++ b/tools/canvas_ui_tool.py
@@ -1,12 +1,18 @@
-"""Canvas UI control tool.
+"""Legacy conversation compatibility for the former Python Canvas UI tool.
-Shipped with Agent Canvas. Mounted into the agent-server container at
-``/canvas-tools`` and loaded via ``tool_module_qualnames`` so the agent can
-direct the frontend (navigate to a file, switch tabs, show a preview).
+New conversations define ``canvas_ui_control`` through the agent-server's
+``client_tools`` JSON API. The distinct name avoids colliding with this
+process-global legacy registration. This module remains importable because
+persisted conversations store
+``tool_module_qualnames = {"canvas_ui": "canvas_ui_tool"}`` and may contain
+``CanvasUIAction`` / ``CanvasUIObservation`` events. Agent-server imports the
+module before restoring those conversations so the legacy event kinds and tool
+registration remain resolvable.
The server-side executor is a no-op that returns an acknowledgment. The actual
UI effect happens client-side: the frontend watches the WebSocket stream for
-``ActionEvent``s with ``tool_name == "canvas_ui"`` and dispatches the command.
+legacy ``canvas_ui`` and current ``canvas_ui_control`` ActionEvents and dispatches
+the command.
"""
from collections.abc import Sequence
@@ -127,7 +133,7 @@ class CanvasUITool(ToolDefinition[CanvasUIAction, CanvasUIObservation]):
]
-# Auto-register at import time. The agent-server imports this module via
-# tool_module_qualnames; this call wires the tool into the registry so it can
-# be referenced by name in conversation tool lists.
+# Persisted pre-client_tools conversations import this module by qualname before
+# restoring their agent and events. Keep the registration until those records
+# have a server-side migration path.
register_tool("canvas_ui", CanvasUITool)