Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: extension, unified, safe and shell-desktop integration tests use the computer_* tools (LPT-934)
alexshpunt-clanker[bot]b77db68cec7d
diff --git a/test/extension.integration.test.ts b/test/extension.integration.test.tsindex 05bba9b..3d02067 100644--- a/test/extension.integration.test.ts+++ b/test/extension.integration.test.ts@@ -8,7 +8,7 @@ import { expect, test } from "vitest";type Action = { metadata: { semanticAction: Record<string, unknown> } };const action = (details: unknown) => (details as Action).metadata.semanticAction;-const open = (id: string) => toolCall({ id, name: "computer_insert", arguments: { path: "computer:machine/local", text: '{"action":"open","app":"Notepad"}' } });+const open = (id: string) => toolCall({ id, name: "computer_open", arguments: { at: "computer:machine/local", app: "Notepad" } });test("standalone Pi opens two windows with first observations, rejects invented refs, and releases MCP", async () => {await mkdir(".tmp", { recursive: true });@@ -22,18 +22,18 @@ test("standalone Pi opens two windows with first observations, rejects inventedconst result = await new PiIntegrationTest({testName: "standalone-new-window-and-refusal", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/fake-extension.ts")],- tools: ["computer_insert", "computer_write"], isolateUserResources: true,+ tools: ["computer_open", "computer_type", "computer_delete"], isolateUserResources: true,conversation: [assistantMessage([open("first")], { stopReason: "toolUse" }),assistantMessage([open("second")], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "stale", name: "computer_write", arguments: {- path: "computer:observation/not-an-observation", content: "Do not dispatch",+ assistantMessage([toolCall({ id: "stale", name: "computer_type", arguments: {+ window: "computer:window/not-a-window", text: "Do not dispatch",} })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("The standalone contract was exercised.")]),],- }).run("Open two new test windows with first observations, refuse an invented write reference, and release runtime.");- expect(getToolCallNames(result)).toEqual(["computer_insert", "computer_insert", "computer_write", "computer_insert"]);+ }).run("Open two new test windows with first observations, refuse an invented window reference, and release runtime.");+ expect(getToolCallNames(result)).toEqual(["computer_open", "computer_open", "computer_type", "computer_delete"]);const first = action(getToolExecutionDetails(getToolExecution(result, "first")));const second = action(getToolExecutionDetails(getToolExecution(result, "second")));expect(first).toMatchObject({ status: "observed", opened: true, mode: "yolo", observation_id: expect.any(String),@@ -44,12 +44,12 @@ test("standalone Pi opens two windows with first observations, rejects inventedexpect(getToolResultText(result, "first")).toMatch(/^Window: computer:local\/w1[a-z]{2}\nObservation: computer:local\/o1[a-z]{2}$/m);expect(getToolExecution(result, "first").isError).toBe(false);expect(getToolExecution(result, "stale").isError).toBe(true);- expect(getToolResultText(result, "stale")).toContain("Observation reference expired");+ expect(getToolResultText(result, "stale")).toContain("Unknown or closed window reference");const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));expect(calls.filter(call => call.name === "launch_app")).toHaveLength(2);expect(calls.filter(call => call.name === "get_window_state")).toHaveLength(2);expect(calls.filter(call => call.name === "type_text")).toHaveLength(0);expect(calls).toContainEqual({ event: "stdin-eof" });- expect(result.tuiRenderedOutput).toContain("computer_insert");+ expect(result.tuiRenderedOutput).toContain("computer_open");} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/safe-extension.integration.test.ts b/test/safe-extension.integration.test.tsindex 419cd5b..4df1cff 100644--- a/test/safe-extension.integration.test.ts+++ b/test/safe-extension.integration.test.ts@@ -6,7 +6,7 @@ import {} from "pi-coding-agent-test";import { expect, test } from "vitest";-const open = { path: "computer:machine/local", text: '{"action":"open","app":"Notepad"}' };+const open = { at: "computer:machine/local", app: "Notepad" };test("real Pi restores Safe, filters foreign windows, rejects policy arguments, and opens an owned window", async () => {await mkdir(".tmp", { recursive: true });@@ -20,11 +20,11 @@ test("real Pi restores Safe, filters foreign windows, rejects policy arguments,const result = await new PiIntegrationTest({testName: "safe-restoration-and-tool-boundary", artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve("test/fixtures/safe-extension.ts")],- tools: ["computer_search", "computer_insert"], isolateUserResources: true,+ tools: ["computer_search", "computer_open", "computer_delete"], isolateUserResources: true,conversation: [assistantMessage([toolCall({ id: "foreign", name: "computer_search", arguments: { path: "computer:machine/local/windows", query: "" } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "promote", name: "computer_insert", arguments: { ...open, mode: "yolo" } })], { stopReason: "toolUse" }),- assistantMessage([toolCall({ id: "owned", name: "computer_insert", arguments: open })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "promote", name: "computer_open", arguments: { ...open, mode: "yolo" } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "owned", name: "computer_open", arguments: open })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Safe remained selected.")]),],diff --git a/test/shell-desktop.integration.test.ts b/test/shell-desktop.integration.test.tsindex fec774f..9543f65 100644--- a/test/shell-desktop.integration.test.ts+++ b/test/shell-desktop.integration.test.ts@@ -25,7 +25,7 @@ text("observed");`;try {const result = await new PiIntegrationTest({ testName: `shell-desktop-${mode}`, artifactsDir: testArtifactsDir(import.meta.filename), cwd,extensions: [path.resolve(mode === "safe" ? "test/fixtures/safe-extension.ts" : "test/fixtures/shell-desktop-extension.ts")],- isolateUserResources: true, tools: mode === "safe" ? ["computer_search", "computer_insert"] : ["codemode", "computer_insert"],+ isolateUserResources: true, tools: mode === "safe" ? ["computer_search", "computer_delete"] : ["codemode", "computer_delete"],conversation: [assistantMessage([toolCall({ id: "discover", name: mode === "safe" ? "computer_search" : "codemode",arguments: mode === "safe" ? { path: "computer:machine/local/windows", query: "" } : { code: discover } })], { stopReason: "toolUse" }),diff --git a/test/unified-tools.integration.test.ts b/test/unified-tools.integration.test.tsindex 451e9be..c0ac0a6 100644--- a/test/unified-tools.integration.test.ts+++ b/test/unified-tools.integration.test.ts@@ -25,13 +25,13 @@ test("production Pi exposes only computer tools, with no direct or Codemode browassistantMessage([toolCall({ id: "apps", name: "codemode",arguments: { code: 'text(await tools.computer_search({path:"computer:machine/local/apps",query:"Notepad"}));' } })], { stopReason: "toolUse" }),assistantMessage([toolCall({ id: "close", name: "codemode",- arguments: { code: 'text((await tools.computer_insert({path:"computer:session",text:JSON.stringify({action:"close"})})).outcome);' } })], { stopReason: "toolUse" }),+ arguments: { code: 'text((await tools.computer_delete({path:"computer:session"})).outcome);' } })], { stopReason: "toolUse" }),assistantMessage([text("Unified tool loadout checked.")]),],}).run("Check the production loadout. Only discover apps; never open a window or send input.");// Without Pi Agent IDE, production computer use offers deferred tools that tool_search loads.- const standalone = ["computer_browser", "computer_click", "computer_clipboard", "computer_delete", "computer_display", "computer_drag", "computer_files", "computer_hover", "computer_insert", "computer_open",- "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_wait", "computer_window", "computer_write"];+ const standalone = ["computer_browser", "computer_click", "computer_clipboard", "computer_delete", "computer_display", "computer_drag", "computer_files", "computer_hover", "computer_open",+ "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_wait", "computer_window"];expect(getToolExecutionDetails(getToolExecution(result, "registry"))).toEqual({names: standalone,exposure: Object.fromEntries(standalone.map(name => [name, "deferred"])),