Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests for lean computer-use text (LPT-995)
alexshpunt-clanker[bot]c467e2a33ac5
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.tsindex 004e1b2..3de596f 100644--- a/test/extension-modes.integration.test.ts+++ b/test/extension-modes.integration.test.ts@@ -68,8 +68,13 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and thexpect(getToolResultText(result, "display")).not.toContain("unknown");expect(getToolResultText(result, "display")).toContain("Display addresses name their machine");expect(getToolResultText(result, "delete")).toContain("Computer closed");- expect(getToolResultText(result, "guide")).toMatch(/^# Computer resources/);+ expect(getToolResultText(result, "guide")).toMatch(/^# Computer\n/);+ // The guide names the standalone tools, never a placeholder.+ expect(getToolResultText(result, "guide")).toContain("computer_read");+ expect(getToolResultText(result, "guide")).not.toMatch(/\{(read|search|delete)\}/);expect(getToolResultText(result, "session")).toContain("computer:machine/local");+ // The guide was read already, so the first computer address does not bring it again.+ expect(getToolResultText(result, "session")).not.toContain("# Guide: computer");expect(getToolResultText(result, "close")).toContain("Computer closed");}, 120_000);@@ -77,10 +82,30 @@ test("an IDE that refuses this plugin API version leaves computer use standaloneconst result = await run("incompatible-ide", ["builtin:tool-search", path.join(root, "test/fixtures/incompatible-ide-extension.ts"), computerUse], ["read"], [{ id: "find", name: "tool_search", arguments: { query: "computer read windows displays" } },{ id: "session", name: "computer_read", arguments: { path: "computer:session" } },+ { id: "again", name: "computer_read", arguments: { path: "computer:session" } },]);- expect(getToolCallNames(result)).toEqual(["tool_search", "computer_read"]);+ expect(getToolCallNames(result)).toEqual(["tool_search", "computer_read", "computer_read"]);expect(getToolExecution(result, "session").isError, getToolResultText(result, "session")).toBe(false);expect(getToolResultText(result, "session")).toContain("computer:machine/local");+ // As with the IDE, the first read of a computer address brings the guide with the standalone names, and only once.+ expect(getToolResultText(result, "session")).toContain("# Guide: computer");+ expect(getToolResultText(result, "session")).toContain("computer_read of a viewport only looks");+ expect(getToolResultText(result, "again")).not.toContain("# Guide: computer");+}, 120_000);++test("without Pi Agent IDE, a script's first computer_read brings the guide to the script's own result", async () => {+ const result = await run("standalone-codemode", [path.join(root, "test/fixtures/codemode-extension.ts"), computerUse], ["codemode"], [+ // The script prints nothing of what it read; the guide still reaches the model once.+ { id: "script", name: "codemode", arguments: { code: `await tools.computer_read({ path: "computer:session" }); await tools.computer_read({ path: "computer:session" }); text("read twice");` } },+ { id: "after", name: "codemode", arguments: { code: `text(await tools.computer_read({ path: "computer:session" }));` } },+ { id: "close", name: "codemode", arguments: { code: `await tools.computer_delete({ path: "computer:session" }); text("closed");` } },+ ]);+ expect(getToolExecution(result, "script").isError, getToolResultText(result, "script")).toBe(false);+ expect(getToolResultText(result, "script")).toContain("read twice");+ expect(getToolResultText(result, "script").split("# Guide: computer").length).toBe(2);+ expect(getToolResultText(result, "script")).toContain("computer_read of a viewport only looks");+ expect(getToolResultText(result, "after")).toContain("computer:machine/local");+ expect(getToolResultText(result, "after")).not.toContain("# Guide: computer");}, 120_000);test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses Write and Insert, and adds only the typed computer tools", async () => {@@ -110,7 +135,11 @@ test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses W"computer_click", "computer_hover", "computer_drag", "computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);for (const id of ["session", "close", "delete"])expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);- expect(getToolResultText(result, "session")).toContain("computer:session");+ expect(getToolResultText(result, "session")).toContain("computer:machine/local");+ // The first Read of a computer address brings the guide, with the IDE's tool names filled in.+ expect(getToolResultText(result, "session")).toContain("# Guide: computer");+ expect(getToolResultText(result, "session")).toContain("read of a viewport only looks");+ expect(getToolResultText(result, "session")).not.toMatch(/\{(read|search|delete)\}|computer_read/);// Write and Insert only name the typed tool that does what they were sent; nothing reaches a machine.for (const id of ["insert", "write"]) expect(getToolExecution(result, id).isError).toBe(true);expect(getToolResultText(result, "insert")).toContain("Insert does not act on computer resources; nothing was sent. Use computer_open.");diff --git a/test/fixtures/codemode-extension.ts b/test/fixtures/codemode-extension.tsnew file mode 100644index 0000000..07155b0--- /dev/null+++ b/test/fixtures/codemode-extension.ts@@ -0,0 +1,6 @@+import { createCodemodeExtension, type ExtensionAPI } from "@earendil-works/pi-coding-agent";++/** Turn Codemode on in a real Pi run, so a test can call the computer tools from a script. */+export default function codemode(pi: ExtensionAPI): void {+ createCodemodeExtension({ mode: "on" })(pi);+}diff --git a/test/fixtures/standalone-computer.ts b/test/fixtures/standalone-computer.tsindex 18b9ff7..77f6fb2 100644--- a/test/fixtures/standalone-computer.ts+++ b/test/fixtures/standalone-computer.ts@@ -3,6 +3,7 @@ import { ComputerResources } from "../../src/computer-resources.ts";import type { ComputerMode, ComputerResult, ComputerSession } from "../../src/session.ts";import { registerStandaloneTools } from "../../src/standalone-tools.ts";import { registerComputerTools } from "../../src/computer-tools.ts";+import { STANDALONE_TOOL_NAMES } from "../../src/tool-names.ts";export const STANDALONE_TOOLS = ["computer_read", "computer_search", "computer_delete", "computer_open", "computer_window", "computer_display", "computer_wait","computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag", "computer_scroll",@@ -16,7 +17,7 @@ export function registerFixtureComputerTools(pi: ExtensionAPI, getSession: (cwd:closeSession: () => Promise<ComputerResult>, getMode?: () => ComputerMode): ComputerResources {const resources = new ComputerResources(getSession, closeSession, undefined, undefined, getMode);pi.on("session_start", () => {- registerComputerTools(pi, resources);+ registerComputerTools(pi, resources, STANDALONE_TOOL_NAMES);registerStandaloneTools(pi, resources, "Fixture computer guide.");pi.setActiveTools([...new Set([...pi.getActiveTools(), ...STANDALONE_TOOLS])]);});diff --git a/test/tool-names.test.ts b/test/tool-names.test.tsnew file mode 100644index 0000000..f2b8808--- /dev/null+++ b/test/tool-names.test.ts@@ -0,0 +1,24 @@+import { StringEnum, Type, validateToolArguments } from "@earendil-works/pi-ai";+import { expect, test } from "vitest";+import { fillToolNames, IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES } from "../src/tool-names.ts";++const schema = Type.Object({+ window: Type.String({ description: "Window address from {search}, {read} or a name." }),+ views: Type.Optional(Type.Array(Type.String(), { description: "Views of one {read}; {delete} releases it." })),+}, { additionalProperties: false });++test("tool names are filled per mode and the declared JSON keeps its shape", () => {+ for (const names of [IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES]) {+ const filled = fillToolNames(schema, names);+ const expected = JSON.stringify(schema).replaceAll("{search}", names.search).replaceAll("{read}", names.read).replaceAll("{delete}", names.delete);+ expect(JSON.stringify(filled)).toBe(expected);+ }+ expect((fillToolNames(schema, STANDALONE_TOOL_NAMES).properties.window as { description?: string }).description).toBe("Window address from computer_search, computer_read or a name.");+});++test("a filled enum schema still accepts its values and rejects others", () => {+ const tool = { name: "t", description: "", parameters: fillToolNames(Type.Object({ mode: StringEnum(["wait", "check"] as const, { description: "{read}" }) }), IDE_TOOL_NAMES) };+ const call = (mode: string) => validateToolArguments(tool, { type: "toolCall", id: "1", name: "t", arguments: { mode } });+ expect(call("check")).toEqual({ mode: "check" });+ expect(() => call("bogus")).toThrow();+});