Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: cover deferred Computer Use discovery and fallback
alexshpunt-clanker[bot]0010d883fdf8
diff --git a/test/deferred-capabilities.integration.test.ts b/test/deferred-capabilities.integration.test.tsnew file mode 100644index 0000000..cbee423--- /dev/null+++ b/test/deferred-capabilities.integration.test.ts@@ -0,0 +1,143 @@+import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import { getCurrentTools, type Message } from "@earendil-works/pi-ai";+import { assistantMessage, getToolExecution, getToolExecutionDetails, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";+import { expect, test } from "vitest";+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";++const computer = path.resolve("src/index.ts");+const probe = path.resolve("test/fixtures/computer-capability-snapshot.ts");+type Schema = { name: string; description: string; parameters: { properties: Record<string, { description?: string }> }; promptGuidelines: string[] };+type Snapshot = { tools: Schema[]; computerTools: string[]; activeTools: string[]; entries: { data: { active: string[] } }[]; additions: Record<string, string> };+const call = (id: string, name: string, args: Record<string, unknown> = {}) =>+ assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });+const take = (id: string) => call(id, "computer_capability_snapshot");+const snapshot = (result: Awaited<ReturnType<PiIntegrationTest["run"]>>, id: string) =>+ JSON.parse(getToolResultText(result, id)) as Snapshot;+const active = (value: Snapshot) => value.entries.at(-1)?.data.active ?? [];+const schema = (value: Snapshot, name: string) => {+ const tool = value.tools.find(tool => tool.name === name);+ if (!tool) throw Error("Missing shared tool: " + name);+ return tool;+};++async function run(name: string, extensions: string[], conversation: ReturnType<typeof assistantMessage>[], mode = "on", excluded: string[] = [], transport: "rpc" | "tui" = "rpc") {+ await mkdir(".tmp/computer-capabilities", { recursive: true });+ const cwd = await mkdtemp(path.resolve(".tmp/computer-capabilities/case-"));+ const agentDir = path.join(cwd, "agent");+ try {+ await mkdir(agentDir);+ await mkdir(path.join(cwd, ".pi/pi-agent-ide"), { recursive: true });+ await writeFile(path.join(cwd, ".pi/pi-agent-ide/extensions.json"), JSON.stringify({ noPostProcessing: transport !== "tui" }));+ await writeFile(path.join(agentDir, "settings.json"), JSON.stringify({+ defaultTools: ["+codemode", "+tool_search", ...excluded.map(name => "-" + name)],+ codemode: { mode, inlineBudget: 0 },+ }));+ await writeFile(path.join(cwd, "example.txt"), "ordinary-file-content\n");+ const started = path.join(cwd, "native-started");+ const trap = path.join(cwd, "trap-driver.mjs");+ await writeFile(trap, 'import { writeFileSync } from "node:fs"; writeFileSync(process.argv[2], "unexpected"); process.exit(1);\n');+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [trap, started] }));+ const result = await new PiIntegrationTest({+ testName: name, cwd, rawMode: false, transport,+ piCommand: PI_COMMAND, artifactsDir: testArtifactsDir(import.meta.filename),+ extensions: [...extensions, probe, "builtin:codemode", "builtin:tool-search"],+ environment: { PI_CODING_AGENT_DIR: agentDir },+ conversation: [...conversation, assistantMessage([text("Done")])],+ }).run("Verify computer capability activation without opening native resources");+ await expect(access(started)).rejects.toMatchObject({ code: "ENOENT" });+ return result;+ } finally { await rm(cwd, { recursive: true, force: true }); }+}++test.each([+ { computerFirst: false, discovered: "ide_capability_computer" },+ { computerFirst: true, discovered: "ide_capability_computer" },+ { computerFirst: false, discovered: "computer_window" },+])("computer discovery defers shared metadata ($discovered, computerFirst=$computerFirst)", async ({ computerFirst, discovered }) => {+ const base = snapshot(await run("computer-capability-base", [IDE_EXTENSION], [take("base")]), "base");+ const result = await run("computer-capability-discovery-" + discovered + "-" + computerFirst,+ computerFirst ? [computer, IDE_EXTENSION] : [IDE_EXTENSION, computer], [+ take("initial"),+ call("file", "read", { path: "example.txt" }), take("after-file"),+ call("miss", "tool_search", { query: "zxqv704839", limit: 1 }), take("after-miss"),+ call("discover", "tool_search", { query: discovered, limit: 1 }), take("activated"),+ call("repeat", "tool_search", { query: discovered, limit: 1 }), take("repeated"),+ ]);+ for (const id of ["initial", "after-file", "after-miss"]) {+ const value = snapshot(result, id);+ const metadata = (tools: Schema[]) => tools.map(({ name, description, parameters }) => ({ name, description, parameters }));+ expect(metadata(value.tools)).toEqual(metadata(base.tools));+ if (value.additions.search) expect(schema(value, "search").promptGuidelines).not.toContain(value.additions.search);+ expect(active(value)).not.toContain("computer");+ }+ expect(getToolExecution(result, "file").isError).toBe(false);+ expect(getToolResultText(result, "file")).toContain("ordinary-file-content");+ const activated = snapshot(result, "activated");+ const loaded = (getToolExecutionDetails(getToolExecution(result, "discover")) as { loaded: string[] }).loaded;+ expect(loaded).toHaveLength(1);+ if (discovered === "ide_capability_computer") expect(loaded).toEqual([discovered]);+ else expect(loaded[0]).not.toBe("ide_capability_computer");+ expect(activated.computerTools).toEqual(expect.arrayContaining(loaded));+ expect(active(activated)).toContain("computer");+ const repeated = snapshot(result, "repeated");+ // Native discovery can load the next matching action; the capability contribution stays unchanged.+ expect(repeated.tools).toEqual(activated.tools);+ expect(repeated.entries).toEqual(activated.entries);+ for (const [tool, field, addition] of [+ ["read", "path", "path"], ["read", "views", "views"], ["delete", "path", "delete"],+ ]) {+ expect(schema(activated, tool).parameters.properties[field].description).toContain(activated.additions[addition]);+ }+ if (activated.additions.search) expect(schema(activated, "search").promptGuidelines).toContain(activated.additions.search);+ const initialDeclarations = getCurrentTools(result.providerRequests[0]?.messages as Message[]);+ const finalDeclarations = getCurrentTools(result.providerRequests.at(-1)?.messages as Message[]);+ for (const name of ["read", "search", "delete"]) {+ const initial = initialDeclarations.find(tool => tool.name === name);+ const final = finalDeclarations.find(tool => tool.name === name);+ expect(initial?.parameters).toEqual(schema(base, name).parameters);+ expect(final?.parameters).toEqual(schema(activated, name).parameters);+ }+}, 120_000);++test.each(["on", "only"])("Codemode computer search stays inactive until its first call (%s)", async mode => {+ const result = await run("computer-capability-codemode-" + mode, [IDE_EXTENSION, computer], [+ take("initial"),+ call("describe", "codemode", { code: 'text(await searchTools("computer window")); text(await describeTool("computer_window")); text(await describeNamespace("computer"));' }),+ take("described"),+ call("first-use", "codemode", { code: 'text(await tools.computer_window({window:"computer:window/not-a-window",action:"name",as:"unused"}));' }),+ take("used"),+ ], mode);+ expect(getToolExecution(result, "describe").isError).toBe(false);+ expect(snapshot(result, "described")).toEqual(snapshot(result, "initial"));+ expect(active(snapshot(result, "used"))).toContain("computer");+}, 120_000);++test("excluding the resource loader keeps it hidden while computer actions remain usable", async () => {+ const result = await run("computer-capability-excluded-loader", [IDE_EXTENSION, computer], [+ call("catalog-before", "codemode", { code: 'if (ALL_TOOLS.some(({name}) => name === "ide_capability_computer")) throw new Error("Excluded loader is callable");' }),+ take("initial"),+ call("discover-action", "tool_search", { query: "computer_window", limit: 1 }),+ take("activated"),+ call("catalog-after", "codemode", { code: 'if (ALL_TOOLS.some(({name}) => name === "ide_capability_computer")) throw new Error("Excluded loader is callable");' }),+ ], "on", ["ide_capability_computer"]);+ for (const id of ["catalog-before", "catalog-after"]) {+ expect(getToolExecution(result, id).isError).toBe(false);+ }+ expect(active(snapshot(result, "initial"))).not.toContain("computer");+ const loaded = (getToolExecutionDetails(getToolExecution(result, "discover-action")) as { loaded: string[] }).loaded;+ expect(loaded).toHaveLength(1);+ expect(loaded[0]).not.toBe("ide_capability_computer");+ expect(snapshot(result, "activated").computerTools).toEqual(expect.arrayContaining(loaded));+ expect(active(snapshot(result, "activated"))).toContain("computer");+}, 120_000);++test("the resource loader runs through the real Pi TUI", async () => {+ const result = await run("computer-capability-native-tui", [IDE_EXTENSION, computer], [+ call("discover", "tool_search", { query: "ide_capability_computer", limit: 1 }),+ call("load", "ide_capability_computer"),+ ], "on", [], "tui");+ expect(getToolExecution(result, "load").isError).toBe(false);+ expect(getToolExecutionDetails(getToolExecution(result, "load"))).toMatchObject({ capability: "computer" });+ expect(getToolExecutionDetails(getToolExecution(result, "discover"))).toMatchObject({ loaded: ["ide_capability_computer"] });+}, 120_000);diff --git a/test/deferred-capability-fallback.integration.test.ts b/test/deferred-capability-fallback.integration.test.tsnew file mode 100644index 0000000..33b0267--- /dev/null+++ b/test/deferred-capability-fallback.integration.test.ts@@ -0,0 +1,135 @@+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import { assistantMessage, getToolExecution, getToolExecutionDetails, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";+import { expect, test } from "vitest";+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";++const computer = path.resolve("src/index.ts");+const probe = path.resolve("test/fixtures/computer-capability-fallback.ts");+const fallbackNames = ["computer_read", "computer_search", "computer_delete"];+type Schema = {+ name: string; exposure?: string; namespace?: { name: string }; promptGuidelines?: string[];+ parameters: { required?: string[]; additionalProperties?: boolean; properties: Record<string, { type?: string; minimum?: number; description?: string }> };+};+type Snapshot = {+ tools: Schema[]; active: string[]; entries: { data: { active: string[] } }[];+ additions: Record<string, string>;+};+const call = (id: string, name: string, args: Record<string, unknown> = {}) =>+ assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });+const take = (id: string) => call(id, "computer_fallback_snapshot");+const snapshot = (result: Awaited<ReturnType<PiIntegrationTest["run"]>>, id: string) =>+ JSON.parse(getToolResultText(result, id)) as Snapshot;+const schema = (value: Snapshot, name: string) => value.tools.find(tool => tool.name === name);+const active = (value: Snapshot) => value.entries.at(-1)?.data.active ?? [];+const discovery = (id: string, name: string) => call(id, "tool_search", { query: name, limit: 1 });++async function run(name: string, withIde: boolean, disabled: string[], conversation: ReturnType<typeof assistantMessage>[]) {+ await mkdir(".tmp/computer-capabilities", { recursive: true });+ const cwd = await mkdtemp(path.resolve(".tmp/computer-capabilities/fallback-"));+ const agentDir = path.join(cwd, "agent");+ const started = path.join(cwd, "native-started");+ try {+ await mkdir(agentDir);+ await writeFile(path.join(agentDir, "settings.json"), JSON.stringify({ defaultTools: ["+tool_search"] }));+ await mkdir(path.join(cwd, ".pi/pi-agent-ide"), { recursive: true });+ await writeFile(path.join(cwd, ".pi/pi-agent-ide/extensions.json"), JSON.stringify({ disabled, noPostProcessing: true }));+ const trap = path.join(cwd, "trap-driver.mjs");+ await writeFile(trap, 'import { writeFileSync } from "node:fs"; writeFileSync(process.argv[2], "unexpected"); process.exit(1);\n');+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [trap, started] }));+ const result = await new PiIntegrationTest({+ testName: name, cwd, rawMode: false, transport: "rpc",+ piCommand: PI_COMMAND, artifactsDir: testArtifactsDir(import.meta.filename),+ extensions: [...(withIde ? [IDE_EXTENSION] : []), computer, probe, "builtin:tool-search"],+ environment: { PI_CODING_AGENT_DIR: agentDir },+ conversation: [...conversation, assistantMessage([text("Done")])],+ }).run("Verify standalone and partial IDE computer contracts without starting native resources");+ await expect(access(started)).rejects.toMatchObject({ code: "ENOENT" });+ return result;+ } finally { await rm(cwd, { recursive: true, force: true }); }+}++function assertFallbackDeclarations(value: Snapshot) {+ for (const name of fallbackNames) {+ const tool = schema(value, name);+ expect(tool).toMatchObject({ exposure: "deferred", namespace: { name: "computer" }, parameters: { additionalProperties: false } });+ }+ expect(schema(value, "computer_read")?.parameters.required).toEqual(["path"]);+ expect(schema(value, "computer_search")?.parameters.required).toEqual(["path", "query"]);+ expect(schema(value, "computer_delete")?.parameters.required).toEqual(["path"]);+ expect(schema(value, "computer_read")?.parameters.properties.limit).toMatchObject({ type: "integer", minimum: 0 });+ expect(schema(value, "computer_search")?.parameters.properties.limit).toMatchObject({ type: "integer", minimum: 1 });+ expect(schema(value, "computer_read")?.parameters.properties.path.description).toContain(value.additions.standalonePath);+ expect(schema(value, "computer_read")?.parameters.properties.views.description).toBe(value.additions.standaloneViews);+ expect(schema(value, "computer_search")?.parameters.properties.path.description).toBe(value.additions.search);+ expect(schema(value, "computer_delete")?.parameters.properties.path.description).toBe(value.additions.delete);+}++test.each([false, true])("standalone resource tools keep deferred contracts and validate before native access (IDE host=%s)", async withIde => {+ const result = await run("computer-capability-standalone-" + withIde, withIde, withIde ? ["read.core", "search.core", "editor.core"] : [], [+ take("initial"),+ call("enable-fallback", "computer_fallback_snapshot", { enableFallback: true }),+ call("guide", "computer_read", { path: "docs:computer", offset: 1, limit: 1 }),+ call("invalid-read", "computer_read", { path: "docs:computer", limit: -1 }),+ call("invalid-search", "computer_search", { path: "computer:machine/local/apps" }),+ call("close", "computer_delete", { path: "computer:session" }),+ take("after"),+ ]);+ const initial = snapshot(result, "initial");+ assertFallbackDeclarations(initial);+ for (const name of fallbackNames) expect(initial.active).not.toContain(name);+ expect(schema(initial, "ide_capability_computer")).toBeUndefined();+ expect(active(initial)).not.toContain("computer");+ const after = snapshot(result, "after");+ assertFallbackDeclarations(after);+ for (const name of fallbackNames) expect(after.active).toContain(name);+ expect(schema(after, "ide_capability_computer")).toBeUndefined();+ const guide = await readFile(path.resolve("docs/computer-resources.md"), "utf8");+ expect(getToolResultText(result, "guide").split("\n")[0]).toBe(guide.split("\n")[0]);+ expect(getToolExecution(result, "guide").isError).toBe(false);+ expect(getToolExecutionDetails(getToolExecution(result, "guide"))).toMatchObject({ source: "docs:computer", startLine: 1, endLine: 1 });+ expect(getToolExecution(result, "invalid-read").isError).toBe(true);+ expect(getToolExecution(result, "invalid-search").isError).toBe(true);+ expect(getToolExecution(result, "close").isError).toBe(false);+ expect(getToolExecutionDetails(getToolExecution(result, "close"))).toMatchObject({ metadata: { semanticAction: { ok: true, effect: "applied" } } });+}, 120_000);++test.each([+ { disabled: "read.core", accepted: ["search", "delete"], rejected: "read" },+ { disabled: "search.core", accepted: ["read", "delete"], rejected: "search" },+ { disabled: "editor.core", accepted: ["read", "search"], rejected: "delete" },+])("partial IDE keeps only accepted resource metadata deferred ($disabled disabled)", async ({ disabled, accepted, rejected }) => {+ const result = await run("computer-capability-partial-" + disabled, true, [disabled], [+ take("initial"),+ discovery("load", "ide_capability_computer"),+ take("activated"),+ call("enable-fallback", "computer_fallback_snapshot", { enableFallback: true }),+ call("guide", "computer_read", { path: "docs:computer", offset: 1, limit: 1 }),+ take("after-fallback"),+ ]);+ const initial = snapshot(result, "initial");+ assertFallbackDeclarations(initial);+ expect(active(initial)).not.toContain("computer");+ for (const name of fallbackNames) expect(initial.active).not.toContain(name);+ expect(schema(initial, "ide_capability_computer")?.exposure).toBe("deferred");+ const activated = snapshot(result, "activated");+ expect(active(activated)).toContain("computer");+ expect(schema(activated, rejected)).toEqual(schema(initial, rejected));+ if (accepted.includes("read")) {+ for (const field of ["path", "views"]) {+ if (initial.additions[field]) expect(schema(initial, "read")?.parameters.properties[field].description).not.toContain(initial.additions[field]);+ expect(schema(activated, "read")?.parameters.properties[field].description).toContain(activated.additions[field]);+ }+ }+ if (accepted.includes("search")) {+ expect(schema(initial, "search")?.promptGuidelines ?? []).not.toContain(initial.additions.search);+ if (activated.additions.search) expect(schema(activated, "search")?.promptGuidelines).toContain(activated.additions.search);+ }+ if (accepted.includes("delete")) {+ if (initial.additions.delete) expect(schema(initial, "delete")?.parameters.properties.path.description).not.toContain(initial.additions.delete);+ expect(schema(activated, "delete")?.parameters.properties.path.description).toContain(activated.additions.delete);+ }+ assertFallbackDeclarations(snapshot(result, "after-fallback"));+ expect(getToolExecution(result, "guide").isError).toBe(false);+ expect(getToolExecutionDetails(getToolExecution(result, "guide"))).toMatchObject({ source: "docs:computer", startLine: 1, endLine: 1 });+}, 120_000);diff --git a/test/fixtures/computer-capability-fallback.ts b/test/fixtures/computer-capability-fallback.tsnew file mode 100644index 0000000..012079c--- /dev/null+++ b/test/fixtures/computer-capability-fallback.ts@@ -0,0 +1,25 @@+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";+import { Type } from "typebox";+import { computerPathHelp, computerViewsHelp, COMPUTER_SEARCH_HELP, COMPUTER_DELETE_HELP } from "../../src/provider.ts";+import { IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES } from "../../src/tool-names.ts";++export default function fallbackSnapshot(pi: ExtensionAPI): void {+ pi.registerTool({+ name: "computer_fallback_snapshot", label: "Computer fallback snapshot",+ description: "Inspect computer fallback declarations and capability state without accessing native resources.",+ exposure: "model-only", parameters: Type.Object({ enableFallback: Type.Optional(Type.Boolean()) }),+ async execute(_id, input, _signal, _update, ctx) {+ // Select known declarations explicitly so these contract checks do not depend on BM25 prose ranking.+ if (input.enableFallback) pi.setActiveTools([...new Set([...pi.getActiveTools(), "computer_read", "computer_search", "computer_delete"])]);+ const entries = ctx.sessionManager.getBranch().filter(entry => entry.type === "custom" && entry.customType === "ide-capabilities/v1");+ return {+ content: [{ type: "text", text: JSON.stringify({+ tools: pi.getAllTools().filter(tool => ["read", "search", "delete", "computer_read", "computer_search", "computer_delete", "ide_capability_computer"].includes(tool.name)),+ active: pi.getActiveTools(), entries,+ additions: { standalonePath: computerPathHelp(STANDALONE_TOOL_NAMES), standaloneViews: computerViewsHelp(STANDALONE_TOOL_NAMES), path: computerPathHelp(IDE_TOOL_NAMES), views: computerViewsHelp(IDE_TOOL_NAMES), search: COMPUTER_SEARCH_HELP, delete: COMPUTER_DELETE_HELP },+ }) }],+ details: undefined,+ };+ },+ });+}diff --git a/test/fixtures/computer-capability-snapshot.ts b/test/fixtures/computer-capability-snapshot.tsnew file mode 100644index 0000000..8a4d95e--- /dev/null+++ b/test/fixtures/computer-capability-snapshot.ts@@ -0,0 +1,28 @@+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";+import { Type } from "typebox";+import { computerPathHelp, computerViewsHelp, COMPUTER_SEARCH_HELP, COMPUTER_DELETE_HELP } from "../../src/provider.ts";+import { IDE_TOOL_NAMES } from "../../src/tool-names.ts";++export default function snapshot(pi: ExtensionAPI): void {+ pi.registerTool({+ name: "computer_capability_snapshot", label: "Capability snapshot",+ description: "Inspect the shared tools and persisted capability state without starting native resources.",+ exposure: "model-only", parameters: Type.Object({}),+ async execute(_id, _input, _signal, _update, ctx) {+ const tools = pi.getAllTools();+ const entries = ctx.sessionManager.getBranch().filter(entry => entry.type === "custom" && entry.customType === "ide-capabilities/v1");+ return {+ content: [{ type: "text", text: JSON.stringify({+ tools: tools.filter(tool => ["read", "search", "delete"].includes(tool.name)).map(tool => ({+ name: tool.name, description: tool.description, parameters: tool.parameters, promptGuidelines: tool.promptGuidelines ?? [],+ })),+ activeTools: pi.getActiveTools(),+ computerTools: tools.filter(tool => tool.namespace?.name === "computer").map(tool => tool.name),+ entries,+ additions: { path: computerPathHelp(IDE_TOOL_NAMES), views: computerViewsHelp(IDE_TOOL_NAMES), search: COMPUTER_SEARCH_HELP, delete: COMPUTER_DELETE_HELP },+ }) }],+ details: undefined,+ };+ },+ });+}