Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: cover deferred Computer Use discovery and fallback

alexshpunt-clanker[bot]0010d883fdf8
diff --git a/test/deferred-capabilities.integration.test.ts b/test/deferred-capabilities.integration.test.ts
new file mode 100644
index 0000000..cbee423
--- /dev/null
+++ b/test/deferred-capabilities.integration.test.ts
@@ -0,0 +1,143 @@
+import { access, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { getCurrentTools, type Message } from "@earendil-works/pi-ai";
+import { assistantMessage, getToolExecution, getToolExecutionDetails, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
+import { expect, test } from "vitest";
+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";
+
+const computer = path.resolve("src/index.ts");
+const probe = path.resolve("test/fixtures/computer-capability-snapshot.ts");
+type Schema = { name: string; description: string; parameters: { properties: Record<string, { description?: string }> }; promptGuidelines: string[] };
+type Snapshot = { tools: Schema[]; computerTools: string[]; activeTools: string[]; entries: { data: { active: string[] } }[]; additions: Record<string, string> };
+const call = (id: string, name: string, args: Record<string, unknown> = {}) =>
+ assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });
+const take = (id: string) => call(id, "computer_capability_snapshot");
+const snapshot = (result: Awaited<ReturnType<PiIntegrationTest["run"]>>, id: string) =>
+ JSON.parse(getToolResultText(result, id)) as Snapshot;
+const active = (value: Snapshot) => value.entries.at(-1)?.data.active ?? [];
+const schema = (value: Snapshot, name: string) => {
+ const tool = value.tools.find(tool => tool.name === name);
+ if (!tool) throw Error("Missing shared tool: " + name);
+ return tool;
+};
+
+async function run(name: string, extensions: string[], conversation: ReturnType<typeof assistantMessage>[], mode = "on", excluded: string[] = [], transport: "rpc" | "tui" = "rpc") {
+ await mkdir(".tmp/computer-capabilities", { recursive: true });
+ const cwd = await mkdtemp(path.resolve(".tmp/computer-capabilities/case-"));
+ const agentDir = path.join(cwd, "agent");
+ try {
+ await mkdir(agentDir);
+ await mkdir(path.join(cwd, ".pi/pi-agent-ide"), { recursive: true });
+ await writeFile(path.join(cwd, ".pi/pi-agent-ide/extensions.json"), JSON.stringify({ noPostProcessing: transport !== "tui" }));
+ await writeFile(path.join(agentDir, "settings.json"), JSON.stringify({
+ defaultTools: ["+codemode", "+tool_search", ...excluded.map(name => "-" + name)],
+ codemode: { mode, inlineBudget: 0 },
+ }));
+ await writeFile(path.join(cwd, "example.txt"), "ordinary-file-content\n");
+ const started = path.join(cwd, "native-started");
+ const trap = path.join(cwd, "trap-driver.mjs");
+ await writeFile(trap, 'import { writeFileSync } from "node:fs"; writeFileSync(process.argv[2], "unexpected"); process.exit(1);\n');
+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [trap, started] }));
+ const result = await new PiIntegrationTest({
+ testName: name, cwd, rawMode: false, transport,
+ piCommand: PI_COMMAND, artifactsDir: testArtifactsDir(import.meta.filename),
+ extensions: [...extensions, probe, "builtin:codemode", "builtin:tool-search"],
+ environment: { PI_CODING_AGENT_DIR: agentDir },
+ conversation: [...conversation, assistantMessage([text("Done")])],
+ }).run("Verify computer capability activation without opening native resources");
+ await expect(access(started)).rejects.toMatchObject({ code: "ENOENT" });
+ return result;
+ } finally { await rm(cwd, { recursive: true, force: true }); }
+}
+
+test.each([
+ { computerFirst: false, discovered: "ide_capability_computer" },
+ { computerFirst: true, discovered: "ide_capability_computer" },
+ { computerFirst: false, discovered: "computer_window" },
+])("computer discovery defers shared metadata ($discovered, computerFirst=$computerFirst)", async ({ computerFirst, discovered }) => {
+ const base = snapshot(await run("computer-capability-base", [IDE_EXTENSION], [take("base")]), "base");
+ const result = await run("computer-capability-discovery-" + discovered + "-" + computerFirst,
+ computerFirst ? [computer, IDE_EXTENSION] : [IDE_EXTENSION, computer], [
+ take("initial"),
+ call("file", "read", { path: "example.txt" }), take("after-file"),
+ call("miss", "tool_search", { query: "zxqv704839", limit: 1 }), take("after-miss"),
+ call("discover", "tool_search", { query: discovered, limit: 1 }), take("activated"),
+ call("repeat", "tool_search", { query: discovered, limit: 1 }), take("repeated"),
+ ]);
+ for (const id of ["initial", "after-file", "after-miss"]) {
+ const value = snapshot(result, id);
+ const metadata = (tools: Schema[]) => tools.map(({ name, description, parameters }) => ({ name, description, parameters }));
+ expect(metadata(value.tools)).toEqual(metadata(base.tools));
+ if (value.additions.search) expect(schema(value, "search").promptGuidelines).not.toContain(value.additions.search);
+ expect(active(value)).not.toContain("computer");
+ }
+ expect(getToolExecution(result, "file").isError).toBe(false);
+ expect(getToolResultText(result, "file")).toContain("ordinary-file-content");
+ const activated = snapshot(result, "activated");
+ const loaded = (getToolExecutionDetails(getToolExecution(result, "discover")) as { loaded: string[] }).loaded;
+ expect(loaded).toHaveLength(1);
+ if (discovered === "ide_capability_computer") expect(loaded).toEqual([discovered]);
+ else expect(loaded[0]).not.toBe("ide_capability_computer");
+ expect(activated.computerTools).toEqual(expect.arrayContaining(loaded));
+ expect(active(activated)).toContain("computer");
+ const repeated = snapshot(result, "repeated");
+ // Native discovery can load the next matching action; the capability contribution stays unchanged.
+ expect(repeated.tools).toEqual(activated.tools);
+ expect(repeated.entries).toEqual(activated.entries);
+ for (const [tool, field, addition] of [
+ ["read", "path", "path"], ["read", "views", "views"], ["delete", "path", "delete"],
+ ]) {
+ expect(schema(activated, tool).parameters.properties[field].description).toContain(activated.additions[addition]);
+ }
+ if (activated.additions.search) expect(schema(activated, "search").promptGuidelines).toContain(activated.additions.search);
+ const initialDeclarations = getCurrentTools(result.providerRequests[0]?.messages as Message[]);
+ const finalDeclarations = getCurrentTools(result.providerRequests.at(-1)?.messages as Message[]);
+ for (const name of ["read", "search", "delete"]) {
+ const initial = initialDeclarations.find(tool => tool.name === name);
+ const final = finalDeclarations.find(tool => tool.name === name);
+ expect(initial?.parameters).toEqual(schema(base, name).parameters);
+ expect(final?.parameters).toEqual(schema(activated, name).parameters);
+ }
+}, 120_000);
+
+test.each(["on", "only"])("Codemode computer search stays inactive until its first call (%s)", async mode => {
+ const result = await run("computer-capability-codemode-" + mode, [IDE_EXTENSION, computer], [
+ take("initial"),
+ call("describe", "codemode", { code: 'text(await searchTools("computer window")); text(await describeTool("computer_window")); text(await describeNamespace("computer"));' }),
+ take("described"),
+ call("first-use", "codemode", { code: 'text(await tools.computer_window({window:"computer:window/not-a-window",action:"name",as:"unused"}));' }),
+ take("used"),
+ ], mode);
+ expect(getToolExecution(result, "describe").isError).toBe(false);
+ expect(snapshot(result, "described")).toEqual(snapshot(result, "initial"));
+ expect(active(snapshot(result, "used"))).toContain("computer");
+}, 120_000);
+
+test("excluding the resource loader keeps it hidden while computer actions remain usable", async () => {
+ const result = await run("computer-capability-excluded-loader", [IDE_EXTENSION, computer], [
+ call("catalog-before", "codemode", { code: 'if (ALL_TOOLS.some(({name}) => name === "ide_capability_computer")) throw new Error("Excluded loader is callable");' }),
+ take("initial"),
+ call("discover-action", "tool_search", { query: "computer_window", limit: 1 }),
+ take("activated"),
+ call("catalog-after", "codemode", { code: 'if (ALL_TOOLS.some(({name}) => name === "ide_capability_computer")) throw new Error("Excluded loader is callable");' }),
+ ], "on", ["ide_capability_computer"]);
+ for (const id of ["catalog-before", "catalog-after"]) {
+ expect(getToolExecution(result, id).isError).toBe(false);
+ }
+ expect(active(snapshot(result, "initial"))).not.toContain("computer");
+ const loaded = (getToolExecutionDetails(getToolExecution(result, "discover-action")) as { loaded: string[] }).loaded;
+ expect(loaded).toHaveLength(1);
+ expect(loaded[0]).not.toBe("ide_capability_computer");
+ expect(snapshot(result, "activated").computerTools).toEqual(expect.arrayContaining(loaded));
+ expect(active(snapshot(result, "activated"))).toContain("computer");
+}, 120_000);
+
+test("the resource loader runs through the real Pi TUI", async () => {
+ const result = await run("computer-capability-native-tui", [IDE_EXTENSION, computer], [
+ call("discover", "tool_search", { query: "ide_capability_computer", limit: 1 }),
+ call("load", "ide_capability_computer"),
+ ], "on", [], "tui");
+ expect(getToolExecution(result, "load").isError).toBe(false);
+ expect(getToolExecutionDetails(getToolExecution(result, "load"))).toMatchObject({ capability: "computer" });
+ expect(getToolExecutionDetails(getToolExecution(result, "discover"))).toMatchObject({ loaded: ["ide_capability_computer"] });
+}, 120_000);
diff --git a/test/deferred-capability-fallback.integration.test.ts b/test/deferred-capability-fallback.integration.test.ts
new file mode 100644
index 0000000..33b0267
--- /dev/null
+++ b/test/deferred-capability-fallback.integration.test.ts
@@ -0,0 +1,135 @@
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { assistantMessage, getToolExecution, getToolExecutionDetails, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
+import { expect, test } from "vitest";
+import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";
+
+const computer = path.resolve("src/index.ts");
+const probe = path.resolve("test/fixtures/computer-capability-fallback.ts");
+const fallbackNames = ["computer_read", "computer_search", "computer_delete"];
+type Schema = {
+ name: string; exposure?: string; namespace?: { name: string }; promptGuidelines?: string[];
+ parameters: { required?: string[]; additionalProperties?: boolean; properties: Record<string, { type?: string; minimum?: number; description?: string }> };
+};
+type Snapshot = {
+ tools: Schema[]; active: string[]; entries: { data: { active: string[] } }[];
+ additions: Record<string, string>;
+};
+const call = (id: string, name: string, args: Record<string, unknown> = {}) =>
+ assistantMessage([toolCall({ id, name, arguments: args })], { stopReason: "toolUse" });
+const take = (id: string) => call(id, "computer_fallback_snapshot");
+const snapshot = (result: Awaited<ReturnType<PiIntegrationTest["run"]>>, id: string) =>
+ JSON.parse(getToolResultText(result, id)) as Snapshot;
+const schema = (value: Snapshot, name: string) => value.tools.find(tool => tool.name === name);
+const active = (value: Snapshot) => value.entries.at(-1)?.data.active ?? [];
+const discovery = (id: string, name: string) => call(id, "tool_search", { query: name, limit: 1 });
+
+async function run(name: string, withIde: boolean, disabled: string[], conversation: ReturnType<typeof assistantMessage>[]) {
+ await mkdir(".tmp/computer-capabilities", { recursive: true });
+ const cwd = await mkdtemp(path.resolve(".tmp/computer-capabilities/fallback-"));
+ const agentDir = path.join(cwd, "agent");
+ const started = path.join(cwd, "native-started");
+ try {
+ await mkdir(agentDir);
+ await writeFile(path.join(agentDir, "settings.json"), JSON.stringify({ defaultTools: ["+tool_search"] }));
+ await mkdir(path.join(cwd, ".pi/pi-agent-ide"), { recursive: true });
+ await writeFile(path.join(cwd, ".pi/pi-agent-ide/extensions.json"), JSON.stringify({ disabled, noPostProcessing: true }));
+ const trap = path.join(cwd, "trap-driver.mjs");
+ await writeFile(trap, 'import { writeFileSync } from "node:fs"; writeFileSync(process.argv[2], "unexpected"); process.exit(1);\n');
+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath, args: [trap, started] }));
+ const result = await new PiIntegrationTest({
+ testName: name, cwd, rawMode: false, transport: "rpc",
+ piCommand: PI_COMMAND, artifactsDir: testArtifactsDir(import.meta.filename),
+ extensions: [...(withIde ? [IDE_EXTENSION] : []), computer, probe, "builtin:tool-search"],
+ environment: { PI_CODING_AGENT_DIR: agentDir },
+ conversation: [...conversation, assistantMessage([text("Done")])],
+ }).run("Verify standalone and partial IDE computer contracts without starting native resources");
+ await expect(access(started)).rejects.toMatchObject({ code: "ENOENT" });
+ return result;
+ } finally { await rm(cwd, { recursive: true, force: true }); }
+}
+
+function assertFallbackDeclarations(value: Snapshot) {
+ for (const name of fallbackNames) {
+ const tool = schema(value, name);
+ expect(tool).toMatchObject({ exposure: "deferred", namespace: { name: "computer" }, parameters: { additionalProperties: false } });
+ }
+ expect(schema(value, "computer_read")?.parameters.required).toEqual(["path"]);
+ expect(schema(value, "computer_search")?.parameters.required).toEqual(["path", "query"]);
+ expect(schema(value, "computer_delete")?.parameters.required).toEqual(["path"]);
+ expect(schema(value, "computer_read")?.parameters.properties.limit).toMatchObject({ type: "integer", minimum: 0 });
+ expect(schema(value, "computer_search")?.parameters.properties.limit).toMatchObject({ type: "integer", minimum: 1 });
+ expect(schema(value, "computer_read")?.parameters.properties.path.description).toContain(value.additions.standalonePath);
+ expect(schema(value, "computer_read")?.parameters.properties.views.description).toBe(value.additions.standaloneViews);
+ expect(schema(value, "computer_search")?.parameters.properties.path.description).toBe(value.additions.search);
+ expect(schema(value, "computer_delete")?.parameters.properties.path.description).toBe(value.additions.delete);
+}
+
+test.each([false, true])("standalone resource tools keep deferred contracts and validate before native access (IDE host=%s)", async withIde => {
+ const result = await run("computer-capability-standalone-" + withIde, withIde, withIde ? ["read.core", "search.core", "editor.core"] : [], [
+ take("initial"),
+ call("enable-fallback", "computer_fallback_snapshot", { enableFallback: true }),
+ call("guide", "computer_read", { path: "docs:computer", offset: 1, limit: 1 }),
+ call("invalid-read", "computer_read", { path: "docs:computer", limit: -1 }),
+ call("invalid-search", "computer_search", { path: "computer:machine/local/apps" }),
+ call("close", "computer_delete", { path: "computer:session" }),
+ take("after"),
+ ]);
+ const initial = snapshot(result, "initial");
+ assertFallbackDeclarations(initial);
+ for (const name of fallbackNames) expect(initial.active).not.toContain(name);
+ expect(schema(initial, "ide_capability_computer")).toBeUndefined();
+ expect(active(initial)).not.toContain("computer");
+ const after = snapshot(result, "after");
+ assertFallbackDeclarations(after);
+ for (const name of fallbackNames) expect(after.active).toContain(name);
+ expect(schema(after, "ide_capability_computer")).toBeUndefined();
+ const guide = await readFile(path.resolve("docs/computer-resources.md"), "utf8");
+ expect(getToolResultText(result, "guide").split("\n")[0]).toBe(guide.split("\n")[0]);
+ expect(getToolExecution(result, "guide").isError).toBe(false);
+ expect(getToolExecutionDetails(getToolExecution(result, "guide"))).toMatchObject({ source: "docs:computer", startLine: 1, endLine: 1 });
+ expect(getToolExecution(result, "invalid-read").isError).toBe(true);
+ expect(getToolExecution(result, "invalid-search").isError).toBe(true);
+ expect(getToolExecution(result, "close").isError).toBe(false);
+ expect(getToolExecutionDetails(getToolExecution(result, "close"))).toMatchObject({ metadata: { semanticAction: { ok: true, effect: "applied" } } });
+}, 120_000);
+
+test.each([
+ { disabled: "read.core", accepted: ["search", "delete"], rejected: "read" },
+ { disabled: "search.core", accepted: ["read", "delete"], rejected: "search" },
+ { disabled: "editor.core", accepted: ["read", "search"], rejected: "delete" },
+])("partial IDE keeps only accepted resource metadata deferred ($disabled disabled)", async ({ disabled, accepted, rejected }) => {
+ const result = await run("computer-capability-partial-" + disabled, true, [disabled], [
+ take("initial"),
+ discovery("load", "ide_capability_computer"),
+ take("activated"),
+ call("enable-fallback", "computer_fallback_snapshot", { enableFallback: true }),
+ call("guide", "computer_read", { path: "docs:computer", offset: 1, limit: 1 }),
+ take("after-fallback"),
+ ]);
+ const initial = snapshot(result, "initial");
+ assertFallbackDeclarations(initial);
+ expect(active(initial)).not.toContain("computer");
+ for (const name of fallbackNames) expect(initial.active).not.toContain(name);
+ expect(schema(initial, "ide_capability_computer")?.exposure).toBe("deferred");
+ const activated = snapshot(result, "activated");
+ expect(active(activated)).toContain("computer");
+ expect(schema(activated, rejected)).toEqual(schema(initial, rejected));
+ if (accepted.includes("read")) {
+ for (const field of ["path", "views"]) {
+ if (initial.additions[field]) expect(schema(initial, "read")?.parameters.properties[field].description).not.toContain(initial.additions[field]);
+ expect(schema(activated, "read")?.parameters.properties[field].description).toContain(activated.additions[field]);
+ }
+ }
+ if (accepted.includes("search")) {
+ expect(schema(initial, "search")?.promptGuidelines ?? []).not.toContain(initial.additions.search);
+ if (activated.additions.search) expect(schema(activated, "search")?.promptGuidelines).toContain(activated.additions.search);
+ }
+ if (accepted.includes("delete")) {
+ if (initial.additions.delete) expect(schema(initial, "delete")?.parameters.properties.path.description).not.toContain(initial.additions.delete);
+ expect(schema(activated, "delete")?.parameters.properties.path.description).toContain(activated.additions.delete);
+ }
+ assertFallbackDeclarations(snapshot(result, "after-fallback"));
+ expect(getToolExecution(result, "guide").isError).toBe(false);
+ expect(getToolExecutionDetails(getToolExecution(result, "guide"))).toMatchObject({ source: "docs:computer", startLine: 1, endLine: 1 });
+}, 120_000);
diff --git a/test/fixtures/computer-capability-fallback.ts b/test/fixtures/computer-capability-fallback.ts
new file mode 100644
index 0000000..012079c
--- /dev/null
+++ b/test/fixtures/computer-capability-fallback.ts
@@ -0,0 +1,25 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { computerPathHelp, computerViewsHelp, COMPUTER_SEARCH_HELP, COMPUTER_DELETE_HELP } from "../../src/provider.ts";
+import { IDE_TOOL_NAMES, STANDALONE_TOOL_NAMES } from "../../src/tool-names.ts";
+
+export default function fallbackSnapshot(pi: ExtensionAPI): void {
+ pi.registerTool({
+ name: "computer_fallback_snapshot", label: "Computer fallback snapshot",
+ description: "Inspect computer fallback declarations and capability state without accessing native resources.",
+ exposure: "model-only", parameters: Type.Object({ enableFallback: Type.Optional(Type.Boolean()) }),
+ async execute(_id, input, _signal, _update, ctx) {
+ // Select known declarations explicitly so these contract checks do not depend on BM25 prose ranking.
+ if (input.enableFallback) pi.setActiveTools([...new Set([...pi.getActiveTools(), "computer_read", "computer_search", "computer_delete"])]);
+ const entries = ctx.sessionManager.getBranch().filter(entry => entry.type === "custom" && entry.customType === "ide-capabilities/v1");
+ return {
+ content: [{ type: "text", text: JSON.stringify({
+ tools: pi.getAllTools().filter(tool => ["read", "search", "delete", "computer_read", "computer_search", "computer_delete", "ide_capability_computer"].includes(tool.name)),
+ active: pi.getActiveTools(), entries,
+ additions: { standalonePath: computerPathHelp(STANDALONE_TOOL_NAMES), standaloneViews: computerViewsHelp(STANDALONE_TOOL_NAMES), path: computerPathHelp(IDE_TOOL_NAMES), views: computerViewsHelp(IDE_TOOL_NAMES), search: COMPUTER_SEARCH_HELP, delete: COMPUTER_DELETE_HELP },
+ }) }],
+ details: undefined,
+ };
+ },
+ });
+}
diff --git a/test/fixtures/computer-capability-snapshot.ts b/test/fixtures/computer-capability-snapshot.ts
new file mode 100644
index 0000000..8a4d95e
--- /dev/null
+++ b/test/fixtures/computer-capability-snapshot.ts
@@ -0,0 +1,28 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+import { computerPathHelp, computerViewsHelp, COMPUTER_SEARCH_HELP, COMPUTER_DELETE_HELP } from "../../src/provider.ts";
+import { IDE_TOOL_NAMES } from "../../src/tool-names.ts";
+
+export default function snapshot(pi: ExtensionAPI): void {
+ pi.registerTool({
+ name: "computer_capability_snapshot", label: "Capability snapshot",
+ description: "Inspect the shared tools and persisted capability state without starting native resources.",
+ exposure: "model-only", parameters: Type.Object({}),
+ async execute(_id, _input, _signal, _update, ctx) {
+ const tools = pi.getAllTools();
+ const entries = ctx.sessionManager.getBranch().filter(entry => entry.type === "custom" && entry.customType === "ide-capabilities/v1");
+ return {
+ content: [{ type: "text", text: JSON.stringify({
+ tools: tools.filter(tool => ["read", "search", "delete"].includes(tool.name)).map(tool => ({
+ name: tool.name, description: tool.description, parameters: tool.parameters, promptGuidelines: tool.promptGuidelines ?? [],
+ })),
+ activeTools: pi.getActiveTools(),
+ computerTools: tools.filter(tool => tool.namespace?.name === "computer").map(tool => tool.name),
+ entries,
+ additions: { path: computerPathHelp(IDE_TOOL_NAMES), views: computerViewsHelp(IDE_TOOL_NAMES), search: COMPUTER_SEARCH_HELP, delete: COMPUTER_DELETE_HELP },
+ }) }],
+ details: undefined,
+ };
+ },
+ });
+}