Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: integration checks use element crops for window detail and load computer_view (LPT-931)
alexshpunt-clanker[bot]c3ed372ad9f7
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.tsindex b5ec8e1..5fb5218 100644--- a/test/extension-modes.integration.test.ts+++ b/test/extension-modes.integration.test.ts@@ -52,7 +52,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th// tool_search loads the best matches of each query, at most 8 by default, so the guide names one query per cluster.// A query lists only tools it newly loads; together the guide's cluster queries load every computer tool.const loaded = ["find", "open", "more", "files", "input", "keys"].map(id => getToolResultText(result, id)).join("\n");- for (const tool of ["computer_search", "computer_delete", "computer_open", "computer_window", "computer_display",+ for (const tool of ["computer_search", "computer_delete", "computer_open", "computer_window", "computer_view", "computer_display","computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag","computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);// Input goes to a window address and refuses anything else before native work.@@ -104,7 +104,7 @@ test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses Wconst loaded = ["find", "open", "more", "files", "input", "keys"].map(id => getToolResultText(result, id)).join("\n");expect(loaded).not.toContain("computer_read");expect(loaded).not.toContain("computer_delete");- for (const tool of ["computer_open", "computer_window", "computer_display", "computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record",+ for (const tool of ["computer_open", "computer_window", "computer_view", "computer_display", "computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record","computer_click", "computer_hover", "computer_drag", "computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);for (const id of ["session", "close", "delete"])expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.tsindex 7b319b4..b60a3d8 100644--- a/test/fixtures/coordinate-scroll-extension.ts+++ b/test/fixtures/coordinate-scroll-extension.ts@@ -135,8 +135,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {{ amount: 1 }, { button: "left" }, { modifiers: ["ctrl"] }]) await rejected(first, extra);assert((await journal()).filter(row => row.name === "scroll_at").length === sent, "An invalid scroll reached native input");} else {- // A detail crop is readonly: it grants no input and issues no observation.- const cropped = await ok("read", { path: reference(view, "observation"), views: ["region:50,50,100,100"] });+ // An element crop is readonly: it grants no input and issues no observation.+ const cropped = await ok("read", { path: reference(view, "observation"), views: [`element:crop,${field(view.details, "hint")}`] });assert(cropped.content.some(block => block.type === "image" && block.mimeType === "image/jpeg") && field(cropped.details, "input") === "none" &&field(cropped.details, "observation_id") === undefined, "Detail granted input authority");await control("capability_off"); await observe(first);diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.tsindex e2aed0a..2a4c5ad 100644--- a/test/fixtures/ide-provider-extension.ts+++ b/test/fixtures/ide-provider-extension.ts@@ -113,8 +113,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {let observation = reference(view, "observation");if (scenario === "workflow") {const nativeBeforeDetail = (await captures()).length;- const detail = await ok("read", { path: observation, views: ["region:80,60,40,20"] });- assert(detail.content.some(block => block.type === "image" && block.mimeType === "image/jpeg"), "Rectangular Read did not return native detail");+ const detail = await ok("read", { path: observation, views: [`element:crop,${hintOf(view)}`] });+ assert(detail.content.some(block => block.type === "image" && block.mimeType === "image/jpeg"), "Element crop did not return native detail");assert((await captures()).length === nativeBeforeDetail, "Detail silently created a new grounding overview");await refused("insert", { path: observation, text: JSON.stringify({ action: "click", x: 100, y: 80 }) }, "computer_click");view = await ok("read", { path: first });diff --git a/test/ide-provider.integration.test.ts b/test/ide-provider.integration.test.tsindex d42052f..e0b7b82 100644--- a/test/ide-provider.integration.test.ts+++ b/test/ide-provider.integration.test.ts@@ -56,7 +56,7 @@ async function runScenario(providerFirst: boolean, scenario: "workflow" | "admisfor (const row of inputs) expect(row.arguments.target).toEqual({ kind: "window", pid: 42, window_id: 91 });if (scenario === "workflow") {expect(inputs[0].arguments).toMatchObject({ text: ' {"looks":"like JSON"}\n"quoted text"\n ', delay_ms: 0, delivery_mode: "foreground", element_token: expect.any(String) });- expect(calls.filter(row => row.name === "zoom")).toEqual([{ name: "zoom", arguments: { pid: 42, window_id: 91, x1: 80, y1: 60, x2: 120, y2: 80 } }]);+ expect(calls.filter(row => row.name === "zoom")).toEqual([{ name: "zoom", arguments: { pid: 42, window_id: 91, x1: 80, y1: 60, x2: 160, y2: 100 } }]);expect(details.localText).toBe("Local source stays local\nSecond local line");}expect(result.tuiRenderedOutput).not.toContain("Failed to load extension");diff --git a/test/unified-tools.integration.test.ts b/test/unified-tools.integration.test.tsindex c0ac0a6..79da4a5 100644--- a/test/unified-tools.integration.test.ts+++ b/test/unified-tools.integration.test.ts@@ -31,7 +31,7 @@ test("production Pi exposes only computer tools, with no direct or Codemode brow}).run("Check the production loadout. Only discover apps; never open a window or send input.");// Without Pi Agent IDE, production computer use offers deferred tools that tool_search loads.const standalone = ["computer_browser", "computer_click", "computer_clipboard", "computer_delete", "computer_display", "computer_drag", "computer_files", "computer_hover", "computer_open",- "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_wait", "computer_window"];+ "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_view", "computer_wait", "computer_window"];expect(getToolExecutionDetails(getToolExecution(result, "registry"))).toEqual({names: standalone,exposure: Object.fromEntries(standalone.map(name => [name, "deferred"])),