Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: integration checks use element crops for window detail and load computer_view (LPT-931)

alexshpunt-clanker[bot]c3ed372ad9f7
diff --git a/test/extension-modes.integration.test.ts b/test/extension-modes.integration.test.ts
index b5ec8e1..5fb5218 100644
--- a/test/extension-modes.integration.test.ts
+++ b/test/extension-modes.integration.test.ts
@@ -52,7 +52,7 @@ test("without Pi Agent IDE, tool_search finds the deferred computer tools and th
// tool_search loads the best matches of each query, at most 8 by default, so the guide names one query per cluster.
// A query lists only tools it newly loads; together the guide's cluster queries load every computer tool.
const loaded = ["find", "open", "more", "files", "input", "keys"].map(id => getToolResultText(result, id)).join("\n");
- for (const tool of ["computer_search", "computer_delete", "computer_open", "computer_window", "computer_display",
+ for (const tool of ["computer_search", "computer_delete", "computer_open", "computer_window", "computer_view", "computer_display",
"computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record", "computer_click", "computer_hover", "computer_drag",
"computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);
// Input goes to a window address and refuses anything else before native work.
@@ -104,7 +104,7 @@ test("with Pi Agent IDE, computer use extends Read, Search and Delete, refuses W
const loaded = ["find", "open", "more", "files", "input", "keys"].map(id => getToolResultText(result, id)).join("\n");
expect(loaded).not.toContain("computer_read");
expect(loaded).not.toContain("computer_delete");
- for (const tool of ["computer_open", "computer_window", "computer_display", "computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record",
+ for (const tool of ["computer_open", "computer_window", "computer_view", "computer_display", "computer_wait", "computer_browser", "computer_files", "computer_clipboard", "computer_record",
"computer_click", "computer_hover", "computer_drag", "computer_scroll", "computer_type", "computer_press", "computer_select", "computer_timeline"]) expect(loaded).toContain(`- ${tool}:`);
for (const id of ["session", "close", "delete"])
expect(getToolExecution(result, id).isError, getToolResultText(result, id)).toBe(false);
diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.ts
index 7b319b4..b60a3d8 100644
--- a/test/fixtures/coordinate-scroll-extension.ts
+++ b/test/fixtures/coordinate-scroll-extension.ts
@@ -135,8 +135,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
{ amount: 1 }, { button: "left" }, { modifiers: ["ctrl"] }]) await rejected(first, extra);
assert((await journal()).filter(row => row.name === "scroll_at").length === sent, "An invalid scroll reached native input");
} else {
- // A detail crop is readonly: it grants no input and issues no observation.
- const cropped = await ok("read", { path: reference(view, "observation"), views: ["region:50,50,100,100"] });
+ // An element crop is readonly: it grants no input and issues no observation.
+ const cropped = await ok("read", { path: reference(view, "observation"), views: [`element:crop,${field(view.details, "hint")}`] });
assert(cropped.content.some(block => block.type === "image" && block.mimeType === "image/jpeg") && field(cropped.details, "input") === "none" &&
field(cropped.details, "observation_id") === undefined, "Detail granted input authority");
await control("capability_off"); await observe(first);
diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.ts
index e2aed0a..2a4c5ad 100644
--- a/test/fixtures/ide-provider-extension.ts
+++ b/test/fixtures/ide-provider-extension.ts
@@ -113,8 +113,8 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
let observation = reference(view, "observation");
if (scenario === "workflow") {
const nativeBeforeDetail = (await captures()).length;
- const detail = await ok("read", { path: observation, views: ["region:80,60,40,20"] });
- assert(detail.content.some(block => block.type === "image" && block.mimeType === "image/jpeg"), "Rectangular Read did not return native detail");
+ const detail = await ok("read", { path: observation, views: [`element:crop,${hintOf(view)}`] });
+ assert(detail.content.some(block => block.type === "image" && block.mimeType === "image/jpeg"), "Element crop did not return native detail");
assert((await captures()).length === nativeBeforeDetail, "Detail silently created a new grounding overview");
await refused("insert", { path: observation, text: JSON.stringify({ action: "click", x: 100, y: 80 }) }, "computer_click");
view = await ok("read", { path: first });
diff --git a/test/ide-provider.integration.test.ts b/test/ide-provider.integration.test.ts
index d42052f..e0b7b82 100644
--- a/test/ide-provider.integration.test.ts
+++ b/test/ide-provider.integration.test.ts
@@ -56,7 +56,7 @@ async function runScenario(providerFirst: boolean, scenario: "workflow" | "admis
for (const row of inputs) expect(row.arguments.target).toEqual({ kind: "window", pid: 42, window_id: 91 });
if (scenario === "workflow") {
expect(inputs[0].arguments).toMatchObject({ text: ' {"looks":"like JSON"}\n"quoted text"\n ', delay_ms: 0, delivery_mode: "foreground", element_token: expect.any(String) });
- expect(calls.filter(row => row.name === "zoom")).toEqual([{ name: "zoom", arguments: { pid: 42, window_id: 91, x1: 80, y1: 60, x2: 120, y2: 80 } }]);
+ expect(calls.filter(row => row.name === "zoom")).toEqual([{ name: "zoom", arguments: { pid: 42, window_id: 91, x1: 80, y1: 60, x2: 160, y2: 100 } }]);
expect(details.localText).toBe("Local source stays local\nSecond local line");
}
expect(result.tuiRenderedOutput).not.toContain("Failed to load extension");
diff --git a/test/unified-tools.integration.test.ts b/test/unified-tools.integration.test.ts
index c0ac0a6..79da4a5 100644
--- a/test/unified-tools.integration.test.ts
+++ b/test/unified-tools.integration.test.ts
@@ -31,7 +31,7 @@ test("production Pi exposes only computer tools, with no direct or Codemode brow
}).run("Check the production loadout. Only discover apps; never open a window or send input.");
// Without Pi Agent IDE, production computer use offers deferred tools that tool_search loads.
const standalone = ["computer_browser", "computer_click", "computer_clipboard", "computer_delete", "computer_display", "computer_drag", "computer_files", "computer_hover", "computer_open",
- "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_wait", "computer_window"];
+ "computer_press", "computer_read", "computer_record", "computer_scroll", "computer_search", "computer_select", "computer_timeline", "computer_type", "computer_view", "computer_wait", "computer_window"];
expect(getToolExecutionDetails(getToolExecution(result, "registry"))).toEqual({
names: standalone,
exposure: Object.fromEntries(standalone.map(name => [name, "deferred"])),