Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test: input, granularity, UI resource and capture-failure integration tests use the computer_* tools (LPT-934)

alexshpunt-clanker[bot]12a7ea3bbd82
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.ts
index 812d810..cfb66be 100644
--- a/test/complete-input.integration.test.ts
+++ b/test/complete-input.integration.test.ts
@@ -5,37 +5,27 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTex
import { expect, test } from "vitest";
import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
-test("real Pi Codemode validates a whole sequence, forwards complete steps and stops without replay", async () => {
+test("real Pi sends keys, chords and drags in the background and keeps partial and refused outcomes apart", async () => {
await mkdir(".tmp/cua-gestures", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/cua-gestures/pi-contract-")); const log = path.join(cwd, "calls.jsonl");
await mkdir(path.join(cwd, ".pi"));
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],
}));
- // Any refused input spends its observation, so the valid sequence acts on a fresh overview.
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
if (opened.outcome !== "observed" || !opened.window || !opened.observation) throw Error("Open returned no window and observation");
if (!(await windows()).includes(opened.window)) throw Error("Discovery lost the opened window");
-// These checks keep one observation for pixel input, so they turn the default successor off.
-const act = async (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});
-const invalid = await act(opened.observation, [{action:"key",key:"a"},{action:"key",key:"a",duration_ms:100}]);
-if (invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");
-const current = observation(await tools.computer_read({path:opened.window}));
-const success = await act(current, [{action:"key",key:"a"},{action:"hotkey",keys:["shift","b"],wait_ms:10},
- {action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4}]);
-if (success.outcome !== "delivered" || success.steps?.length !== 3) throw Error("Missing completed sequence");
-text(JSON.stringify(success.steps));
-const retired = await act(current, [{action:"key",key:"c"}]);
-if (retired.outcome !== "refused") throw Error("Overview still armed");
-for (const key of ["escape","delete"]) {
- const view = observation(await tools.computer_read({path:opened.window}));
- const stopped = await act(view, [{action:"key",key:"a"},{action:"key",key},{action:"key",key:"c"}]);
- if (stopped.outcome !== (key === "escape" ? "unknown" : "refused") || stopped.effect !== "unknown" || stopped.steps?.length !== 2) throw Error("False completed prefix");
- text(JSON.stringify(stopped.steps));
- const stale = await act(view, [{action:"key",key:"c"}]);
- if (stale.outcome !== "refused") throw Error("Stopped sequence could replay");
-}
+const press = (key, extra) => tools.computer_press({window:opened.window,key,...extra});
+for (const key of ["a","shift+b"]) if ((await press(key)).outcome !== "delivered") throw Error("Key refused: " + key);
+const drag = await tools.computer_drag({window:opened.window,from:{x:30,y:40},to:{x:100,y:80},duration_ms:100});
+if (drag.outcome !== "delivered") throw Error("Drag refused: " + drag.text);
+const partial = await press("escape");
+if (partial.outcome !== "unknown" || partial.effect !== "unknown") throw Error("Partial key reported as " + partial.outcome);
+text(partial.text);
+const refused = await press("delete", {foreground:false});
+if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Background refusal reported as " + refused.outcome);
+text(refused.text);
text("Complete sequence guards passed");`;
try {
const result = await new PiIntegrationTest({
@@ -47,7 +37,7 @@ text("Complete sequence guards passed");`;
assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
assistantMessage([text("Done")]),
],
- }).run("Use complete known-target sequences. Never replay a completed prefix or uncertain input.");
+ }).run("Send known keys and a drag. Never replay uncertain input.");
expect(getToolExecution(result, "sequence").isError, getToolResultText(result, "sequence")).toBe(false);
expect(getToolResultText(result, "sequence")).toContain("Complete sequence guards passed");
const nested = nestedCalls(getToolExecutions(result), "sequence");
@@ -55,19 +45,15 @@ text("Complete sequence guards passed");`;
expect(discovery.computer.state.windows[0].source_identity).toEqual(opened.computer.source_identity);
expect(opened.computer.source.identity.session_id).toBe(opened.computer.source_identity.identity.session_id);
expect(opened.computer.source.membership).toMatchObject({ monitor_id: null, virtual_desktop_id: null });
- const sequences = nested.filter(call => call.name === "computer_insert" && call.computer.state?.sequence).map(call => call.computer);
- expect(sequences.map(item => [item.status, item.state.sequence.completed, item.state.sequence.attempted])).toEqual([
- ["delivered", 3, 3], ["unknown", 1, 2], ["refused", 1, 2]]);
- expect(sequences[0].state.sequence.steps.every((step: { action_capability: { background: string; scope: string } }) =>
- step.action_capability.background === "delivered" && step.action_capability.scope === "attempted_action_only")).toBe(true);
- expect(sequences[1].state.sequence.steps[1].action_capability.background).toBe("unknown");
- expect(sequences[2].state.sequence.steps[1].action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });
+ const presses = nested.filter(call => call.name === "computer_press").map(call => call.computer);
+ expect(presses.map(item => item.status)).toEqual(["delivered", "delivered", "unknown", "refused"]);
+ expect(presses[3].action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
const input = calls.filter(c => ["press_key", "hotkey", "drag"].includes(c.name));
- expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key", "press_key", "press_key"]);
- expect(input.map(c => c.arguments.target)).toEqual(Array(7).fill({ kind: "window", pid: 42, window_id: 91 }));
+ expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key"]);
+ expect(input.map(c => c.arguments.key ?? c.arguments.keys)).toEqual(["a", ["shift", "b"], undefined, "escape", "delete"]);
+ expect(input.map(c => c.arguments.target)).toEqual(Array(5).fill({ kind: "window", pid: 42, window_id: 91 }));
expect(input.every(c => c.arguments.delivery_mode === "background")).toBe(true);
- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(4);
expect(calls.some(c => c.name === "bring_to_front")).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }
});
diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.ts
index b623ede..21c03fc 100644
--- a/test/observation-granularity.integration.test.ts
+++ b/test/observation-granularity.integration.test.ts
@@ -5,7 +5,7 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTex
import { expect, test } from "vitest";
import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
-test("real Pi returns only requested window parts and admits only their current input", async () => {
+test("real Pi returns only requested window parts and grounds input in the latest Read", async () => {
await mkdir(".tmp/observe-granularity", { recursive: true });
const cwd = await mkdtemp(path.resolve(".tmp/observe-granularity/pi-contract-"));
const log = path.join(cwd, "calls.jsonl");
@@ -13,32 +13,26 @@ test("real Pi returns only requested window parts and admits only their current
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
command: process.execPath, args: [path.resolve("test/fixtures/observation-granularity-driver.mjs"), log],
}));
- // Any refused input spends its observation, so each refusal check uses a fresh overview of the requested part.
const code = `${SCRIPT_HELPERS}
const win = address(await windows(), "window");
const read = views => tools.computer_read({path:win,...(views?{views}:{})});
-// Only the reads are under test here: clicks skip the default successor, so every native capture is a read.
-const click = async (view,target) => (await insert(observation(view),{action:"click",...target,successor:false})).outcome;
+const click = async target => (await tools.computer_click({window:win,...target})).outcome;
const full=await read();
if(typeof full==="string"||!hints(full).length)throw Error("Full default changed");
const fullHint=hints(full)[0];
-let image=await read(["image"]);
+const image=await read(["image"]);
if(typeof image==="string"||hints(image).length||/^UI:/m.test(image.note))throw Error("Image inherited UI");
-if(await click(image,{hint:fullHint})!=="refused")throw Error("Retired hint admitted");
-image=await read(["image"]);
-// The peer keeps its first snapshot's native token; an image-only overview must not admit it.
-if(await click(image,{element_token:"s1:0"})!=="refused")throw Error("Retained native token admitted");
-image=await read(["image"]);
-if(await click(image,{x:75,y:60})!=="delivered")throw Error("Image pixel input missing");
-if(await click(image,{x:75,y:60})!=="refused")throw Error("Consumed image replayed");
+if(await click({target:fullHint})!=="refused")throw Error("A ref of an earlier Read admitted");
+if(await click({x:75,y:60})!=="delivered")throw Error("Image pixel input missing");
let ui=await read(["ui"]);
if(typeof ui!=="string"||!/image_bounds=null/.test(ui))throw Error("UI inherited pixels");
-if(await click(ui,{x:75,y:60})!=="refused")throw Error("UI pixel input admitted");
+// A point needs the window's pixels: after a UI-only Read the click captures the window first.
+if(await click({x:75,y:60})!=="delivered")throw Error("Point after a UI Read refused");
ui=await read(["ui"]);
const inspected=await tools.computer_read({path:observation(ui),views:["element:inspect,"+hints(ui)[0]]});
if(typeof inspected!=="string"||!/\\| Button \\| Save \\|/.test(inspected))throw Error("UI inspection changed");
-if(await click(ui,{hint:hints(ui)[0]})!=="delivered")throw Error("UI hint missing");
-text("Three mode contracts passed; exactly two native clicks.");
+if(await click({target:hints(ui)[0]})!=="delivered")throw Error("UI ref missing");
+text("Three mode contracts passed; exactly three native clicks.");
`;
try {
const result = await new PiIntegrationTest({
@@ -54,19 +48,19 @@ text("Three mode contracts passed; exactly two native clicks.");
expect(getToolExecution(result, "parts").isError, getToolResultText(result, "parts")).toBe(false);
expect(getToolResultText(result, "parts")).toContain("Three mode contracts passed");
const reads = nestedCalls(getToolExecutions(result), "parts").filter(call => call.name === "computer_read" && call.computer.granularity);
- expect(reads.map(call => call.computer.granularity)).toEqual(["full", "image", "image", "image", "ui", "ui"]);
+ expect(reads.map(call => call.computer.granularity)).toEqual(["full", "image", "ui", "ui"]);
for (const call of reads.filter(call => call.computer.granularity === "image")) {
expect(call.computer.source.semantic_scope).toBe("none");
expect(call.computer.ui).toBeUndefined();
}
for (const call of reads.filter(call => call.computer.granularity === "ui")) expect(call.computer.agent_image).toBeUndefined();
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
- const native = calls.filter(c => c.name === "get_window_state");
- expect(native).toHaveLength(6);
- for (const index of [1, 2, 3]) expect(native[index].arguments.include_accessibility_tree).toBe(false);
- for (const index of [4, 5]) expect(native[index].arguments.include_screenshot).toBe(false);
- expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token ?? [c.arguments.x,c.arguments.y]))
- .toEqual([[75,60],"s6:0"]);
+ const clicks = calls.flatMap((c, index) => c.name === "click" ? [index] : []);
+ expect(clicks.map(index => calls[index].arguments.element_token ?? [calls[index].arguments.x, calls[index].arguments.y]))
+ .toEqual([[75, 60], [75, 60], expect.stringMatching(/^s\d+:0$/)]);
+ // The second point click captured pixels after the UI-only Read before it clicked.
+ const before = calls.slice(0, clicks[1]).filter(c => c.name === "get_window_state").at(-1);
+ expect(before.arguments.include_screenshot).not.toBe(false);
expect(calls.some(c => c.name === "zoom" || c.name === "bring_to_front")).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }
});
diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.ts
index c40eef8..f11233e 100644
--- a/test/ui-resources.integration.test.ts
+++ b/test/ui-resources.integration.test.ts
@@ -12,13 +12,11 @@ test("real Pi Codemode refines full resources locally and dispatches only a curr
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
command: process.execPath, args: [path.resolve("test/fixtures/ui-resources-driver.mjs"), log],
}));
- // Any refused input spends its observation, so each refusal check acts on a fresh overview.
const code = `${SCRIPT_HELPERS}
const listed = await windows();
const [win, otherWin] = [...listed.matchAll(/computer:[\\w-]+\\/w\\d+[a-z]{2}/g)].map(match => match[0]);
const read = views => tools.computer_read({path:win,...(views?{views}:{})});
-// Only local refinement is under test here: clicks skip the default successor, so every native capture is a read.
-const click = async (view,hint) => (await insert(observation(view),{action:"click",hint,successor:false})).outcome;
+const click = async (window,target) => (await tools.computer_click({window,target})).outcome;
const first = await read(['capture:{"max_elements":200,"max_depth":15,"guides":{"grid":true,"action":true}}']);
if (!/UI: collected=200, complete=false/.test(body(first))) throw Error("Lost full resource");
const candidates = await tools.computer_search({path:observation(first),query:"Save",limit:2});
@@ -28,17 +26,16 @@ const inspect = await tools.computer_read({path:observation(first),views:["eleme
if (typeof inspect !== "string" || hints(inspect)[0] !== uiHint) throw Error("Inspection rebound target");
const marked = await tools.computer_read({path:observation(first),views:["element:highlight,"+uiHint]});
if (typeof marked === "string") throw Error("Missing selected image");
-if ((await click(first, disabledHint)) !== "refused") throw Error("Disabled input admitted");
+if ((await click(win, disabledHint)) !== "refused") throw Error("Disabled input admitted");
const current = await read();
const other = await tools.computer_read({path:otherWin});
-if ((await click(current, hints(other)[1])) !== "refused") throw Error("Foreign hint admitted");
+if ((await click(win, hints(other)[1])) !== "refused") throw Error("Foreign hint admitted");
let latest = await read(['capture:{"max_elements":3,"hints":false}']);
-if ((await click(latest, uiHint)) !== "refused") throw Error("Old alias rebound");
+if ((await click(win, uiHint)) !== "refused") throw Error("Old alias rebound");
latest = await read(['capture:{"max_elements":3,"hints":false}']);
-const target = observation(latest);
-const delivered = await insert(target,{action:"click",hint:hints(latest)[1],successor:false});
-if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched");
-if ((await insert(target,{action:"click",hint:hints(latest)[1],successor:false})).outcome !== "refused") throw Error("Consumed hint replayed");
+const delivered = await tools.computer_click({window:win,target:hints(latest)[1]});
+if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched: " + delivered.text);
+if ((await click(win, hints(latest)[1])) !== "refused") throw Error("Spent ref replayed");
text("UI resource guards passed: 200 collected, 199 search matches, one native click");`;
try {
const result = await new PiIntegrationTest({
@@ -64,10 +61,12 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native c
expect(inspect.computer.agent_image).toBeUndefined();
expect(inspect.computer.ui.elements[0].element_token).toBe(search.computer.ui.elements[0].element_token);
expect(marked.computer.agent_image.annotation.selected_hint).toBe(search.computer.ui.elements[0].hint);
- const delivered = nested.filter(call => call.name === "computer_insert" && call.computer.status === "delivered");
+ const delivered = nested.filter(call => call.name === "computer_click" && call.computer.status === "delivered");
expect(delivered.map(call => call.computer.state.path)).toEqual(["ax"]);
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(5);
+ // Five Reads ground the one click; the captures after it show what the click changed.
+ const click = calls.findIndex(c => c.name === "click");
+ expect(calls.slice(0, click).filter(c => c.name === "get_window_state")).toHaveLength(5);
expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token)).toEqual(["s5:1"]);
expect(calls.some(c => ["bring_to_front", "scroll", "zoom"].includes(c.name))).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }
diff --git a/test/window-capture-failure.integration.test.ts b/test/window-capture-failure.integration.test.ts
index 65a046f..b5a790e 100644
--- a/test/window-capture-failure.integration.test.ts
+++ b/test/window-capture-failure.integration.test.ts
@@ -19,7 +19,7 @@ if(typeof before==="string")throw Error("Missing first image");
let failure="admitted";
try{await tools.computer_read({path:win});}catch(error){failure=String(error.message);}
if(failure==="admitted")throw Error("Failed capture returned a result");
-const replay=await insert(observation(before),{action:"click",x:20,y:20});
+const replay=await tools.computer_click({window:win,x:20,y:20});
if(replay.outcome!=="refused")throw Error("Failed capture retained old input");
const ui=await tools.computer_read({path:win,views:["ui"]});
if(typeof ui!=="string"||!/^Observation: /m.test(ui))throw Error("UI-only inherited failed pixels");
@@ -47,7 +47,8 @@ text("Minimized screenshot refused with native cause; old input refused; UI-only
expect(ui.computer.agent_image).toBeUndefined();
expect(ui.computer.state.screenshot_error).toBeUndefined();
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(3);
+ // The click first tried to capture the pixels its point needs; that capture failed too, so nothing was sent.
+ expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(4);
expect(calls.every(c => ["fixture_identity", "list_windows", "get_window_state"].includes(c.name))).toBe(true);
} finally { await rm(cwd, { recursive: true, force: true }); }
});