Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test: input, granularity, UI resource and capture-failure integration tests use the computer_* tools (LPT-934)
alexshpunt-clanker[bot]12a7ea3bbd82
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.tsindex 812d810..cfb66be 100644--- a/test/complete-input.integration.test.ts+++ b/test/complete-input.integration.test.ts@@ -5,37 +5,27 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTeximport { expect, test } from "vitest";import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";-test("real Pi Codemode validates a whole sequence, forwards complete steps and stops without replay", async () => {+test("real Pi sends keys, chords and drags in the background and keeps partial and refused outcomes apart", async () => {await mkdir(".tmp/cua-gestures", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/cua-gestures/pi-contract-")); const log = path.join(cwd, "calls.jsonl");await mkdir(path.join(cwd, ".pi"));await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({command: process.execPath, args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log],}));- // Any refused input spends its observation, so the valid sequence acts on a fresh overview.const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");if (opened.outcome !== "observed" || !opened.window || !opened.observation) throw Error("Open returned no window and observation");if (!(await windows()).includes(opened.window)) throw Error("Discovery lost the opened window");-// These checks keep one observation for pixel input, so they turn the default successor off.-const act = async (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});-const invalid = await act(opened.observation, [{action:"key",key:"a"},{action:"key",key:"a",duration_ms:100}]);-if (invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");-const current = observation(await tools.computer_read({path:opened.window}));-const success = await act(current, [{action:"key",key:"a"},{action:"hotkey",keys:["shift","b"],wait_ms:10},- {action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4}]);-if (success.outcome !== "delivered" || success.steps?.length !== 3) throw Error("Missing completed sequence");-text(JSON.stringify(success.steps));-const retired = await act(current, [{action:"key",key:"c"}]);-if (retired.outcome !== "refused") throw Error("Overview still armed");-for (const key of ["escape","delete"]) {- const view = observation(await tools.computer_read({path:opened.window}));- const stopped = await act(view, [{action:"key",key:"a"},{action:"key",key},{action:"key",key:"c"}]);- if (stopped.outcome !== (key === "escape" ? "unknown" : "refused") || stopped.effect !== "unknown" || stopped.steps?.length !== 2) throw Error("False completed prefix");- text(JSON.stringify(stopped.steps));- const stale = await act(view, [{action:"key",key:"c"}]);- if (stale.outcome !== "refused") throw Error("Stopped sequence could replay");-}+const press = (key, extra) => tools.computer_press({window:opened.window,key,...extra});+for (const key of ["a","shift+b"]) if ((await press(key)).outcome !== "delivered") throw Error("Key refused: " + key);+const drag = await tools.computer_drag({window:opened.window,from:{x:30,y:40},to:{x:100,y:80},duration_ms:100});+if (drag.outcome !== "delivered") throw Error("Drag refused: " + drag.text);+const partial = await press("escape");+if (partial.outcome !== "unknown" || partial.effect !== "unknown") throw Error("Partial key reported as " + partial.outcome);+text(partial.text);+const refused = await press("delete", {foreground:false});+if (refused.outcome !== "refused" || refused.effect !== "not-applied") throw Error("Background refusal reported as " + refused.outcome);+text(refused.text);text("Complete sequence guards passed");`;try {const result = await new PiIntegrationTest({@@ -47,7 +37,7 @@ text("Complete sequence guards passed");`;assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),assistantMessage([text("Done")]),],- }).run("Use complete known-target sequences. Never replay a completed prefix or uncertain input.");+ }).run("Send known keys and a drag. Never replay uncertain input.");expect(getToolExecution(result, "sequence").isError, getToolResultText(result, "sequence")).toBe(false);expect(getToolResultText(result, "sequence")).toContain("Complete sequence guards passed");const nested = nestedCalls(getToolExecutions(result), "sequence");@@ -55,19 +45,15 @@ text("Complete sequence guards passed");`;expect(discovery.computer.state.windows[0].source_identity).toEqual(opened.computer.source_identity);expect(opened.computer.source.identity.session_id).toBe(opened.computer.source_identity.identity.session_id);expect(opened.computer.source.membership).toMatchObject({ monitor_id: null, virtual_desktop_id: null });- const sequences = nested.filter(call => call.name === "computer_insert" && call.computer.state?.sequence).map(call => call.computer);- expect(sequences.map(item => [item.status, item.state.sequence.completed, item.state.sequence.attempted])).toEqual([- ["delivered", 3, 3], ["unknown", 1, 2], ["refused", 1, 2]]);- expect(sequences[0].state.sequence.steps.every((step: { action_capability: { background: string; scope: string } }) =>- step.action_capability.background === "delivered" && step.action_capability.scope === "attempted_action_only")).toBe(true);- expect(sequences[1].state.sequence.steps[1].action_capability.background).toBe("unknown");- expect(sequences[2].state.sequence.steps[1].action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });+ const presses = nested.filter(call => call.name === "computer_press").map(call => call.computer);+ expect(presses.map(item => item.status)).toEqual(["delivered", "delivered", "unknown", "refused"]);+ expect(presses[3].action_capability).toMatchObject({ background: "unavailable", foreground: "suggested" });const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));const input = calls.filter(c => ["press_key", "hotkey", "drag"].includes(c.name));- expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key", "press_key", "press_key"]);- expect(input.map(c => c.arguments.target)).toEqual(Array(7).fill({ kind: "window", pid: 42, window_id: 91 }));+ expect(input.map(c => c.name)).toEqual(["press_key", "hotkey", "drag", "press_key", "press_key"]);+ expect(input.map(c => c.arguments.key ?? c.arguments.keys)).toEqual(["a", ["shift", "b"], undefined, "escape", "delete"]);+ expect(input.map(c => c.arguments.target)).toEqual(Array(5).fill({ kind: "window", pid: 42, window_id: 91 }));expect(input.every(c => c.arguments.delivery_mode === "background")).toBe(true);- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(4);expect(calls.some(c => c.name === "bring_to_front")).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.tsindex b623ede..21c03fc 100644--- a/test/observation-granularity.integration.test.ts+++ b/test/observation-granularity.integration.test.ts@@ -5,7 +5,7 @@ import { assistantMessage, getToolExecution, getToolExecutions, getToolResultTeximport { expect, test } from "vitest";import { nestedCalls, SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";-test("real Pi returns only requested window parts and admits only their current input", async () => {+test("real Pi returns only requested window parts and grounds input in the latest Read", async () => {await mkdir(".tmp/observe-granularity", { recursive: true });const cwd = await mkdtemp(path.resolve(".tmp/observe-granularity/pi-contract-"));const log = path.join(cwd, "calls.jsonl");@@ -13,32 +13,26 @@ test("real Pi returns only requested window parts and admits only their currentawait writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({command: process.execPath, args: [path.resolve("test/fixtures/observation-granularity-driver.mjs"), log],}));- // Any refused input spends its observation, so each refusal check uses a fresh overview of the requested part.const code = `${SCRIPT_HELPERS}const win = address(await windows(), "window");const read = views => tools.computer_read({path:win,...(views?{views}:{})});-// Only the reads are under test here: clicks skip the default successor, so every native capture is a read.-const click = async (view,target) => (await insert(observation(view),{action:"click",...target,successor:false})).outcome;+const click = async target => (await tools.computer_click({window:win,...target})).outcome;const full=await read();if(typeof full==="string"||!hints(full).length)throw Error("Full default changed");const fullHint=hints(full)[0];-let image=await read(["image"]);+const image=await read(["image"]);if(typeof image==="string"||hints(image).length||/^UI:/m.test(image.note))throw Error("Image inherited UI");-if(await click(image,{hint:fullHint})!=="refused")throw Error("Retired hint admitted");-image=await read(["image"]);-// The peer keeps its first snapshot's native token; an image-only overview must not admit it.-if(await click(image,{element_token:"s1:0"})!=="refused")throw Error("Retained native token admitted");-image=await read(["image"]);-if(await click(image,{x:75,y:60})!=="delivered")throw Error("Image pixel input missing");-if(await click(image,{x:75,y:60})!=="refused")throw Error("Consumed image replayed");+if(await click({target:fullHint})!=="refused")throw Error("A ref of an earlier Read admitted");+if(await click({x:75,y:60})!=="delivered")throw Error("Image pixel input missing");let ui=await read(["ui"]);if(typeof ui!=="string"||!/image_bounds=null/.test(ui))throw Error("UI inherited pixels");-if(await click(ui,{x:75,y:60})!=="refused")throw Error("UI pixel input admitted");+// A point needs the window's pixels: after a UI-only Read the click captures the window first.+if(await click({x:75,y:60})!=="delivered")throw Error("Point after a UI Read refused");ui=await read(["ui"]);const inspected=await tools.computer_read({path:observation(ui),views:["element:inspect,"+hints(ui)[0]]});if(typeof inspected!=="string"||!/\\| Button \\| Save \\|/.test(inspected))throw Error("UI inspection changed");-if(await click(ui,{hint:hints(ui)[0]})!=="delivered")throw Error("UI hint missing");-text("Three mode contracts passed; exactly two native clicks.");+if(await click({target:hints(ui)[0]})!=="delivered")throw Error("UI ref missing");+text("Three mode contracts passed; exactly three native clicks.");`;try {const result = await new PiIntegrationTest({@@ -54,19 +48,19 @@ text("Three mode contracts passed; exactly two native clicks.");expect(getToolExecution(result, "parts").isError, getToolResultText(result, "parts")).toBe(false);expect(getToolResultText(result, "parts")).toContain("Three mode contracts passed");const reads = nestedCalls(getToolExecutions(result), "parts").filter(call => call.name === "computer_read" && call.computer.granularity);- expect(reads.map(call => call.computer.granularity)).toEqual(["full", "image", "image", "image", "ui", "ui"]);+ expect(reads.map(call => call.computer.granularity)).toEqual(["full", "image", "ui", "ui"]);for (const call of reads.filter(call => call.computer.granularity === "image")) {expect(call.computer.source.semantic_scope).toBe("none");expect(call.computer.ui).toBeUndefined();}for (const call of reads.filter(call => call.computer.granularity === "ui")) expect(call.computer.agent_image).toBeUndefined();const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));- const native = calls.filter(c => c.name === "get_window_state");- expect(native).toHaveLength(6);- for (const index of [1, 2, 3]) expect(native[index].arguments.include_accessibility_tree).toBe(false);- for (const index of [4, 5]) expect(native[index].arguments.include_screenshot).toBe(false);- expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token ?? [c.arguments.x,c.arguments.y]))- .toEqual([[75,60],"s6:0"]);+ const clicks = calls.flatMap((c, index) => c.name === "click" ? [index] : []);+ expect(clicks.map(index => calls[index].arguments.element_token ?? [calls[index].arguments.x, calls[index].arguments.y]))+ .toEqual([[75, 60], [75, 60], expect.stringMatching(/^s\d+:0$/)]);+ // The second point click captured pixels after the UI-only Read before it clicked.+ const before = calls.slice(0, clicks[1]).filter(c => c.name === "get_window_state").at(-1);+ expect(before.arguments.include_screenshot).not.toBe(false);expect(calls.some(c => c.name === "zoom" || c.name === "bring_to_front")).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }});diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.tsindex c40eef8..f11233e 100644--- a/test/ui-resources.integration.test.ts+++ b/test/ui-resources.integration.test.ts@@ -12,13 +12,11 @@ test("real Pi Codemode refines full resources locally and dispatches only a currawait writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({command: process.execPath, args: [path.resolve("test/fixtures/ui-resources-driver.mjs"), log],}));- // Any refused input spends its observation, so each refusal check acts on a fresh overview.const code = `${SCRIPT_HELPERS}const listed = await windows();const [win, otherWin] = [...listed.matchAll(/computer:[\\w-]+\\/w\\d+[a-z]{2}/g)].map(match => match[0]);const read = views => tools.computer_read({path:win,...(views?{views}:{})});-// Only local refinement is under test here: clicks skip the default successor, so every native capture is a read.-const click = async (view,hint) => (await insert(observation(view),{action:"click",hint,successor:false})).outcome;+const click = async (window,target) => (await tools.computer_click({window,target})).outcome;const first = await read(['capture:{"max_elements":200,"max_depth":15,"guides":{"grid":true,"action":true}}']);if (!/UI: collected=200, complete=false/.test(body(first))) throw Error("Lost full resource");const candidates = await tools.computer_search({path:observation(first),query:"Save",limit:2});@@ -28,17 +26,16 @@ const inspect = await tools.computer_read({path:observation(first),views:["elemeif (typeof inspect !== "string" || hints(inspect)[0] !== uiHint) throw Error("Inspection rebound target");const marked = await tools.computer_read({path:observation(first),views:["element:highlight,"+uiHint]});if (typeof marked === "string") throw Error("Missing selected image");-if ((await click(first, disabledHint)) !== "refused") throw Error("Disabled input admitted");+if ((await click(win, disabledHint)) !== "refused") throw Error("Disabled input admitted");const current = await read();const other = await tools.computer_read({path:otherWin});-if ((await click(current, hints(other)[1])) !== "refused") throw Error("Foreign hint admitted");+if ((await click(win, hints(other)[1])) !== "refused") throw Error("Foreign hint admitted");let latest = await read(['capture:{"max_elements":3,"hints":false}']);-if ((await click(latest, uiHint)) !== "refused") throw Error("Old alias rebound");+if ((await click(win, uiHint)) !== "refused") throw Error("Old alias rebound");latest = await read(['capture:{"max_elements":3,"hints":false}']);-const target = observation(latest);-const delivered = await insert(target,{action:"click",hint:hints(latest)[1],successor:false});-if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched");-if ((await insert(target,{action:"click",hint:hints(latest)[1],successor:false})).outcome !== "refused") throw Error("Consumed hint replayed");+const delivered = await tools.computer_click({window:win,target:hints(latest)[1]});+if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched: " + delivered.text);+if ((await click(win, hints(latest)[1])) !== "refused") throw Error("Spent ref replayed");text("UI resource guards passed: 200 collected, 199 search matches, one native click");`;try {const result = await new PiIntegrationTest({@@ -64,10 +61,12 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native cexpect(inspect.computer.agent_image).toBeUndefined();expect(inspect.computer.ui.elements[0].element_token).toBe(search.computer.ui.elements[0].element_token);expect(marked.computer.agent_image.annotation.selected_hint).toBe(search.computer.ui.elements[0].hint);- const delivered = nested.filter(call => call.name === "computer_insert" && call.computer.status === "delivered");+ const delivered = nested.filter(call => call.name === "computer_click" && call.computer.status === "delivered");expect(delivered.map(call => call.computer.state.path)).toEqual(["ax"]);const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(5);+ // Five Reads ground the one click; the captures after it show what the click changed.+ const click = calls.findIndex(c => c.name === "click");+ expect(calls.slice(0, click).filter(c => c.name === "get_window_state")).toHaveLength(5);expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token)).toEqual(["s5:1"]);expect(calls.some(c => ["bring_to_front", "scroll", "zoom"].includes(c.name))).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }diff --git a/test/window-capture-failure.integration.test.ts b/test/window-capture-failure.integration.test.tsindex 65a046f..b5a790e 100644--- a/test/window-capture-failure.integration.test.ts+++ b/test/window-capture-failure.integration.test.ts@@ -19,7 +19,7 @@ if(typeof before==="string")throw Error("Missing first image");let failure="admitted";try{await tools.computer_read({path:win});}catch(error){failure=String(error.message);}if(failure==="admitted")throw Error("Failed capture returned a result");-const replay=await insert(observation(before),{action:"click",x:20,y:20});+const replay=await tools.computer_click({window:win,x:20,y:20});if(replay.outcome!=="refused")throw Error("Failed capture retained old input");const ui=await tools.computer_read({path:win,views:["ui"]});if(typeof ui!=="string"||!/^Observation: /m.test(ui))throw Error("UI-only inherited failed pixels");@@ -47,7 +47,8 @@ text("Minimized screenshot refused with native cause; old input refused; UI-onlyexpect(ui.computer.agent_image).toBeUndefined();expect(ui.computer.state.screenshot_error).toBeUndefined();const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));- expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(3);+ // The click first tried to capture the pixels its point needs; that capture failed too, so nothing was sent.+ expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(4);expect(calls.every(c => ["fixture_identity", "list_windows", "get_window_state"].includes(c.name))).toBe(true);} finally { await rm(cwd, { recursive: true, force: true }); }});