Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Adapt tests to pixel successors (LPT-915)
alexshpunt-clanker[bot]224294382752
diff --git a/test/capture-drag.integration.test.ts b/test/capture-drag.integration.test.tsindex 2768d7b..e6e7559 100644--- a/test/capture-drag.integration.test.ts+++ b/test/capture-drag.integration.test.ts@@ -13,7 +13,8 @@ test("real Pi keeps drag capture binding and stops on resize refusal until reobsargs: [path.resolve("test/fixtures/complete-input-driver.mjs"), log, "capture-drag"] }));const code = `${SCRIPT_HELPERS}const opened = await openApp("Notepad");-const act = (view, sequence) => insert(view, {action:"sequence",sequence});+// These checks keep one observation for pixel input, so they turn the default successor off.+const act = (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});const drag={action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4};const stopped=await act(opened.observation,[{action:"key",key:"r"},drag,{action:"key",key:"b"}]);if(stopped.outcome!=="refused" || stopped.code!=="capture_frame_mismatch" || stopped.effect!=="unknown") throw Error("Resize refusal or completed prefix lost");diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.tsindex 84ea1c2..a905053 100644--- a/test/complete-input.integration.test.ts+++ b/test/complete-input.integration.test.ts@@ -17,7 +17,8 @@ test("real Pi Codemode validates a whole sequence, forwards complete steps and sconst opened = await openApp("Notepad");if (opened.outcome !== "observed" || !opened.window || !opened.observation) throw Error("Open returned no window and observation");if (!(await windows()).includes(opened.window)) throw Error("Discovery lost the opened window");-const act = async (view, sequence) => insert(view, {action:"sequence",sequence});+// These checks keep one observation for pixel input, so they turn the default successor off.+const act = async (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});const invalid = await act(opened.observation, [{action:"key",key:"a"},{action:"key",key:"a",duration_ms:100}]);if (invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");const current = observation(await tools.computer_read({path:opened.window}));diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.tsindex 39b478b..9c34e28 100644--- a/test/fixtures/coordinate-scroll-extension.ts+++ b/test/fixtures/coordinate-scroll-extension.ts@@ -74,6 +74,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {};const journal = async () => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));const captures = async () => (await journal()).filter(row => row.name === "get_window_state");+ // Whether a call captured the window before its first native input. The action's successor captures it after the+ // input (LPT-914, LPT-915); a capture before it would silently refresh the input's authority.+ const capturedBeforeInput = async (from: number) => {+ const rows = (await journal()).slice(from), input = rows.findIndex(row => row.name === "scroll_at");+ return rows.slice(0, input < 0 ? rows.length : input).some(row => row.name === "get_window_state");+ };const control = async (operation: string) => {if (operation === "safe") mode = "safe";else if (operation === "capability_off") capability = false;@@ -85,9 +91,9 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {views: [granularity, ...(granularity === "ui" ? [] : ['capture:{"hints":false}'])] });const command = (extra: Record<string, unknown> = {}) => ({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents", ...extra });const act = async (observation: string, command: Record<string, unknown>) => {- const before = (await captures()).length;+ const before = (await journal()).length;const result = await call("insert", { path: observation, text: JSON.stringify(command) });- assert((await captures()).length === before, "Insert resolution silently refreshed native capture authority");+ assert(!await capturedBeforeInput(before), "Insert resolution silently refreshed native capture authority");return result;};const rejected = async (observation: string, command: Record<string, unknown>) => {diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.tsindex 94ba2b8..4ef57dd 100644--- a/test/fixtures/ide-provider-extension.ts+++ b/test/fixtures/ide-provider-extension.ts@@ -81,6 +81,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {const journal = async () => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));const inputs = async () => (await journal()).filter(row => ["type_text", "click", "press_key", "hotkey"].includes(row.name));const captures = async () => (await journal()).filter(row => row.name === "get_window_state");+ // Whether a call captured the window before its first native input. The action's successor captures it after the+ // input (LPT-914, LPT-915); a capture before it would silently refresh the input's authority.+ const capturedBeforeInput = async (from: number) => {+ const rows = (await journal()).slice(from), input = rows.findIndex(row => ["type_text", "click", "press_key", "hotkey"].includes(row.name));+ return rows.slice(0, input < 0 ? rows.length : input).some(row => row.name === "get_window_state");+ };const control = async (operation: string) => {if (operation === "safe") mode = "safe";else await driver!.call("fixture_control", { operation }, signal);@@ -106,17 +112,16 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {assert((await inputs()).length === beforeDenied, "Consumed detail observation dispatched input");view = await ok("read", { path: first }); observation = reference(view, "observation");const hint = hintOf(view), literal = ' {"looks":"like JSON"}\n"quoted text"\n ';- const nativeBeforeWrite = (await captures()).length;+ const nativeBeforeWrite = (await journal()).length;await ok("write", { path: `${observation}?hint=${encodeURIComponent(hint)}&delivery=foreground&delay_ms=0`, content: literal });- // Only the successor's UI-only walks follow the input; no pixel capture refreshes input authority.- assert((await captures()).slice(nativeBeforeWrite).every(row => row.arguments?.include_screenshot === false), "Write resolution refreshed native capture");+ assert(!await capturedBeforeInput(nativeBeforeWrite), "Write resolution refreshed native capture");const written = (await inputs()).at(-1);assert(written?.name === "type_text" && written.arguments.text === literal && written.arguments.delay_ms === 0, "Literal Write normalized text, added a command or lost delay");await denied("write", { path: observation, content: "replay" });view = await ok("read", { path: first }); observation = reference(view, "observation");- const nativeBeforeInsert = (await captures()).length;+ const nativeBeforeInsert = (await journal()).length;await ok("insert", { path: observation, text: JSON.stringify({ action: "click", hint: hintOf(view) }) });- assert((await captures()).slice(nativeBeforeInsert).every(row => row.arguments?.include_screenshot === false), "Insert resolution refreshed native capture");+ assert(!await capturedBeforeInput(nativeBeforeInsert), "Insert resolution refreshed native capture");const uiOnly = await ok("read", { path: first, views: ["ui"] });assert(!uiOnly.content.some(block => block.type === "image"), "UI text view included an image");assert(textOf(uiOnly).includes("Fixture text field") && textOf(uiOnly).includes("Fixture button"), "UI text view lost observed structure");@@ -158,12 +163,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {assert((await inputs()).length === 0, "Invalid/Safe/display requests sent native input");view = await ok("read", { path: first }); observation = reference(view, "observation");await control("partial");- const beforePartial = (await captures()).length;+ const beforePartial = (await journal()).length;const partial = await denied("insert", { path: observation, text: JSON.stringify({ action: "sequence", sequence: [{ action: "type_text", x: 100, y: 80, text: "interrupted prefix" }, { action: "type_text", text: "must not run" },] }) });assert(/unknown|partial|uncertain|interrupted/i.test(textOf(partial)), "Interrupted result lost its uncertainty");- assert((await captures()).length === beforePartial, "Interrupted action resolution refreshed native capture");+ assert(!await capturedBeforeInput(beforePartial), "Interrupted action resolution refreshed native capture");await denied("insert", { path: observation, text: JSON.stringify({ action: "click", x: 100, y: 80 }) });assert((await inputs()).length === 1 && (await inputs())[0].name === "type_text" && (await inputs())[0].arguments.text === "interrupted prefix", "Interrupted prefix was replayed or later sequence input was sent");view = await ok("read", { path: first }); observation = reference(view, "observation");diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.tsindex 357253f..d64b78a 100644--- a/test/observation-granularity.integration.test.ts+++ b/test/observation-granularity.integration.test.ts@@ -17,7 +17,8 @@ test("real Pi returns only requested window parts and admits only their currentconst code = `${SCRIPT_HELPERS}const win = address(await windows(), "window");const read = views => tools.computer_read({path:win,...(views?{views}:{})});-const click = async (view,target) => (await insert(observation(view),{action:"click",...target})).outcome;+// Only the reads are under test here: clicks skip the default successor, so every native capture is a read.+const click = async (view,target) => (await insert(observation(view),{action:"click",...target,successor:false})).outcome;const full=await read();if(typeof full==="string"||!hints(full).length)throw Error("Full default changed");const fullHint=hints(full)[0];@@ -60,11 +61,8 @@ text("Three mode contracts passed; exactly two native clicks.");}for (const call of reads.filter(call => call.computer.granularity === "ui")) expect(call.computer.agent_image).toBeUndefined();const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));- // The click on a hint is followed by the successor's UI-only walks (LPT-914); the six reads come first.- const all = calls.filter(c => c.name === "get_window_state"), lastClick = calls.findLastIndex(c => c.name === "click");- const native = all.filter(c => calls.indexOf(c) < lastClick);+ const native = calls.filter(c => c.name === "get_window_state");expect(native).toHaveLength(6);- expect(all.slice(6).every(c => c.arguments.include_screenshot === false)).toBe(true);for (const index of [1, 2, 3]) expect(native[index].arguments.include_accessibility_tree).toBe(false);for (const index of [4, 5]) expect(native[index].arguments.include_screenshot).toBe(false);expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token ?? [c.arguments.x,c.arguments.y]))diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.tsindex 9921974..9605dd2 100644--- a/test/ui-resources.integration.test.ts+++ b/test/ui-resources.integration.test.ts@@ -17,7 +17,8 @@ test("real Pi Codemode refines full resources locally and dispatches only a currconst listed = await windows();const [win, otherWin] = [...listed.matchAll(/computer:window\\/[\\w-]+/g)].map(match => match[0]);const read = views => tools.computer_read({path:win,...(views?{views}:{})});-const click = async (view,hint) => (await insert(observation(view),{action:"click",hint})).outcome;+// Only local refinement is under test here: clicks skip the default successor, so every native capture is a read.+const click = async (view,hint) => (await insert(observation(view),{action:"click",hint,successor:false})).outcome;const first = await read(['capture:{"max_elements":200,"max_depth":15,"guides":{"grid":true,"action":true}}']);if (!/UI: collected=200, complete=false/.test(body(first))) throw Error("Lost full resource");const candidates = await tools.computer_search({path:observation(first),query:"Save",limit:2});@@ -35,9 +36,9 @@ let latest = await read(['capture:{"max_elements":3,"hints":false}']);if ((await click(latest, uiHint)) !== "refused") throw Error("Old alias rebound");latest = await read(['capture:{"max_elements":3,"hints":false}']);const target = observation(latest);-const delivered = await insert(target,{action:"click",hint:hints(latest)[1]});+const delivered = await insert(target,{action:"click",hint:hints(latest)[1],successor:false});if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched");-if ((await insert(target,{action:"click",hint:hints(latest)[1]})).outcome !== "refused") throw Error("Consumed hint replayed");+if ((await insert(target,{action:"click",hint:hints(latest)[1],successor:false})).outcome !== "refused") throw Error("Consumed hint replayed");text("UI resource guards passed: 200 collected, 199 search matches, one native click");`;try {const result = await new PiIntegrationTest({@@ -66,10 +67,7 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native cconst delivered = nested.filter(call => call.name === "computer_insert" && call.computer.status === "delivered");expect(delivered.map(call => call.computer.state.path)).toEqual(["ax"]);const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));- // Five reads before the click, then the successor's UI-only walk after it (LPT-914).- const walks = calls.filter(c => c.name === "get_window_state"), click = calls.findIndex(c => c.name === "click");- expect(walks.filter(c => calls.indexOf(c) < click)).toHaveLength(5);- expect(walks.filter(c => calls.indexOf(c) > click).every(c => c.arguments.include_screenshot === false)).toBe(true);+ expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(5);expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token)).toEqual(["s5:1"]);expect(calls.some(c => ["bring_to_front", "scroll", "zoom"].includes(c.name))).toBe(false);} finally { await rm(cwd, { recursive: true, force: true }); }