Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Adapt tests to pixel successors (LPT-915)

alexshpunt-clanker[bot]224294382752
diff --git a/test/capture-drag.integration.test.ts b/test/capture-drag.integration.test.ts
index 2768d7b..e6e7559 100644
--- a/test/capture-drag.integration.test.ts
+++ b/test/capture-drag.integration.test.ts
@@ -13,7 +13,8 @@ test("real Pi keeps drag capture binding and stops on resize refusal until reobs
args: [path.resolve("test/fixtures/complete-input-driver.mjs"), log, "capture-drag"] }));
const code = `${SCRIPT_HELPERS}
const opened = await openApp("Notepad");
-const act = (view, sequence) => insert(view, {action:"sequence",sequence});
+// These checks keep one observation for pixel input, so they turn the default successor off.
+const act = (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});
const drag={action:"drag",from_x:30,from_y:40,to_x:100,to_y:80,duration_ms:100,steps:4};
const stopped=await act(opened.observation,[{action:"key",key:"r"},drag,{action:"key",key:"b"}]);
if(stopped.outcome!=="refused" || stopped.code!=="capture_frame_mismatch" || stopped.effect!=="unknown") throw Error("Resize refusal or completed prefix lost");
diff --git a/test/complete-input.integration.test.ts b/test/complete-input.integration.test.ts
index 84ea1c2..a905053 100644
--- a/test/complete-input.integration.test.ts
+++ b/test/complete-input.integration.test.ts
@@ -17,7 +17,8 @@ test("real Pi Codemode validates a whole sequence, forwards complete steps and s
const opened = await openApp("Notepad");
if (opened.outcome !== "observed" || !opened.window || !opened.observation) throw Error("Open returned no window and observation");
if (!(await windows()).includes(opened.window)) throw Error("Discovery lost the opened window");
-const act = async (view, sequence) => insert(view, {action:"sequence",sequence});
+// These checks keep one observation for pixel input, so they turn the default successor off.
+const act = async (view, sequence) => insert(view, {action:"sequence",sequence,successor:false});
const invalid = await act(opened.observation, [{action:"key",key:"a"},{action:"key",key:"a",duration_ms:100}]);
if (invalid.outcome !== "refused" || invalid.effect !== "not-applied") throw Error("Invalid tail admitted");
const current = observation(await tools.computer_read({path:opened.window}));
diff --git a/test/fixtures/coordinate-scroll-extension.ts b/test/fixtures/coordinate-scroll-extension.ts
index 39b478b..9c34e28 100644
--- a/test/fixtures/coordinate-scroll-extension.ts
+++ b/test/fixtures/coordinate-scroll-extension.ts
@@ -74,6 +74,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
};
const journal = async () => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));
const captures = async () => (await journal()).filter(row => row.name === "get_window_state");
+ // Whether a call captured the window before its first native input. The action's successor captures it after the
+ // input (LPT-914, LPT-915); a capture before it would silently refresh the input's authority.
+ const capturedBeforeInput = async (from: number) => {
+ const rows = (await journal()).slice(from), input = rows.findIndex(row => row.name === "scroll_at");
+ return rows.slice(0, input < 0 ? rows.length : input).some(row => row.name === "get_window_state");
+ };
const control = async (operation: string) => {
if (operation === "safe") mode = "safe";
else if (operation === "capability_off") capability = false;
@@ -85,9 +91,9 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
views: [granularity, ...(granularity === "ui" ? [] : ['capture:{"hints":false}'])] });
const command = (extra: Record<string, unknown> = {}) => ({ action: "scroll", x: 100, y: 80, delta_y: 2, unit: "wheel_detents", ...extra });
const act = async (observation: string, command: Record<string, unknown>) => {
- const before = (await captures()).length;
+ const before = (await journal()).length;
const result = await call("insert", { path: observation, text: JSON.stringify(command) });
- assert((await captures()).length === before, "Insert resolution silently refreshed native capture authority");
+ assert(!await capturedBeforeInput(before), "Insert resolution silently refreshed native capture authority");
return result;
};
const rejected = async (observation: string, command: Record<string, unknown>) => {
diff --git a/test/fixtures/ide-provider-extension.ts b/test/fixtures/ide-provider-extension.ts
index 94ba2b8..4ef57dd 100644
--- a/test/fixtures/ide-provider-extension.ts
+++ b/test/fixtures/ide-provider-extension.ts
@@ -81,6 +81,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
const journal = async () => (await readFile(log!, "utf8")).trim().split("\n").filter(Boolean).map(line => JSON.parse(line));
const inputs = async () => (await journal()).filter(row => ["type_text", "click", "press_key", "hotkey"].includes(row.name));
const captures = async () => (await journal()).filter(row => row.name === "get_window_state");
+ // Whether a call captured the window before its first native input. The action's successor captures it after the
+ // input (LPT-914, LPT-915); a capture before it would silently refresh the input's authority.
+ const capturedBeforeInput = async (from: number) => {
+ const rows = (await journal()).slice(from), input = rows.findIndex(row => ["type_text", "click", "press_key", "hotkey"].includes(row.name));
+ return rows.slice(0, input < 0 ? rows.length : input).some(row => row.name === "get_window_state");
+ };
const control = async (operation: string) => {
if (operation === "safe") mode = "safe";
else await driver!.call("fixture_control", { operation }, signal);
@@ -106,17 +112,16 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
assert((await inputs()).length === beforeDenied, "Consumed detail observation dispatched input");
view = await ok("read", { path: first }); observation = reference(view, "observation");
const hint = hintOf(view), literal = ' {"looks":"like JSON"}\n"quoted text"\n ';
- const nativeBeforeWrite = (await captures()).length;
+ const nativeBeforeWrite = (await journal()).length;
await ok("write", { path: `${observation}?hint=${encodeURIComponent(hint)}&delivery=foreground&delay_ms=0`, content: literal });
- // Only the successor's UI-only walks follow the input; no pixel capture refreshes input authority.
- assert((await captures()).slice(nativeBeforeWrite).every(row => row.arguments?.include_screenshot === false), "Write resolution refreshed native capture");
+ assert(!await capturedBeforeInput(nativeBeforeWrite), "Write resolution refreshed native capture");
const written = (await inputs()).at(-1);
assert(written?.name === "type_text" && written.arguments.text === literal && written.arguments.delay_ms === 0, "Literal Write normalized text, added a command or lost delay");
await denied("write", { path: observation, content: "replay" });
view = await ok("read", { path: first }); observation = reference(view, "observation");
- const nativeBeforeInsert = (await captures()).length;
+ const nativeBeforeInsert = (await journal()).length;
await ok("insert", { path: observation, text: JSON.stringify({ action: "click", hint: hintOf(view) }) });
- assert((await captures()).slice(nativeBeforeInsert).every(row => row.arguments?.include_screenshot === false), "Insert resolution refreshed native capture");
+ assert(!await capturedBeforeInput(nativeBeforeInsert), "Insert resolution refreshed native capture");
const uiOnly = await ok("read", { path: first, views: ["ui"] });
assert(!uiOnly.content.some(block => block.type === "image"), "UI text view included an image");
assert(textOf(uiOnly).includes("Fixture text field") && textOf(uiOnly).includes("Fixture button"), "UI text view lost observed structure");
@@ -158,12 +163,12 @@ export default async function fixture(pi: ExtensionAPI): Promise<void> {
assert((await inputs()).length === 0, "Invalid/Safe/display requests sent native input");
view = await ok("read", { path: first }); observation = reference(view, "observation");
await control("partial");
- const beforePartial = (await captures()).length;
+ const beforePartial = (await journal()).length;
const partial = await denied("insert", { path: observation, text: JSON.stringify({ action: "sequence", sequence: [
{ action: "type_text", x: 100, y: 80, text: "interrupted prefix" }, { action: "type_text", text: "must not run" },
] }) });
assert(/unknown|partial|uncertain|interrupted/i.test(textOf(partial)), "Interrupted result lost its uncertainty");
- assert((await captures()).length === beforePartial, "Interrupted action resolution refreshed native capture");
+ assert(!await capturedBeforeInput(beforePartial), "Interrupted action resolution refreshed native capture");
await denied("insert", { path: observation, text: JSON.stringify({ action: "click", x: 100, y: 80 }) });
assert((await inputs()).length === 1 && (await inputs())[0].name === "type_text" && (await inputs())[0].arguments.text === "interrupted prefix", "Interrupted prefix was replayed or later sequence input was sent");
view = await ok("read", { path: first }); observation = reference(view, "observation");
diff --git a/test/observation-granularity.integration.test.ts b/test/observation-granularity.integration.test.ts
index 357253f..d64b78a 100644
--- a/test/observation-granularity.integration.test.ts
+++ b/test/observation-granularity.integration.test.ts
@@ -17,7 +17,8 @@ test("real Pi returns only requested window parts and admits only their current
const code = `${SCRIPT_HELPERS}
const win = address(await windows(), "window");
const read = views => tools.computer_read({path:win,...(views?{views}:{})});
-const click = async (view,target) => (await insert(observation(view),{action:"click",...target})).outcome;
+// Only the reads are under test here: clicks skip the default successor, so every native capture is a read.
+const click = async (view,target) => (await insert(observation(view),{action:"click",...target,successor:false})).outcome;
const full=await read();
if(typeof full==="string"||!hints(full).length)throw Error("Full default changed");
const fullHint=hints(full)[0];
@@ -60,11 +61,8 @@ text("Three mode contracts passed; exactly two native clicks.");
}
for (const call of reads.filter(call => call.computer.granularity === "ui")) expect(call.computer.agent_image).toBeUndefined();
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
- // The click on a hint is followed by the successor's UI-only walks (LPT-914); the six reads come first.
- const all = calls.filter(c => c.name === "get_window_state"), lastClick = calls.findLastIndex(c => c.name === "click");
- const native = all.filter(c => calls.indexOf(c) < lastClick);
+ const native = calls.filter(c => c.name === "get_window_state");
expect(native).toHaveLength(6);
- expect(all.slice(6).every(c => c.arguments.include_screenshot === false)).toBe(true);
for (const index of [1, 2, 3]) expect(native[index].arguments.include_accessibility_tree).toBe(false);
for (const index of [4, 5]) expect(native[index].arguments.include_screenshot).toBe(false);
expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token ?? [c.arguments.x,c.arguments.y]))
diff --git a/test/ui-resources.integration.test.ts b/test/ui-resources.integration.test.ts
index 9921974..9605dd2 100644
--- a/test/ui-resources.integration.test.ts
+++ b/test/ui-resources.integration.test.ts
@@ -17,7 +17,8 @@ test("real Pi Codemode refines full resources locally and dispatches only a curr
const listed = await windows();
const [win, otherWin] = [...listed.matchAll(/computer:window\\/[\\w-]+/g)].map(match => match[0]);
const read = views => tools.computer_read({path:win,...(views?{views}:{})});
-const click = async (view,hint) => (await insert(observation(view),{action:"click",hint})).outcome;
+// Only local refinement is under test here: clicks skip the default successor, so every native capture is a read.
+const click = async (view,hint) => (await insert(observation(view),{action:"click",hint,successor:false})).outcome;
const first = await read(['capture:{"max_elements":200,"max_depth":15,"guides":{"grid":true,"action":true}}']);
if (!/UI: collected=200, complete=false/.test(body(first))) throw Error("Lost full resource");
const candidates = await tools.computer_search({path:observation(first),query:"Save",limit:2});
@@ -35,9 +36,9 @@ let latest = await read(['capture:{"max_elements":3,"hints":false}']);
if ((await click(latest, uiHint)) !== "refused") throw Error("Old alias rebound");
latest = await read(['capture:{"max_elements":3,"hints":false}']);
const target = observation(latest);
-const delivered = await insert(target,{action:"click",hint:hints(latest)[1]});
+const delivered = await insert(target,{action:"click",hint:hints(latest)[1],successor:false});
if (delivered.outcome !== "delivered") throw Error("Current hint not dispatched");
-if ((await insert(target,{action:"click",hint:hints(latest)[1]})).outcome !== "refused") throw Error("Consumed hint replayed");
+if ((await insert(target,{action:"click",hint:hints(latest)[1],successor:false})).outcome !== "refused") throw Error("Consumed hint replayed");
text("UI resource guards passed: 200 collected, 199 search matches, one native click");`;
try {
const result = await new PiIntegrationTest({
@@ -66,10 +67,7 @@ text("UI resource guards passed: 200 collected, 199 search matches, one native c
const delivered = nested.filter(call => call.name === "computer_insert" && call.computer.status === "delivered");
expect(delivered.map(call => call.computer.state.path)).toEqual(["ax"]);
const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
- // Five reads before the click, then the successor's UI-only walk after it (LPT-914).
- const walks = calls.filter(c => c.name === "get_window_state"), click = calls.findIndex(c => c.name === "click");
- expect(walks.filter(c => calls.indexOf(c) < click)).toHaveLength(5);
- expect(walks.filter(c => calls.indexOf(c) > click).every(c => c.arguments.include_screenshot === false)).toBe(true);
+ expect(calls.filter(c => c.name === "get_window_state")).toHaveLength(5);
expect(calls.filter(c => c.name === "click").map(c => c.arguments.element_token)).toEqual(["s5:1"]);
expect(calls.some(c => ["bring_to_front", "scroll", "zoom"].includes(c.name))).toBe(false);
} finally { await rm(cwd, { recursive: true, force: true }); }