Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: a navigation shows the whole page; the page start comes from the topmost web element; live checks of the change list (LPT-928)

alexshpunt-clanker[bot]35dabd3644a2
diff --git a/docs/computer-resources.md b/docs/computer-resources.md
index 230536d..0979338 100644
--- a/docs/computer-resources.md
+++ b/docs/computer-resources.md
@@ -124,7 +124,7 @@ Pixels changed in 2 areas (overview pixels; 0.07% of the window); image 1 outlin
Browser chrome outside the page also changed (x=0 y=0 w=1280 h=80, 2 areas): tab strip, address bar or loading state; not listed.
```
-The area at or nearest to the input point comes first, otherwise the largest. In a window whose web content fills most of it, changes outside the page, such as a loading tab or the address bar, are summarised in one line. When most of the window or page changed, as after a navigation, image 1 is the whole window or page after the action instead of an overview and crops.
+The area at or nearest to the input point comes first, otherwise the largest. In a browser window whose page elements reach UI Automation (Edge does; Chrome often does not), changes above the page, such as a loading tab or the address bar, are summarised in one line. When most of the window or page changed, or a page action navigated to another URL, image 1 is the whole window or page after the action instead of an overview and crops.
This shows a stroke on a canvas or a moved sprite when the UI tree says nothing. Changes under 12 pixels and caret-shaped spots do not count. When the first capture shows no change, it looks again for up to 300 ms, then says `Nothing changed in the UI or the window's pixels within N ms`; check the target before repeating the action. When something changed, it captures until the window settles, for up to 1 second.
diff --git a/scripts/web-fixture.mjs b/scripts/web-fixture.mjs
index 42c8529..3612962 100644
--- a/scripts/web-fixture.mjs
+++ b/scripts/web-fixture.mjs
@@ -22,6 +22,12 @@ const pages = {
<p role="status" id="out">idle</p>
<iframe title="Inner frame" src="/frame" width="300" height="90"></iframe>
<script>function out(text) { document.querySelector('#out').textContent = text; }</script>`),
+ // Change list witness: a grid of tiles that tick when clicked, a large banner elsewhere and a link to another page.
+ "/checks": page("Checks", `<h1>Checks</h1>
+<div style="display:grid;grid-template-columns:repeat(4,60px);gap:8px;margin:20px">${Array.from({ length: 8 }, (_, i) => `<label style="display:block;width:60px;height:60px;background:#dde"><input type="checkbox" aria-label="Tile ${i + 1}" style="width:20px;height:20px;margin:20px"></label>`).join("")}</div>
+<p id="banner" style="width:600px;height:120px;background:#fff">quiet</p>
+<a href="/">Leave</a>
+<script>document.addEventListener('change', () => { const b = document.querySelector('#banner'); b.style.background = '#fc6'; b.textContent = 'ticked'; });</script>`),
// Native input witness: every key, button, wheel, pen and pointer-lock movement is journaled with its details.
"/input": page("Input log", `<h1>Input log</h1>
<div id="pad" style="position:fixed;inset:80px 0 0 0;background:#eef;touch-action:none"></div>
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 951d274..9d728dc 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -14,7 +14,7 @@ import { validateWebAction, validateWebLaunch, type WebAction } from "./web-brow
import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";
import { changeCrops, changeOverview, decodeFrame, diffFrames, MOST_CHANGED, type DiffContext } from "./pixel-diff.ts";
import type { WindowFrame } from "./session.ts";
-import { webContentBox, type UiCaptureOptions, type UiElement, type UiResource } from "./ui-resources.ts";
+import { webContentTop, type UiCaptureOptions, type UiElement, type UiResource } from "./ui-resources.ts";
import { displayMode } from "./display-mode.ts";
import { isSemanticAction } from "./semantic-actions.ts";
import { accessibleName, ariaRole, checkedState, isVisible, nativeAriaSnapshot, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";
@@ -37,8 +37,10 @@ async function cropImages(before: Buffer, after: Buffer, diff: StateDiff): Promi
const pixels = diff.pixels;
if (!pixels?.box || pixels.size_changed) return [];
const image = (png: Buffer) => ({ type: "image" as const, data: png.toString("base64"), mimeType: "image/png" });
- if (pixels.share >= MOST_CHANGED) {
+ // A navigation replaces the page even when the old and new pages look alike, so it gets the whole page too.
+ if (pixels.share >= MOST_CHANGED || diff.page?.url) {
pixels.full = true;
+ if (diff.page?.url) pixels.navigated = true;
return [image(after)];
}
const [overview, cropped] = await Promise.all([changeOverview(after, pixels).catch(() => undefined), changeCrops(before, after, pixels).catch(() => [])]);
@@ -46,15 +48,18 @@ async function cropImages(before: Buffer, after: Buffer, diff: StateDiff): Promi
pixels.crops = cropped.map(item => ({ ...item.region, scale: item.scale }));
return [...(overview ? [image(overview)] : []), ...cropped.flatMap(item => [image(item.before), image(item.after)])];
}
+/** Web content up to this far above its topmost element still counts as page, for the page's own margin. */
+const PAGE_MARGIN = 12;
/**
* What a pixel comparison of a window knows about the action: its input point in the input observation's pixels,
- * and in a browser window the web page, so changes outside it count as browser chrome. A window whose web content
- * covers most of it is treated as a browser; an app with a small embedded web view is not.
+ * and in a browser window where its web page starts, so changes wholly above it count as browser chrome. A window
+ * whose web content starts in its upper half is treated as a browser; an app with a web view lower down is not.
*/
function diffContext(ui: UiResource | undefined, image: { width: number; height: number } | undefined, focus: InputPoint | undefined): DiffContext {
- const page = webContentBox(ui);
- const browser = page && image && page.width * page.height >= 0.6 * image.width * image.height;
- return { ...(focus ? { focus } : {}), ...(browser ? { page } : {}) };
+ const top = webContentTop(ui);
+ const start = top === undefined ? undefined : Math.max(0, top - PAGE_MARGIN);
+ const page = image && start !== undefined && start > 0 && start < image.height / 2 ? { x: 0, y: start, width: image.width, height: image.height - start } : undefined;
+ return { ...(focus ? { focus } : {}), ...(page ? { page } : {}) };
}
/**
* Where an input command lands in its observation's pixels: the end of a drag, the last pointed event of a timeline,
diff --git a/src/provider.ts b/src/provider.ts
index fb65cdc..c7a970f 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -226,7 +226,8 @@ function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {
const rect = (b: PixelBox) => `x=${b.x} y=${b.y} w=${b.width} h=${b.height}`;
const chrome = pixels.chrome ? [`Browser chrome outside the page also changed (${rect(pixels.chrome.box)}, ${pixels.chrome.regions} ${pixels.chrome.regions === 1 ? "area" : "areas"}): tab strip, address bar or loading state; not listed.`] : [];
const page = pixels.space === "page", whole = page ? "page" : "window", unit = page ? "CSS pixels of the page" : "overview pixels";
- if (pixels.full) return [`Most of the ${whole} changed${since} (${percent}% of its pixels): image 1 shows the whole ${whole} after the action.`, ...chrome];
+ if (pixels.full) return [pixels.navigated ? `The page navigated (${percent}% of its pixels changed): image 1 shows the whole page after the action.`
+ : `Most of the ${whole} changed${since} (${percent}% of its pixels): image 1 shows the whole ${whole} after the action.`, ...chrome];
if (!pixels.areas.length) return chrome;
// The overview, when attached, is image 1 and numbers the areas as listed; each crop pair follows it.
const first = pixels.overview ? 2 : 1;
diff --git a/src/successor.ts b/src/successor.ts
index 08ea790..2b5266c 100644
--- a/src/successor.ts
+++ b/src/successor.ts
@@ -7,7 +7,7 @@ import { diffFrames, pixelsChanged, type DiffContext, type PixelBox, type PixelD
* and numbers the listed areas on the window, `crops` the regions of the before/after pairs after it with their
* scale, and `full` when most of the window changed and the only image is the whole window after the action.
*/
-export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; since?: string;
+export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; navigated?: true; since?: string;
/** `page` when the pixels are Playwright screenshots of a browser page in CSS pixels, not window overviews. */
space?: "page" };
diff --git a/src/ui-resources.ts b/src/ui-resources.ts
index 84f1001..1353d36 100644
--- a/src/ui-resources.ts
+++ b/src/ui-resources.ts
@@ -208,14 +208,14 @@ export async function annotateUi(image: Content, resource: UiResource, width: nu
}
/**
- * The part of a window image that shows web page content: the bounds of its largest web content element, such as a
- * browser's document. Undefined when the window shows no web content.
+ * Where web page content starts in a window image: the top of its topmost web content element. Browsers report the
+ * page's elements, not always a document with the viewport's bounds. Undefined when the window shows no web content.
*/
-export function webContentBox(ui: UiResource | undefined): ScreenRect | undefined {
- let best: ScreenRect | undefined;
+export function webContentTop(ui: UiResource | undefined): number | undefined {
+ let top: number | undefined;
for (const element of ui?.elements ?? []) {
const box = element.bounds.window_image;
- if (element.in_web_content && box && (!best || box.width * box.height > best.width * best.height)) best = box;
+ if (element.in_web_content && box && box.width > 0 && box.height > 0 && (top === undefined || box.y < top)) top = box.y;
}
- return best;
+ return top;
}
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 136021e..9365f38 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -201,6 +201,21 @@ test("a page action compares page screenshots and puts the change at its point f
expect(clicked.content.filter(block => block.type === "image")).toHaveLength(3);
});
+test("a page action that navigates shows the whole page after it, even when the pages look alike", async () => {
+ const f = browserFixture();
+ const white = await sharp({ create: { width: 400, height: 300, channels: 3, background: "white" } }).png().toBuffer();
+ const lined = await sharp(white).composite([{ input: { create: { width: 200, height: 10, channels: 3, background: "black" } }, left: 10, top: 10 }]).png().toBuffer();
+ const shots = [white, lined];
+ (f.browser as unknown as { request: unknown }).request = vi.fn(async () => ({ url: "https://example.com/next", title: "Next", png: (shots.shift() ?? lined).toString("base64") }));
+ const window = await f.window();
+ await f.resources.input(window, cwd, clickPlan({ window: "x", target: "e1" }));
+ f.page.mockImplementation(async () => ({ url: "https://example.com/other", title: "Other", snapshot: "- heading \"Other\" [ref=e1]", tabs: [], events: [] }));
+ const left = await f.resources.input(window, cwd, clickPlan({ window: "x", target: "e1" }));
+ expect(left.details.successor?.diff?.pixels).toMatchObject({ full: true, navigated: true });
+ expect(presentComputerResult(left)).toMatch(/The page navigated \([\d.]+% of its pixels changed\): image 1 shows the whole page after the action\./);
+ expect(left.content.filter(block => block.type === "image")).toHaveLength(1);
+});
+
test("typed input on a browser window goes to Playwright with page refs and locators", async () => {
const f = browserFixture();
const window = await f.window();
diff --git a/test/live/typed-input.live.ts b/test/live/typed-input.live.ts
index 7c37220..86e1092 100644
--- a/test/live/typed-input.live.ts
+++ b/test/live/typed-input.live.ts
@@ -142,6 +142,49 @@ test("points of a browser page take clicks, wheel, drags, held keys and a pressu
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
+// LPT-928: the change at the input comes first, with an outlined overview, and a navigation shows the whole page.
+test("a ticked tile is the first change of a page click and of a native click, and a navigation returns the whole page", async () => {
+ const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/checks` });
+ expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
+ const window = `computer:window/${opened.details.window_ref}`;
+ try {
+ const measured = await live.resources.browserAction(requireComputerAddress(window), live.cwd,
+ { action: "evaluate", expression: "JSON.stringify(document.querySelectorAll('input')[5].getBoundingClientRect())" });
+ const box = JSON.parse(JSON.parse(/Evaluation result: (.*)$/m.exec(presentComputerResult(measured))![1])) as { x: number; y: number; width: number; height: number };
+ const page = await act(window, clickPlan({ window, x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) }));
+ const pixels = page.result.details.successor?.diff?.pixels;
+ console.log(page.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
+ expect(pixels, page.body).toMatchObject({ space: "page", overview: true });
+ // The banner changed too and is larger; the ticked tile comes first because the click landed on it.
+ expect(pixels!.areas[0]).toMatchObject({ at_input: true });
+ expect(pixels!.areas[0].width).toBeLessThan(80);
+ expect(pixels!.regions).toBeGreaterThan(1);
+ expect(page.result.content.filter(block => block.type === "image").length).toBeGreaterThanOrEqual(3);
+ // A native click on a tile by ref of a window Read compares the window's pixels. Edge reports page elements to
+ // UI Automation (this Chrome does not), so its refs reach the tiles and its page start is known.
+ const edge = await live.insert(live.machine, { action: "open_browser", browser: "msedge", url: `${fixture.base}/checks` });
+ const edgeWindow = `computer:window/${edge.details.window_ref}`;
+ try {
+ // Edge builds the page's accessibility tree a moment after the page loads.
+ let full = await live.read(edgeWindow, ["full"]);
+ for (let i = 0; i < 20 && !full.details.ui?.elements.some(element => element.in_web_content); i++) {
+ await new Promise(resolve => setTimeout(resolve, 500));
+ full = await live.read(edgeWindow, ["full"]);
+ }
+ const tile = full.details.ui?.elements.find(element => element.role === "CheckBox" && element.name === "Tile 7")?.hint;
+ expect(tile, JSON.stringify(full.details.ui?.elements.filter(element => element.in_web_content).map(element => [element.role, element.name]).slice(0, 20))).toBeDefined();
+ const native = await act(edgeWindow, clickPlan({ window: edgeWindow, target: tile, space: "window" }));
+ console.log(native.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
+ expect(native.result.details.successor?.diff?.pixels?.areas[0], native.body).toMatchObject({ at_input: true });
+ } finally { await deleteComputer(live.resources, edgeWindow, { path: edgeWindow }, live.cwd); }
+ await live.read(window, ["dom"]);
+ const left = await act(window, clickPlan({ window, target: "getByRole('link', { name: 'Leave' })" }));
+ console.log(left.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
+ expect(left.result.details.successor?.diff?.pixels, left.body).toMatchObject({ full: true });
+ expect(left.result.content.filter(block => block.type === "image")).toHaveLength(1);
+ } finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
+});
+
test("the same vocabulary drives a browser page through Playwright by locator and page ref", async () => {
const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/wait` });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
diff --git a/test/ui-resources.test.ts b/test/ui-resources.test.ts
index 0cddc92..b64fdd7 100644
--- a/test/ui-resources.test.ts
+++ b/test/ui-resources.test.ts
@@ -1,6 +1,7 @@
import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { ComputerSession, type Driver, type Content } from "../src/session.ts";
+import { createUiResource, webContentTop } from "../src/ui-resources.ts";
async function fixture() {
let snapshot = 0;
@@ -266,3 +267,13 @@ test("native limits are explicit, while disabled annotation returns exact native
expect(f.driver.call).toHaveBeenLastCalledWith("get_window_state", expect.objectContaining({ max_elements: 2, max_depth: 4 }), undefined);
expect(view.content.filter(b => b.type === "image")).toContainEqual({ type: "image", mimeType: "image/png", data: f.raw.toString("base64") });
});
+
+// LPT-928: where a browser window's page starts, from the elements the browser marks as web content.
+test("web content starts at the top of its topmost web element; native rows do not count", () => {
+ const row = (index: number, y: number, web: boolean) => ({ element_index: index, element_token: `s1:${index}`, role: web ? "CheckBox" : "Button",
+ label: `Row ${index}`, screenshot_frame: { x: 10, y, w: 20, h: 20 }, ...(web ? { in_web_content: true } : {}) });
+ const resource = createUiResource({ elements: [row(0, 40, false), row(1, 178, true), row(2, 100, true)] }, "w1", "o1", 1);
+ expect(resource.elements.map(element => element.in_web_content)).toEqual([undefined, true, true]);
+ expect(webContentTop(resource)).toBe(100);
+ expect(webContentTop(createUiResource({ elements: [row(0, 40, false)] }, "w1", "o2", 2))).toBeUndefined();
+});
\ No newline at end of file