Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Take Playwright CSS and element screenshots from a plain viewport shot so the window does not flash (LPT-968)

alexshpunt-clanker[bot]a2152dba2408
diff --git a/src/machine.ts b/src/machine.ts
index 88878af..c149f32 100644
--- a/src/machine.ts
+++ b/src/machine.ts
@@ -24,6 +24,7 @@ export async function machineConfig(cwd: string, name: string): Promise<NonNulla
const WORKER_FILES: [string, string][] = [
["src/web-worker.ts", fileURLToPath(new URL("./web-worker.ts", import.meta.url))],
["src/aria-match.ts", fileURLToPath(new URL("./aria-match.ts", import.meta.url))],
+ ["src/web-geometry.ts", fileURLToPath(new URL("./web-geometry.ts", import.meta.url))],
["node_modules/playwright-core/package.json", createRequire(import.meta.url).resolve("playwright-core/package.json")],
];
diff --git a/src/session.ts b/src/session.ts
index e309b81..0136458 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -28,6 +28,8 @@ import type { StateDiff } from "./successor.ts";
import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";
import { computerSource } from "./computer-address.ts";
import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
+import { derivedPng } from "./web-capture.ts";
+import type { Derive } from "./web-geometry.ts";
/** Native Windows address. Models use session window references instead. */
export interface WindowTarget { pid: number; window_id: number }
@@ -1383,16 +1385,17 @@ export class ComputerSession {
// Without an observation it shoots the live page: a locator, or a ref of the page's latest snapshot.
if (id !== undefined && entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));
try {
- const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown }>("screenshot",
+ const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown; derive?: Derive }>("screenshot",
{ ...(target ? { target } : {}), highlight: kind === "highlight", ...(css ? { css } : {}) }, signal);
- const bytes = Buffer.from(shot.png, "base64");
+ const plain = Buffer.from(shot.png, "base64");
+ const bytes = shot.derive ? await derivedPng(plain, shot.derive) : plain;
if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw Error("Playwright did not return a PNG.");
return this.result("observed", `Playwright ${kind === "page" ? "page viewport" : kind === "crop" ? "element" : "highlighted viewport"} screenshot. These are page pixels, not native window coordinates; they grant no input.`,
{ url: shot.url, title: shot.title, ...(shot.box ? { box_css: shot.box } : {}) },
{ ...this.addressed(entry), ...(id !== undefined ? { observation_id: id } : {}), agent_image: { route: "playwright_screenshot", kind, coordinate_space: "page_image_pixels", input_authority: false,
output: { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20), bytes: bytes.length, mimeType: "image/png",
hash: createHash("sha256").update(bytes).digest("hex") } } },
- [{ type: "image", data: shot.png, mimeType: "image/png" }]);
+ [{ type: "image", data: bytes.toString("base64"), mimeType: "image/png" }]);
} catch (error) { return this.webFailure(entry, error); }
});
}
diff --git a/src/web-capture.ts b/src/web-capture.ts
new file mode 100644
index 0000000..2e6634f
--- /dev/null
+++ b/src/web-capture.ts
@@ -0,0 +1,22 @@
+import sharp from "sharp";
+import { elementRect, type Derive, type PageGeometry, type PixelRect } from "./web-geometry.ts";
+
+/** A plain viewport PNG cut to one rectangle: what an element screenshot shows. */
+export function cropPng(png: Buffer, rect: PixelRect): Promise<Buffer> {
+ return sharp(png).extract(rect).png().toBuffer();
+}
+
+/** A plain viewport PNG scaled to one image pixel per CSS pixel, within a pixel of the size Chrome's `scale: "css"` gives. */
+export function cssPng(png: Buffer, geometry: PageGeometry): Promise<Buffer> {
+ return sharp(png).resize(geometry.innerWidth, geometry.innerHeight, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();
+}
+
+/** The image a worker's plain viewport screenshot stands for: the element's pixels, or the page in CSS pixels. */
+export async function derivedPng(png: Buffer, derive: Derive): Promise<Buffer> {
+ if (derive.box) {
+ const rect = elementRect(derive.box, derive.geometry, { width: png.readUInt32BE(16), height: png.readUInt32BE(20) });
+ if (!rect) throw new Error("The element is not inside the page screenshot.");
+ return cropPng(png, rect);
+ }
+ return cssPng(png, derive.geometry);
+}
diff --git a/src/web-geometry.ts b/src/web-geometry.ts
new file mode 100644
index 0000000..fc3a036
--- /dev/null
+++ b/src/web-geometry.ts
@@ -0,0 +1,43 @@
+// Pure page-geometry math shared by the Windows worker and the host. No imports: the worker has no sharp.
+
+/**
+ * Headful Chrome shrinks the visible window while it takes a screenshot whose clip or scale differs from the page's
+ * own viewport, so the human sees a flash. A plain viewport screenshot does not. The worker takes that one and the
+ * host derives the CSS-pixel screenshot and the element screenshot from it (see web-capture.ts).
+ */
+
+/** What the worker measured on the page at capture time. */
+export interface PageGeometry {
+ /** window.devicePixelRatio. */
+ dpr: number;
+ scrollX: number;
+ scrollY: number;
+ /** window.innerWidth and innerHeight: the CSS-pixel size of the viewport screenshot. */
+ innerWidth: number;
+ innerHeight: number;
+}
+/** An element's bounding box in CSS pixels of the viewport, as Playwright's boundingBox reports it. */
+export interface CssBox { x: number; y: number; width: number; height: number }
+/** A rectangle of the plain screenshot in its own (device) pixels. */
+export interface PixelRect { left: number; top: number; width: number; height: number }
+
+/** How a plain viewport screenshot becomes the wanted image on the host: scaled to CSS pixels (`css`), or cut to an element's `box`. */
+export interface Derive { geometry: PageGeometry; box?: CssBox; css?: true }
+
+/** Whether the plain screenshot's pixels are device pixels of the viewport; false when Chrome shot something else, such as a zoomed page. */
+export function matchesViewport(image: { width: number; height: number }, geometry: PageGeometry): boolean {
+ return Math.abs(image.width - geometry.innerWidth * geometry.dpr) <= 1 && Math.abs(image.height - geometry.innerHeight * geometry.dpr) <= 1;
+}
+
+/**
+ * Where an element lies in the plain screenshot, or undefined when it is not wholly inside it. Chrome's own element
+ * screenshot covers the integer CSS rectangle around the box in page coordinates, which is what this reproduces.
+ */
+export function elementRect(box: CssBox, geometry: PageGeometry, image: { width: number; height: number }): PixelRect | undefined {
+ const { dpr, scrollX, scrollY } = geometry;
+ const left = Math.floor(box.x + scrollX), top = Math.floor(box.y + scrollY);
+ const right = Math.ceil(box.x + scrollX + box.width), bottom = Math.ceil(box.y + scrollY + box.height);
+ const rect = { left: Math.round((left - scrollX) * dpr), top: Math.round((top - scrollY) * dpr), width: Math.round((right - left) * dpr), height: Math.round((bottom - top) * dpr) };
+ if (rect.width < 1 || rect.height < 1 || rect.left < 0 || rect.top < 0 || rect.left + rect.width > image.width || rect.top + rect.height > image.height) return undefined;
+ return rect;
+}
diff --git a/src/web-worker.ts b/src/web-worker.ts
index 20a0007..190854f 100644
--- a/src/web-worker.ts
+++ b/src/web-worker.ts
@@ -12,6 +12,7 @@ import { ATTACH_APPROVAL_MS, parseBrowserCommandLine } from "./browser-command-l
import type { WebAction, WebDownload, WebEvaluation, WebLaunchOptions, WebLogEntry, WebSnapshotOptions, WebTab, WebTarget, WebTimelineEvent } from "./web-browser.ts";
import type { ConditionResult, UiCondition } from "./ui-conditions.ts";
import { ariaTreeFromYaml, matchesAriaTemplate } from "./aria-match.ts";
+import { elementRect, matchesViewport, type CssBox, type Derive, type PageGeometry } from "./web-geometry.ts";
interface Tab { id: string; page: Page; title: string; load: WebTab["load"]; opener?: string; dialog?: Dialog; onDialog: Set<() => void> }
interface DownloadRecord { info: WebDownload; download: Download; done: Promise<void> }
@@ -589,20 +590,58 @@ async function check({ conditions }: { conditions: UiCondition[] }): Promise<Con
}
}));
}
+/** The page's viewport size in device pixels and the scroll, DPR and zoom behind it, measured in the main frame. */
+async function measure(page: Page): Promise<PageGeometry & { zoom: number }> {
+ return page.evaluate(() => ({ dpr: devicePixelRatio, scrollX, scrollY, innerWidth, innerHeight, zoom: visualViewport?.scale ?? 1 }));
+}
+/**
+ * Headful Chrome shrinks the visible window while it takes a screenshot with a clip or a scale of its own (an element,
+ * or `scale: "css"`), so the human sees a flash. A plain viewport screenshot does not; this takes one and describes
+ * how the host derives the wanted image from it (see web-capture.ts). It returns undefined when that would not be
+ * exact or the element cannot be resolved quickly (pinch-zoomed or scrolling page, element not wholly in view, no
+ * such element): the caller then takes the ordinary screenshot, which also reports the real error.
+ */
+async function plainShot(page: Page, element?: Locator): Promise<{ png: Buffer; derive: Derive } | undefined> {
+ try {
+ let box: CssBox | undefined;
+ if (element) {
+ await element.scrollIntoViewIfNeeded({ timeout: 2_000 });
+ box = await element.boundingBox({ timeout: 2_000 }) ?? undefined;
+ if (!box) return undefined;
+ }
+ const before = await measure(page);
+ const png = await page.screenshot({ timeout: 10_000 });
+ const after = await measure(page);
+ const geometry: PageGeometry = { dpr: before.dpr, scrollX: before.scrollX, scrollY: before.scrollY, innerWidth: before.innerWidth, innerHeight: before.innerHeight };
+ const settled = before.zoom === 1 && JSON.stringify(before) === JSON.stringify(after);
+ const image = { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };
+ if (!settled || !matchesViewport(image, geometry) || (box && !elementRect(box, geometry, image))) return undefined;
+ return { png, derive: { geometry, ...(box ? { box } : {}) } };
+ } catch { return undefined; }
+}
+
async function screenshot({ target, highlight, css }: { target?: WebTarget; highlight?: boolean; css?: boolean }) {
const tab = current(), page = idle(tab);
let box;
let png: Buffer;
- if (target && !highlight) png = await locate(page, target).screenshot({ timeout: 10_000 });
- else if (target) {
+ let derive: Derive | undefined;
+ if (target && !highlight) {
+ const plain = await plainShot(page, locate(page, target));
+ if (plain) ({ png, derive } = plain);
+ else png = await locate(page, target).screenshot({ timeout: 10_000 });
+ } else if (target) {
// Playwright's mask overlays the element in the screenshot only; the page itself is not modified.
const element = locate(page, target);
box = await element.boundingBox({ timeout: 10_000 });
if (!box) throw new Error("The element is not visible, so it cannot be highlighted.");
png = await page.screenshot({ timeout: 10_000, mask: [element], maskColor: "rgba(255, 0, 96, 0.35)" });
// In CSS pixels a page point of an action is a pixel of the screenshot, so a pixel change list can find it.
- } else png = await page.screenshot({ timeout: 10_000, ...(css ? { scale: "css" as const } : {}) });
- return { url: page.url(), title: tab.title, png: png.toString("base64"), ...(box ? { box } : {}) };
+ } else if (css) {
+ const plain = await plainShot(page);
+ if (plain) { png = plain.png; derive = { ...plain.derive, css: true }; }
+ else png = await page.screenshot({ timeout: 10_000, scale: "css" });
+ } else png = await page.screenshot({ timeout: 10_000 });
+ return { url: page.url(), title: tab.title, png: png.toString("base64"), ...(box ? { box } : {}), ...(derive ? { derive } : {}) };
}
async function close() {
diff --git a/test/remote-machine.test.ts b/test/remote-machine.test.ts
index f59ff20..583808f 100644
--- a/test/remote-machine.test.ts
+++ b/test/remote-machine.test.ts
@@ -201,14 +201,14 @@ test("an address no machine issued is refused before anything is sent", async ()
});
test("a remote browser refuses before launch when that machine's worker copy is stale or missing", async () => {
- const local = execFileSync("sha256sum", ["src/web-worker.ts", "src/aria-match.ts", require.resolve("playwright-core/package.json")], { encoding: "utf8" });
+ const local = execFileSync("sha256sum", ["src/web-worker.ts", "src/aria-match.ts", "src/web-geometry.ts", require.resolve("playwright-core/package.json")], { encoding: "utf8" });
const runner = (output: string | Error): CommandRunner => ({ spawn: vi.fn(), run: vi.fn(async () => {
if (output instanceof Error) throw output;
return output;
}) });
const current = runner(local);
await checkRemoteWorker(current, "/pkg", "setup part machine");
- expect(current.run).toHaveBeenCalledWith("sha256sum", ["/pkg/src/web-worker.ts", "/pkg/src/aria-match.ts", "/pkg/node_modules/playwright-core/package.json"], expect.anything());
+ expect(current.run).toHaveBeenCalledWith("sha256sum", ["/pkg/src/web-worker.ts", "/pkg/src/aria-match.ts", "/pkg/src/web-geometry.ts", "/pkg/node_modules/playwright-core/package.json"], expect.anything());
const stale = local.replace(/^[0-9a-f]{64}/, "0".repeat(64));
await expect(checkRemoteWorker(runner(stale), "/pkg", "setup part machine"))
.rejects.toThrow(/out of date \(src\/web-worker\.ts\); nothing was launched\. Refresh it from this package with setup part machine/);
diff --git a/test/web-capture.test.ts b/test/web-capture.test.ts
new file mode 100644
index 0000000..d0c8bc9
--- /dev/null
+++ b/test/web-capture.test.ts
@@ -0,0 +1,60 @@
+import sharp from "sharp";
+import { describe, expect, test } from "vitest";
+import { derivedPng } from "../src/web-capture.ts";
+import { elementRect, matchesViewport, type PageGeometry } from "../src/web-geometry.ts";
+
+const geometry = (over: Partial<PageGeometry> = {}): PageGeometry => ({ dpr: 1.5, scrollX: 0, scrollY: 0, innerWidth: 200, innerHeight: 100, ...over });
+/** A device-pixel viewport screenshot, white with a black block at the given device rectangle. */
+async function viewportPng(width: number, height: number, block?: { left: number; top: number; width: number; height: number }): Promise<Buffer> {
+ const data = Buffer.alloc(width * height * 3, 255);
+ if (block) for (let y = block.top; y < block.top + block.height; y++) for (let x = block.left; x < block.left + block.width; x++) data.fill(0, (y * width + x) * 3, (y * width + x) * 3 + 3);
+ return sharp(data, { raw: { width, height, channels: 3 } }).png().toBuffer();
+}
+const size = async (png: Buffer) => { const { width, height } = await sharp(png).metadata(); return { width, height }; };
+
+describe("matchesViewport", () => {
+ test("accepts device pixels of the viewport within a pixel and refuses another scale", () => {
+ expect(matchesViewport({ width: 300, height: 150 }, geometry())).toBe(true);
+ expect(matchesViewport({ width: 299, height: 151 }, geometry({ innerWidth: 200, innerHeight: 100 }))).toBe(true);
+ expect(matchesViewport({ width: 200, height: 100 }, geometry())).toBe(false);
+ });
+});
+
+describe("elementRect", () => {
+ test("covers the integer CSS rectangle around the box, scaled to device pixels", () => {
+ expect(elementRect({ x: 10.4, y: 20.2, width: 30.3, height: 10.1 }, geometry(), { width: 300, height: 150 }))
+ .toEqual({ left: 15, top: 30, width: 47, height: 17 });
+ });
+
+ test("rounds the integer rectangle from page coordinates, not from the scrolled viewport", () => {
+ // Page y 50.6..60.7 → 50..61; with scroll 30.3 the viewport top is (50 - 30.3) * 1.5 = 29.55 → 30.
+ expect(elementRect({ x: 0, y: 20.3, width: 10, height: 10.1 }, geometry({ scrollY: 30.3 }), { width: 300, height: 150 }))
+ .toEqual({ left: 0, top: 30, width: 15, height: 17 });
+ });
+
+ test("refuses an element that is not wholly inside the screenshot", () => {
+ const image = { width: 300, height: 150 };
+ expect(elementRect({ x: -1, y: 5, width: 10, height: 10 }, geometry(), image)).toBeUndefined();
+ expect(elementRect({ x: 5, y: 95, width: 10, height: 10 }, geometry(), image)).toBeUndefined();
+ expect(elementRect({ x: 5, y: 5, width: 0, height: 10 }, geometry(), image)).toBeUndefined();
+ });
+});
+
+describe("derivedPng", () => {
+ test("cuts an element out of the viewport screenshot pixel for pixel", async () => {
+ const png = await viewportPng(300, 150, { left: 15, top: 30, width: 45, height: 15 });
+ const element = await derivedPng(png, { geometry: geometry(), box: { x: 10, y: 20, width: 30, height: 10 } });
+ expect(await size(element)).toEqual({ width: 45, height: 15 });
+ const { data } = await sharp(element).raw().toBuffer({ resolveWithObject: true });
+ expect([...new Set(data)]).toEqual([0]);
+ });
+
+ test("scales the viewport screenshot to one pixel per CSS pixel", async () => {
+ expect(await size(await derivedPng(await viewportPng(300, 150), { geometry: geometry(), css: true }))).toEqual({ width: 200, height: 100 });
+ });
+
+ test("fails for an element outside the screenshot instead of returning other pixels", async () => {
+ await expect(derivedPng(await viewportPng(300, 150), { geometry: geometry(), box: { x: 190, y: 90, width: 30, height: 30 } }))
+ .rejects.toThrow("The element is not inside the page screenshot.");
+ });
+});