Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Take Playwright CSS and element screenshots from a plain viewport shot so the window does not flash (LPT-968)
alexshpunt-clanker[bot]a2152dba2408
diff --git a/src/machine.ts b/src/machine.tsindex 88878af..c149f32 100644--- a/src/machine.ts+++ b/src/machine.ts@@ -24,6 +24,7 @@ export async function machineConfig(cwd: string, name: string): Promise<NonNullaconst WORKER_FILES: [string, string][] = [["src/web-worker.ts", fileURLToPath(new URL("./web-worker.ts", import.meta.url))],["src/aria-match.ts", fileURLToPath(new URL("./aria-match.ts", import.meta.url))],+ ["src/web-geometry.ts", fileURLToPath(new URL("./web-geometry.ts", import.meta.url))],["node_modules/playwright-core/package.json", createRequire(import.meta.url).resolve("playwright-core/package.json")],];diff --git a/src/session.ts b/src/session.tsindex e309b81..0136458 100644--- a/src/session.ts+++ b/src/session.ts@@ -28,6 +28,8 @@ import type { StateDiff } from "./successor.ts";import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";import { computerSource } from "./computer-address.ts";import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";+import { derivedPng } from "./web-capture.ts";+import type { Derive } from "./web-geometry.ts";/** Native Windows address. Models use session window references instead. */export interface WindowTarget { pid: number; window_id: number }@@ -1383,16 +1385,17 @@ export class ComputerSession {// Without an observation it shoots the live page: a locator, or a ref of the page's latest snapshot.if (id !== undefined && entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));try {- const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown }>("screenshot",+ const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown; derive?: Derive }>("screenshot",{ ...(target ? { target } : {}), highlight: kind === "highlight", ...(css ? { css } : {}) }, signal);- const bytes = Buffer.from(shot.png, "base64");+ const plain = Buffer.from(shot.png, "base64");+ const bytes = shot.derive ? await derivedPng(plain, shot.derive) : plain;if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw Error("Playwright did not return a PNG.");return this.result("observed", `Playwright ${kind === "page" ? "page viewport" : kind === "crop" ? "element" : "highlighted viewport"} screenshot. These are page pixels, not native window coordinates; they grant no input.`,{ url: shot.url, title: shot.title, ...(shot.box ? { box_css: shot.box } : {}) },{ ...this.addressed(entry), ...(id !== undefined ? { observation_id: id } : {}), agent_image: { route: "playwright_screenshot", kind, coordinate_space: "page_image_pixels", input_authority: false,output: { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20), bytes: bytes.length, mimeType: "image/png",hash: createHash("sha256").update(bytes).digest("hex") } } },- [{ type: "image", data: shot.png, mimeType: "image/png" }]);+ [{ type: "image", data: bytes.toString("base64"), mimeType: "image/png" }]);} catch (error) { return this.webFailure(entry, error); }});}diff --git a/src/web-capture.ts b/src/web-capture.tsnew file mode 100644index 0000000..2e6634f--- /dev/null+++ b/src/web-capture.ts@@ -0,0 +1,22 @@+import sharp from "sharp";+import { elementRect, type Derive, type PageGeometry, type PixelRect } from "./web-geometry.ts";++/** A plain viewport PNG cut to one rectangle: what an element screenshot shows. */+export function cropPng(png: Buffer, rect: PixelRect): Promise<Buffer> {+ return sharp(png).extract(rect).png().toBuffer();+}++/** A plain viewport PNG scaled to one image pixel per CSS pixel, within a pixel of the size Chrome's `scale: "css"` gives. */+export function cssPng(png: Buffer, geometry: PageGeometry): Promise<Buffer> {+ return sharp(png).resize(geometry.innerWidth, geometry.innerHeight, { fit: "fill", kernel: "lanczos3" }).png().toBuffer();+}++/** The image a worker's plain viewport screenshot stands for: the element's pixels, or the page in CSS pixels. */+export async function derivedPng(png: Buffer, derive: Derive): Promise<Buffer> {+ if (derive.box) {+ const rect = elementRect(derive.box, derive.geometry, { width: png.readUInt32BE(16), height: png.readUInt32BE(20) });+ if (!rect) throw new Error("The element is not inside the page screenshot.");+ return cropPng(png, rect);+ }+ return cssPng(png, derive.geometry);+}diff --git a/src/web-geometry.ts b/src/web-geometry.tsnew file mode 100644index 0000000..fc3a036--- /dev/null+++ b/src/web-geometry.ts@@ -0,0 +1,43 @@+// Pure page-geometry math shared by the Windows worker and the host. No imports: the worker has no sharp.++/**+ * Headful Chrome shrinks the visible window while it takes a screenshot whose clip or scale differs from the page's+ * own viewport, so the human sees a flash. A plain viewport screenshot does not. The worker takes that one and the+ * host derives the CSS-pixel screenshot and the element screenshot from it (see web-capture.ts).+ */++/** What the worker measured on the page at capture time. */+export interface PageGeometry {+ /** window.devicePixelRatio. */+ dpr: number;+ scrollX: number;+ scrollY: number;+ /** window.innerWidth and innerHeight: the CSS-pixel size of the viewport screenshot. */+ innerWidth: number;+ innerHeight: number;+}+/** An element's bounding box in CSS pixels of the viewport, as Playwright's boundingBox reports it. */+export interface CssBox { x: number; y: number; width: number; height: number }+/** A rectangle of the plain screenshot in its own (device) pixels. */+export interface PixelRect { left: number; top: number; width: number; height: number }++/** How a plain viewport screenshot becomes the wanted image on the host: scaled to CSS pixels (`css`), or cut to an element's `box`. */+export interface Derive { geometry: PageGeometry; box?: CssBox; css?: true }++/** Whether the plain screenshot's pixels are device pixels of the viewport; false when Chrome shot something else, such as a zoomed page. */+export function matchesViewport(image: { width: number; height: number }, geometry: PageGeometry): boolean {+ return Math.abs(image.width - geometry.innerWidth * geometry.dpr) <= 1 && Math.abs(image.height - geometry.innerHeight * geometry.dpr) <= 1;+}++/**+ * Where an element lies in the plain screenshot, or undefined when it is not wholly inside it. Chrome's own element+ * screenshot covers the integer CSS rectangle around the box in page coordinates, which is what this reproduces.+ */+export function elementRect(box: CssBox, geometry: PageGeometry, image: { width: number; height: number }): PixelRect | undefined {+ const { dpr, scrollX, scrollY } = geometry;+ const left = Math.floor(box.x + scrollX), top = Math.floor(box.y + scrollY);+ const right = Math.ceil(box.x + scrollX + box.width), bottom = Math.ceil(box.y + scrollY + box.height);+ const rect = { left: Math.round((left - scrollX) * dpr), top: Math.round((top - scrollY) * dpr), width: Math.round((right - left) * dpr), height: Math.round((bottom - top) * dpr) };+ if (rect.width < 1 || rect.height < 1 || rect.left < 0 || rect.top < 0 || rect.left + rect.width > image.width || rect.top + rect.height > image.height) return undefined;+ return rect;+}diff --git a/src/web-worker.ts b/src/web-worker.tsindex 20a0007..190854f 100644--- a/src/web-worker.ts+++ b/src/web-worker.ts@@ -12,6 +12,7 @@ import { ATTACH_APPROVAL_MS, parseBrowserCommandLine } from "./browser-command-limport type { WebAction, WebDownload, WebEvaluation, WebLaunchOptions, WebLogEntry, WebSnapshotOptions, WebTab, WebTarget, WebTimelineEvent } from "./web-browser.ts";import type { ConditionResult, UiCondition } from "./ui-conditions.ts";import { ariaTreeFromYaml, matchesAriaTemplate } from "./aria-match.ts";+import { elementRect, matchesViewport, type CssBox, type Derive, type PageGeometry } from "./web-geometry.ts";interface Tab { id: string; page: Page; title: string; load: WebTab["load"]; opener?: string; dialog?: Dialog; onDialog: Set<() => void> }interface DownloadRecord { info: WebDownload; download: Download; done: Promise<void> }@@ -589,20 +590,58 @@ async function check({ conditions }: { conditions: UiCondition[] }): Promise<Con}}));}+/** The page's viewport size in device pixels and the scroll, DPR and zoom behind it, measured in the main frame. */+async function measure(page: Page): Promise<PageGeometry & { zoom: number }> {+ return page.evaluate(() => ({ dpr: devicePixelRatio, scrollX, scrollY, innerWidth, innerHeight, zoom: visualViewport?.scale ?? 1 }));+}+/**+ * Headful Chrome shrinks the visible window while it takes a screenshot with a clip or a scale of its own (an element,+ * or `scale: "css"`), so the human sees a flash. A plain viewport screenshot does not; this takes one and describes+ * how the host derives the wanted image from it (see web-capture.ts). It returns undefined when that would not be+ * exact or the element cannot be resolved quickly (pinch-zoomed or scrolling page, element not wholly in view, no+ * such element): the caller then takes the ordinary screenshot, which also reports the real error.+ */+async function plainShot(page: Page, element?: Locator): Promise<{ png: Buffer; derive: Derive } | undefined> {+ try {+ let box: CssBox | undefined;+ if (element) {+ await element.scrollIntoViewIfNeeded({ timeout: 2_000 });+ box = await element.boundingBox({ timeout: 2_000 }) ?? undefined;+ if (!box) return undefined;+ }+ const before = await measure(page);+ const png = await page.screenshot({ timeout: 10_000 });+ const after = await measure(page);+ const geometry: PageGeometry = { dpr: before.dpr, scrollX: before.scrollX, scrollY: before.scrollY, innerWidth: before.innerWidth, innerHeight: before.innerHeight };+ const settled = before.zoom === 1 && JSON.stringify(before) === JSON.stringify(after);+ const image = { width: png.readUInt32BE(16), height: png.readUInt32BE(20) };+ if (!settled || !matchesViewport(image, geometry) || (box && !elementRect(box, geometry, image))) return undefined;+ return { png, derive: { geometry, ...(box ? { box } : {}) } };+ } catch { return undefined; }+}+async function screenshot({ target, highlight, css }: { target?: WebTarget; highlight?: boolean; css?: boolean }) {const tab = current(), page = idle(tab);let box;let png: Buffer;- if (target && !highlight) png = await locate(page, target).screenshot({ timeout: 10_000 });- else if (target) {+ let derive: Derive | undefined;+ if (target && !highlight) {+ const plain = await plainShot(page, locate(page, target));+ if (plain) ({ png, derive } = plain);+ else png = await locate(page, target).screenshot({ timeout: 10_000 });+ } else if (target) {// Playwright's mask overlays the element in the screenshot only; the page itself is not modified.const element = locate(page, target);box = await element.boundingBox({ timeout: 10_000 });if (!box) throw new Error("The element is not visible, so it cannot be highlighted.");png = await page.screenshot({ timeout: 10_000, mask: [element], maskColor: "rgba(255, 0, 96, 0.35)" });// In CSS pixels a page point of an action is a pixel of the screenshot, so a pixel change list can find it.- } else png = await page.screenshot({ timeout: 10_000, ...(css ? { scale: "css" as const } : {}) });- return { url: page.url(), title: tab.title, png: png.toString("base64"), ...(box ? { box } : {}) };+ } else if (css) {+ const plain = await plainShot(page);+ if (plain) { png = plain.png; derive = { ...plain.derive, css: true }; }+ else png = await page.screenshot({ timeout: 10_000, scale: "css" });+ } else png = await page.screenshot({ timeout: 10_000 });+ return { url: page.url(), title: tab.title, png: png.toString("base64"), ...(box ? { box } : {}), ...(derive ? { derive } : {}) };}async function close() {diff --git a/test/remote-machine.test.ts b/test/remote-machine.test.tsindex f59ff20..583808f 100644--- a/test/remote-machine.test.ts+++ b/test/remote-machine.test.ts@@ -201,14 +201,14 @@ test("an address no machine issued is refused before anything is sent", async ()});test("a remote browser refuses before launch when that machine's worker copy is stale or missing", async () => {- const local = execFileSync("sha256sum", ["src/web-worker.ts", "src/aria-match.ts", require.resolve("playwright-core/package.json")], { encoding: "utf8" });+ const local = execFileSync("sha256sum", ["src/web-worker.ts", "src/aria-match.ts", "src/web-geometry.ts", require.resolve("playwright-core/package.json")], { encoding: "utf8" });const runner = (output: string | Error): CommandRunner => ({ spawn: vi.fn(), run: vi.fn(async () => {if (output instanceof Error) throw output;return output;}) });const current = runner(local);await checkRemoteWorker(current, "/pkg", "setup part machine");- expect(current.run).toHaveBeenCalledWith("sha256sum", ["/pkg/src/web-worker.ts", "/pkg/src/aria-match.ts", "/pkg/node_modules/playwright-core/package.json"], expect.anything());+ expect(current.run).toHaveBeenCalledWith("sha256sum", ["/pkg/src/web-worker.ts", "/pkg/src/aria-match.ts", "/pkg/src/web-geometry.ts", "/pkg/node_modules/playwright-core/package.json"], expect.anything());const stale = local.replace(/^[0-9a-f]{64}/, "0".repeat(64));await expect(checkRemoteWorker(runner(stale), "/pkg", "setup part machine")).rejects.toThrow(/out of date \(src\/web-worker\.ts\); nothing was launched\. Refresh it from this package with setup part machine/);diff --git a/test/web-capture.test.ts b/test/web-capture.test.tsnew file mode 100644index 0000000..d0c8bc9--- /dev/null+++ b/test/web-capture.test.ts@@ -0,0 +1,60 @@+import sharp from "sharp";+import { describe, expect, test } from "vitest";+import { derivedPng } from "../src/web-capture.ts";+import { elementRect, matchesViewport, type PageGeometry } from "../src/web-geometry.ts";++const geometry = (over: Partial<PageGeometry> = {}): PageGeometry => ({ dpr: 1.5, scrollX: 0, scrollY: 0, innerWidth: 200, innerHeight: 100, ...over });+/** A device-pixel viewport screenshot, white with a black block at the given device rectangle. */+async function viewportPng(width: number, height: number, block?: { left: number; top: number; width: number; height: number }): Promise<Buffer> {+ const data = Buffer.alloc(width * height * 3, 255);+ if (block) for (let y = block.top; y < block.top + block.height; y++) for (let x = block.left; x < block.left + block.width; x++) data.fill(0, (y * width + x) * 3, (y * width + x) * 3 + 3);+ return sharp(data, { raw: { width, height, channels: 3 } }).png().toBuffer();+}+const size = async (png: Buffer) => { const { width, height } = await sharp(png).metadata(); return { width, height }; };++describe("matchesViewport", () => {+ test("accepts device pixels of the viewport within a pixel and refuses another scale", () => {+ expect(matchesViewport({ width: 300, height: 150 }, geometry())).toBe(true);+ expect(matchesViewport({ width: 299, height: 151 }, geometry({ innerWidth: 200, innerHeight: 100 }))).toBe(true);+ expect(matchesViewport({ width: 200, height: 100 }, geometry())).toBe(false);+ });+});++describe("elementRect", () => {+ test("covers the integer CSS rectangle around the box, scaled to device pixels", () => {+ expect(elementRect({ x: 10.4, y: 20.2, width: 30.3, height: 10.1 }, geometry(), { width: 300, height: 150 }))+ .toEqual({ left: 15, top: 30, width: 47, height: 17 });+ });++ test("rounds the integer rectangle from page coordinates, not from the scrolled viewport", () => {+ // Page y 50.6..60.7 → 50..61; with scroll 30.3 the viewport top is (50 - 30.3) * 1.5 = 29.55 → 30.+ expect(elementRect({ x: 0, y: 20.3, width: 10, height: 10.1 }, geometry({ scrollY: 30.3 }), { width: 300, height: 150 }))+ .toEqual({ left: 0, top: 30, width: 15, height: 17 });+ });++ test("refuses an element that is not wholly inside the screenshot", () => {+ const image = { width: 300, height: 150 };+ expect(elementRect({ x: -1, y: 5, width: 10, height: 10 }, geometry(), image)).toBeUndefined();+ expect(elementRect({ x: 5, y: 95, width: 10, height: 10 }, geometry(), image)).toBeUndefined();+ expect(elementRect({ x: 5, y: 5, width: 0, height: 10 }, geometry(), image)).toBeUndefined();+ });+});++describe("derivedPng", () => {+ test("cuts an element out of the viewport screenshot pixel for pixel", async () => {+ const png = await viewportPng(300, 150, { left: 15, top: 30, width: 45, height: 15 });+ const element = await derivedPng(png, { geometry: geometry(), box: { x: 10, y: 20, width: 30, height: 10 } });+ expect(await size(element)).toEqual({ width: 45, height: 15 });+ const { data } = await sharp(element).raw().toBuffer({ resolveWithObject: true });+ expect([...new Set(data)]).toEqual([0]);+ });++ test("scales the viewport screenshot to one pixel per CSS pixel", async () => {+ expect(await size(await derivedPng(await viewportPng(300, 150), { geometry: geometry(), css: true }))).toEqual({ width: 200, height: 100 });+ });++ test("fails for an element outside the screenshot instead of returning other pixels", async () => {+ await expect(derivedPng(await viewportPng(300, 150), { geometry: geometry(), box: { x: 190, y: 90, width: 30, height: 30 } }))+ .rejects.toThrow("The element is not inside the page screenshot.");+ });+});