Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: page actions show their pixel change from Playwright screenshots in CSS pixels (LPT-928)
alexshpunt-clanker[bot]a7e220064897
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex e0c43a3..951d274 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -12,7 +12,7 @@ import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./seimport { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";-import { changeCrops, changeOverview, decodeFrame, MOST_CHANGED, type DiffContext } from "./pixel-diff.ts";+import { changeCrops, changeOverview, decodeFrame, diffFrames, MOST_CHANGED, type DiffContext } from "./pixel-diff.ts";import type { WindowFrame } from "./session.ts";import { webContentBox, type UiCaptureOptions, type UiElement, type UiResource } from "./ui-resources.ts";import { displayMode } from "./display-mode.ts";@@ -33,15 +33,15 @@ export class ComputerRequestError extends Error {}* after the action; otherwise an overview of the window with the listed areas outlined and numbered, then a before and* an after crop of each of the first areas.*/-async function cropImages(before: WindowFrame, after: WindowFrame, diff: StateDiff): Promise<ComputerResult["content"]> {+async function cropImages(before: Buffer, after: Buffer, diff: StateDiff): Promise<ComputerResult["content"]> {const pixels = diff.pixels;if (!pixels?.box || pixels.size_changed) return [];const image = (png: Buffer) => ({ type: "image" as const, data: png.toString("base64"), mimeType: "image/png" });if (pixels.share >= MOST_CHANGED) {pixels.full = true;- return [image(after.png)];+ return [image(after)];}- const [overview, cropped] = await Promise.all([changeOverview(after.png, pixels).catch(() => undefined), changeCrops(before.png, after.png, pixels).catch(() => [])]);+ const [overview, cropped] = await Promise.all([changeOverview(after, pixels).catch(() => undefined), changeCrops(before, after, pixels).catch(() => [])]);if (overview) pixels.overview = true;pixels.crops = cropped.map(item => ({ ...item.region, scale: item.scale }));return [...(overview ? [image(overview)] : []), ...cropped.flatMap(item => [image(item.before), image(item.after)])];@@ -114,6 +114,8 @@ const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "dointerface FollowUp { successor: boolean; explicit: boolean; frame?: WindowFrame;/** Where the input landed, in the input observation's pixels; the pixel change list puts areas near it first. */focus?: InputPoint;+ /** A Playwright screenshot of the page before a page action, in CSS pixels, for its pixel change list. */+ page?: Buffer;/** The window's session, resolved before the action: a failed action can make the session forget the window. */session?: ComputerSession }/** How long a native successor keeps looking for a change before reporting that nothing changed. */@@ -657,7 +659,7 @@ export class ComputerResources {const since = baseline?.observation_id ?? frame!.observation_id;if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;// The change list replaces the full overview: only crops of the changed area are shown.- const crops = latest ? await cropImages(frame!, latest, diff) : [];+ const crops = latest ? await cropImages(frame!.png, latest.png, diff) : [];return { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } };}if (view.kind === "state" && result.details.status === "observed") {@@ -1104,7 +1106,12 @@ export class ComputerResources {return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), commandObject(value), cwd, signal, { successor: true, explicit: false });}const focus = inputFocus(commandObject(value), snapshot.result.details.ui, address.hint);- const follow = window ? await this.withFrame({ successor: true, explicit: false, ...(focus ? { focus } : {}) }, window, address.ref, cwd) : undefined;+ let follow = window ? await this.withFrame({ successor: true, explicit: false, ...(focus ? { focus } : {}) }, window, address.ref, cwd) : undefined;+ // A page action compares Playwright screenshots of the page, taken before and after it.+ if (follow && window && snapshot.result.details.dom) {+ const page = await this.pageShot(follow.session!, window, address.ref, signal);+ if (page) follow = { ...follow, page };+ }const result = await this.act(address, snapshot, value, cwd, signal);if (!follow || !window) return result;return this.followUp(window, snapshot, result, follow, cwd, signal);@@ -1144,6 +1151,13 @@ export class ComputerResources {return result;}+ /** A Playwright screenshot of a browser window's page in CSS pixels, or undefined when it cannot be taken. */+ private async pageShot(session: ComputerSession, window: string, observation: string, signal?: AbortSignal): Promise<Buffer | undefined> {+ const shot = await session.webImage(window, observation, "page", undefined, signal, true).catch(() => undefined);+ const image = shot?.details.status === "observed" ? shot.content.find(block => block.type === "image") : undefined;+ return image?.type === "image" ? Buffer.from(image.data, "base64") : undefined;+ }+/** Keep the input observation's own pixels as the baseline of the pixel comparison; they must be taken before the action. */private async withFrame(follow: FollowUp, window: string, observation: string | undefined, cwd: string): Promise<FollowUp> {if (!follow.successor) return follow;@@ -1170,6 +1184,14 @@ export class ComputerResources {const after = successor?.details;const diff = before.result.details.dom && after?.dom ? diffDom(before.result.details.dom, after.dom): settled ?? (before.result.details.ui && after?.ui ? diffUi(before.result.details.ui, after.ui) : undefined);+ if (diff && follow.page && after?.dom && after.observation_id) {+ const shot = await this.pageShot(session, window, after.observation_id, signal);+ if (shot) {+ const [a, b] = await Promise.all([decodeFrame(follow.page), decodeFrame(shot)]);+ diff.pixels = { ...diffFrames(a, b, follow.focus ? { focus: follow.focus } : {}), space: "page" };+ crops = await cropImages(follow.page, shot, diff);+ }+ }// The successor replaces the action's own observation as the current one; a wait retired the earlier snapshot.const details = { ...result.details, ...(after?.observation_id ? { observation_id: after.observation_id, ...(after.dom ? { dom: after.dom } : {}) } : {}),successor: after?.observation_id ? { observation_id: after.observation_id, ...(diff ? { diff } : { reason: "The input observation had no comparable UI or DOM." }) }@@ -1216,7 +1238,7 @@ export class ComputerResources {if (waited >= STEADY_MS) { diff.unsettled = true; break; }previous = current;}- return { successor, diff, crops: diff && frame && latest ? await cropImages(frame, latest, diff) : [] };+ return { successor, diff, crops: diff && frame && latest ? await cropImages(frame.png, latest.png, diff) : [] };}/**diff --git a/src/provider.ts b/src/provider.tsindex bc0ad1d..fb65cdc 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -225,11 +225,12 @@ function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {const share = pixels.share * 100, percent = share < 0.1 ? share.toFixed(2) : share.toFixed(1), crops = pixels.crops ?? [];const rect = (b: PixelBox) => `x=${b.x} y=${b.y} w=${b.width} h=${b.height}`;const chrome = pixels.chrome ? [`Browser chrome outside the page also changed (${rect(pixels.chrome.box)}, ${pixels.chrome.regions} ${pixels.chrome.regions === 1 ? "area" : "areas"}): tab strip, address bar or loading state; not listed.`] : [];- if (pixels.full) return [`Most of the window changed${since} (${percent}% of its overview pixels): image 1 shows the whole window after the action.`, ...chrome];+ const page = pixels.space === "page", whole = page ? "page" : "window", unit = page ? "CSS pixels of the page" : "overview pixels";+ if (pixels.full) return [`Most of the ${whole} changed${since} (${percent}% of its pixels): image 1 shows the whole ${whole} after the action.`, ...chrome];if (!pixels.areas.length) return chrome;// The overview, when attached, is image 1 and numbers the areas as listed; each crop pair follows it.const first = pixels.overview ? 2 : 1;- const lines = [`Pixels changed${since} in ${pixels.regions} ${pixels.regions === 1 ? "area" : "areas"} (overview pixels; ${percent}% of the window)${pixels.overview ? "; image 1 outlines and numbers them on the window" : ""}:`,+ const lines = [`Pixels changed${since} in ${pixels.regions} ${pixels.regions === 1 ? "area" : "areas"} (${unit}; ${percent}% of the ${whole})${pixels.overview ? `; image 1 outlines and numbers them on the ${whole}` : ""}:`,...pixels.areas.map((area, i) => {const crop = crops[i], zoom = crop && crop.scale > 1 ? `, enlarged ${crop.scale}x` : "";return ` ${i + 1}. ${rect(area)}${area.at_input ? " (at the input point)" : ""}${crop ? `; images ${first + 2 * i} and ${first + 2 * i + 1} show ${rect(crop)} before and after${zoom}` : ""}`;diff --git a/src/session.ts b/src/session.tsindex 55cab9b..e3fda98 100644--- a/src/session.ts+++ b/src/session.ts@@ -1142,14 +1142,14 @@ export class ComputerSession {}/** Playwright screenshot of the page viewport, one element, or the viewport with that element highlighted. */- webImage(ref: string, id: string, kind: "page" | "crop" | "highlight", target?: WebTarget, signal?: AbortSignal): Promise<ComputerResult> {+ webImage(ref: string, id: string, kind: "page" | "crop" | "highlight", target?: WebTarget, signal?: AbortSignal, css = false): Promise<ComputerResult> {return this.queue(async () => {const entry = await this.webEntry(ref, signal);if (!("ref" in entry)) return entry;if (entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));try {const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown }>("screenshot",- { ...(target ? { target } : {}), highlight: kind === "highlight" }, signal);+ { ...(target ? { target } : {}), highlight: kind === "highlight", ...(css ? { css } : {}) }, signal);const bytes = Buffer.from(shot.png, "base64");if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw Error("Playwright did not return a PNG.");return this.result("observed", `Playwright ${kind === "page" ? "page viewport" : kind === "crop" ? "element" : "highlighted viewport"} screenshot. These are page pixels, not native window coordinates; they grant no input.`,diff --git a/src/successor.ts b/src/successor.tsindex 8cce465..08ea790 100644--- a/src/successor.ts+++ b/src/successor.ts@@ -7,7 +7,9 @@ import { diffFrames, pixelsChanged, type DiffContext, type PixelBox, type PixelD* and numbers the listed areas on the window, `crops` the regions of the before/after pairs after it with their* scale, and `full` when most of the window changed and the only image is the whole window after the action.*/-export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; since?: string };+export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; since?: string;+ /** `page` when the pixels are Playwright screenshots of a browser page in CSS pixels, not window overviews. */+ space?: "page" };/** Compact change list between an action's input observation and its successor. Bounded; full successors stay readable. */export interface StateDiff {diff --git a/src/web-worker.ts b/src/web-worker.tsindex 5dcc6cd..20a0007 100644--- a/src/web-worker.ts+++ b/src/web-worker.ts@@ -589,7 +589,7 @@ async function check({ conditions }: { conditions: UiCondition[] }): Promise<Con}}));}-async function screenshot({ target, highlight }: { target?: WebTarget; highlight?: boolean }) {+async function screenshot({ target, highlight, css }: { target?: WebTarget; highlight?: boolean; css?: boolean }) {const tab = current(), page = idle(tab);let box;let png: Buffer;@@ -600,7 +600,8 @@ async function screenshot({ target, highlight }: { target?: WebTarget; highlightbox = await element.boundingBox({ timeout: 10_000 });if (!box) throw new Error("The element is not visible, so it cannot be highlighted.");png = await page.screenshot({ timeout: 10_000, mask: [element], maskColor: "rgba(255, 0, 96, 0.35)" });- } else png = await page.screenshot({ timeout: 10_000 });+ // In CSS pixels a page point of an action is a pixel of the screenshot, so a pixel change list can find it.+ } else png = await page.screenshot({ timeout: 10_000, ...(css ? { scale: "css" as const } : {}) });return { url: page.url(), title: tab.title, png: png.toString("base64"), ...(box ? { box } : {}) };}diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.tsindex 6d1f28d..136021e 100644--- a/test/computer-tools.test.ts+++ b/test/computer-tools.test.ts@@ -7,7 +7,8 @@ import type { MachineHost } from "../src/machine-host.ts";import { mkdtemp, readFile, rm } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";-import { computerAction, deleteComputer } from "../src/provider.ts";+import { computerAction, deleteComputer, presentComputerResult } from "../src/provider.ts";+import sharp from "sharp";import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";const cwd = "/synthetic-computer-tools";@@ -151,7 +152,8 @@ function browserFixture() {}),};const page = vi.fn(async (_kind: string, _action?: unknown) => ({ url: "https://example.com/next", title: "Next", snapshot: "- heading \"Next\" [ref=e1]", tabs: [], events: [] }));- const launch = vi.fn<WebLauncher>(async () => ({ browser: { close: vi.fn(async () => {}), page } as unknown as WebBrowser,+ const browser = { close: vi.fn(async () => {}), page } as unknown as WebBrowser;+ const launch = vi.fn<WebLauncher>(async () => ({ browser,opened: { pid: target.pid, version: "155", url: "https://example.com", title: "Page", via: "cdp" } }));const session = new ComputerSession(driver, () => "yolo", undefined, undefined, undefined, launch);const resources = new ComputerResources(async () => session, async () => session.close());@@ -160,7 +162,7 @@ function browserFixture() {expect((await session.attachBrowser(ref)).details.status).toBe("delivered");return at(`computer:window/${ref}`);};- return { resources, page, window };+ return { resources, page, window, browser };}test("computer_browser runs a page action on the window and returns the page's next snapshot", async () => {@@ -179,6 +181,26 @@ test("computer_browser runs a page action on the window and returns the page's nexpect(f.page).toHaveBeenCalledTimes(1);});+// LPT-928: a page action shows its visual change too, from Playwright screenshots of the page before and after it.+test("a page action compares page screenshots and puts the change at its point first", async () => {+ const f = browserFixture();+ const white = await sharp({ create: { width: 400, height: 300, channels: 3, background: "white" } }).png().toBuffer();+ const marked = await sharp(white).composite([{ input: { create: { width: 20, height: 20, channels: 3, background: "black" } }, left: 90, top: 140 }]).png().toBuffer();+ const shots = [white, marked];+ (f.browser as unknown as { request: unknown }).request = vi.fn(async (kind: string, args: { css?: boolean }) => {+ expect(kind).toBe("screenshot");+ expect(args.css).toBe(true);+ return { url: "https://example.com/next", title: "Next", png: (shots.shift() ?? marked).toString("base64") };+ });+ const window = await f.window();+ // The first action makes a page snapshot current; the next one is compared with it.+ await f.resources.input(window, cwd, clickPlan({ window: "x", target: "e1" }));+ const clicked = await f.resources.input(window, cwd, clickPlan({ window: "x", x: 100, y: 150 }));+ expect(clicked.details.successor?.diff?.pixels).toMatchObject({ space: "page", overview: true, areas: [{ x: 90, y: 140, width: 20, height: 20, at_input: true }] });+ expect(presentComputerResult(clicked)).toMatch(/Pixels changed in 1 area \(CSS pixels of the page; [\d.]+% of the page\); image 1 outlines and numbers them on the page:\n 1\. x=90 y=140 w=20 h=20 \(at the input point\); images 2 and 3 show/);+ expect(clicked.content.filter(block => block.type === "image")).toHaveLength(3);+});+test("typed input on a browser window goes to Playwright with page refs and locators", async () => {const f = browserFixture();const window = await f.window();