Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: the pixel change list outlines and numbers the changed areas on the window, puts the area at the input first, enlarges small crops and summarises browser chrome (LPT-928)

alexshpunt-clanker[bot]ceabe774ca5f
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index df365ec..e0c43a3 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -12,9 +12,9 @@ import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./se
import { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";
import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";
import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";
-import { changeCrops, decodeFrame } from "./pixel-diff.ts";
+import { changeCrops, changeOverview, decodeFrame, MOST_CHANGED, type DiffContext } from "./pixel-diff.ts";
import type { WindowFrame } from "./session.ts";
-import type { UiCaptureOptions, UiElement, UiResource } from "./ui-resources.ts";
+import { webContentBox, type UiCaptureOptions, type UiElement, type UiResource } from "./ui-resources.ts";
import { displayMode } from "./display-mode.ts";
import { isSemanticAction } from "./semantic-actions.ts";
import { accessibleName, ariaRole, checkedState, isVisible, nativeAriaSnapshot, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";
@@ -28,14 +28,44 @@ export function machineGuide(source: string): string[] {
}
/** A rejected request whose validation completed before native dispatch. */
export class ComputerRequestError extends Error {}
-/** A before and an after PNG crop of each of a diff's largest changed areas; records the cropped regions on the diff. */
+/**
+ * The images that show a diff's pixel change, recorded on the diff: when most of the window changed, the whole window
+ * after the action; otherwise an overview of the window with the listed areas outlined and numbered, then a before and
+ * an after crop of each of the first areas.
+ */
async function cropImages(before: WindowFrame, after: WindowFrame, diff: StateDiff): Promise<ComputerResult["content"]> {
- if (!diff.pixels?.box) return [];
- const cropped = await changeCrops(before.png, after.png, diff.pixels).catch(() => []);
- if (!cropped.length) return [];
- diff.pixels.crops = cropped.map(item => item.region);
- return cropped.flatMap(item => [{ type: "image" as const, data: item.before.toString("base64"), mimeType: "image/png" },
- { type: "image" as const, data: item.after.toString("base64"), mimeType: "image/png" }]);
+ const pixels = diff.pixels;
+ if (!pixels?.box || pixels.size_changed) return [];
+ const image = (png: Buffer) => ({ type: "image" as const, data: png.toString("base64"), mimeType: "image/png" });
+ if (pixels.share >= MOST_CHANGED) {
+ pixels.full = true;
+ return [image(after.png)];
+ }
+ const [overview, cropped] = await Promise.all([changeOverview(after.png, pixels).catch(() => undefined), changeCrops(before.png, after.png, pixels).catch(() => [])]);
+ if (overview) pixels.overview = true;
+ pixels.crops = cropped.map(item => ({ ...item.region, scale: item.scale }));
+ return [...(overview ? [image(overview)] : []), ...cropped.flatMap(item => [image(item.before), image(item.after)])];
+}
+/**
+ * What a pixel comparison of a window knows about the action: its input point in the input observation's pixels,
+ * and in a browser window the web page, so changes outside it count as browser chrome. A window whose web content
+ * covers most of it is treated as a browser; an app with a small embedded web view is not.
+ */
+function diffContext(ui: UiResource | undefined, image: { width: number; height: number } | undefined, focus: InputPoint | undefined): DiffContext {
+ const page = webContentBox(ui);
+ const browser = page && image && page.width * page.height >= 0.6 * image.width * image.height;
+ return { ...(focus ? { focus } : {}), ...(browser ? { page } : {}) };
+}
+/**
+ * Where an input command lands in its observation's pixels: the end of a drag, the last pointed event of a timeline,
+ * its point, or the centre of its ref. Undefined for input without a place, such as keys to the focused element.
+ */
+function inputFocus(command: Record<string, unknown>, ui: UiResource | undefined, hint: string | undefined): InputPoint | undefined {
+ const point = (x: unknown, y: unknown) => typeof x === "number" && typeof y === "number" ? { x, y } : undefined;
+ const events = Array.isArray(command.events) ? (command.events as { x?: unknown; y?: unknown }[]).filter(event => point(event.x, event.y)) : [];
+ const box = hint ? ui?.elements.find(element => element.hint === hint)?.bounds.window_image : undefined;
+ return point(command.to_x, command.to_y) ?? (events.length ? point(events.at(-1)!.x, events.at(-1)!.y) : undefined) ?? point(command.x, command.y)
+ ?? (box ? { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } : undefined);
}
function commandObject(value: string): Record<string, unknown> {
try { return object(JSON.parse(value)); }
@@ -82,6 +112,8 @@ const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "do
* `frame` holds the window's pixels before the action, when the input observation had them.
*/
interface FollowUp { successor: boolean; explicit: boolean; frame?: WindowFrame;
+ /** Where the input landed, in the input observation's pixels; the pixel change list puts areas near it first. */
+ focus?: InputPoint;
/** The window's session, resolved before the action: a failed action can make the session forget the window. */
session?: ComputerSession }
/** How long a native successor keeps looking for a change before reporting that nothing changed. */
@@ -618,8 +650,9 @@ export class ComputerResources {
if (view.changes && result.details.status === "observed") {
const captured = session.windowFrame(windowRef!);
const latest = frame && captured?.observation_id === result.details.observation_id ? captured : undefined;
- const diff = compareCaptures({ ui: baseline?.ui, frame: latest ? await decodeFrame(frame!.png) : undefined },
- { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined });
+ const earlier = latest ? await decodeFrame(frame!.png) : undefined;
+ const diff = compareCaptures({ ui: baseline?.ui, frame: earlier }, { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined },
+ diffContext(result.details.ui, earlier, undefined));
if (!diff) return { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } };
const since = baseline?.observation_id ?? frame!.observation_id;
if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;
@@ -1070,7 +1103,8 @@ export class ComputerResources {
this.snapshots.delete(address.ref);
return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), commandObject(value), cwd, signal, { successor: true, explicit: false });
}
- const follow = window ? await this.withFrame({ successor: true, explicit: false }, window, address.ref, cwd) : undefined;
+ const focus = inputFocus(commandObject(value), snapshot.result.details.ui, address.hint);
+ const follow = window ? await this.withFrame({ successor: true, explicit: false, ...(focus ? { focus } : {}) }, window, address.ref, cwd) : undefined;
const result = await this.act(address, snapshot, value, cwd, signal);
if (!follow || !window) return result;
return this.followUp(window, snapshot, result, follow, cwd, signal);
@@ -1131,7 +1165,7 @@ export class ComputerResources {
let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];
if (status !== "refused") {
if (result.details.dom) successor = result;
- else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal));
+ else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal, follow.focus));
}
const after = successor?.details;
const diff = before.result.details.dom && after?.dom ? diffDom(before.result.details.dom, after.dom)
@@ -1149,11 +1183,12 @@ export class ComputerResources {
* to STEADY_MS, so the change list shows the settled window rather than a frame in the middle of a repaint.
* Captures keep the input observation's overview size so their pixels compare with it.
*/
- private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal):
+ private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal, focus?: InputPoint):
Promise<{ successor: ComputerResult; diff?: StateDiff; crops: ComputerResult["content"] }> {
const started = performance.now();
let options: UiCaptureOptions = { granularity: "full", ...(frame ? frame.size : {}) };
const baseline: Capture = { ui: before.result.details.ui, frame: frame ? await decodeFrame(frame.png) : undefined };
+ const context = diffContext(before.result.details.ui, baseline.frame, focus);
let successor: ComputerResult, diff: StateDiff | undefined, previous: Capture | undefined, latest: WindowFrame | undefined;
for (let attempt = 0; ; attempt++) {
this.retireWindow(window);
@@ -1167,7 +1202,7 @@ export class ComputerResources {
const captured = session.windowFrame(window);
latest = captured?.observation_id === successor.details.observation_id ? captured : undefined;
const current: Capture = { ui: successor.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined };
- diff = compareCaptures(baseline, current);
+ diff = compareCaptures(baseline, current, context);
if (!diff) break;
diff.waited_ms = waited;
if (diffIsEmpty(diff)) {
@@ -1252,7 +1287,9 @@ export class ComputerResources {
window_ref: window, observation_id: observation.details.observation_id };
// The walk that resolved the target is the state before the action: the change list compares with it.
const before: Snapshot = { target: { window_ref: window }, result: observation };
- if (follow) follow = await this.withFrame(follow, window, observation.details.observation_id, cwd);
+ const box = element.bounds.window_image;
+ if (follow) follow = await this.withFrame({ ...follow, ...(box ? { focus: { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) } } : {}) },
+ window, observation.details.observation_id, cwd);
const finish = (result: ComputerResult) => follow ? this.followUp(window, before, result, follow, cwd, signal) : Promise.resolve(result);
this.snapshots.delete(observation.details.observation_id!);
const result = await session.act(nativeCommand as Action, signal);
diff --git a/src/pixel-diff.ts b/src/pixel-diff.ts
index ebaccf0..28a6171 100644
--- a/src/pixel-diff.ts
+++ b/src/pixel-diff.ts
@@ -4,7 +4,9 @@ import sharp from "sharp";
export interface PixelFrame { width: number; height: number; data: Buffer }
export interface PixelBox { x: number; y: number; width: number; height: number }
/** One changed area of the window: nearby changed spots merged, with the number of its changed pixels. */
-export interface PixelArea extends PixelBox { pixels: number }
+export interface PixelArea extends PixelBox { pixels: number; /** The input point of the action lies in this area. */ at_input?: true }
+/** What a comparison knows about the action: where its input landed and which part of the window shows a web page. */
+export interface DiffContext { focus?: { x: number; y: number }; page?: PixelBox }
/** How a window's overview pixels changed between two captures, in overview pixels. */
export interface PixelDiff {
/** Overview size of both captures; when they differ, `size_changed` holds the earlier size and nothing else is compared. */
@@ -12,10 +14,12 @@ export interface PixelDiff {
size_changed?: { width: number; height: number };
/** Box around every counted change; absent when nothing counted changed. */
box?: PixelBox;
- /** The largest changed areas by box size, at most AREA_LIMIT of them. */
+ /** The changed areas, at most AREA_LIMIT of them: those at or nearest to the input point first, otherwise the largest. */
areas: PixelArea[];
- /** All changed areas, including any beyond `areas`. */
+ /** All changed areas, including any beyond `areas` but not the browser chrome ones. */
regions: number;
+ /** Changes of a browser window outside its web page, such as the tab strip and address bar, summarised. */
+ chrome?: { box: PixelBox; regions: number };
/** Changed pixels that count, and their share of the whole overview. */
changed_pixels: number; share: number;
/** Spots too small to count, such as a blinking caret. */
@@ -40,7 +44,7 @@ export async function decodeFrame(png: Buffer): Promise<PixelFrame> {
}
/** Compare two captures of the same window: which pixels changed, where, and how much, ignoring caret-sized spots. */
-export function diffFrames(before: PixelFrame, after: PixelFrame): PixelDiff {
+export function diffFrames(before: PixelFrame, after: PixelFrame, context: DiffContext = {}): PixelDiff {
const { width, height } = after;
if (before.width !== width || before.height !== height)
return { width, height, size_changed: { width: before.width, height: before.height }, areas: [], regions: 0, changed_pixels: 0, share: 0, ignored: 0 };
@@ -88,11 +92,20 @@ export function diffFrames(before: PixelFrame, after: PixelFrame): PixelDiff {
if (pixels < MIN_SPOT_PIXELS || (spot.width <= CARET_WIDTH && spot.height <= CARET_HEIGHT)) { ignored++; continue; }
spots.push(spot);
}
- // Largest by extent: a thin stroke across the canvas outranks a denser but small repainted label.
- const areas = mergeNearby(spots).sort((x, y) => y.width * y.height - x.width * x.height || y.pixels - x.pixels);
- const changed = areas.reduce((sum, area) => sum + area.pixels, 0);
- const box = areas.length ? areas.map(({ pixels: _, ...area }) => area).reduce(union) : undefined;
- return { width, height, ...(box ? { box } : {}), areas: areas.slice(0, AREA_LIMIT), regions: areas.length, changed_pixels: changed, share: changed / (width * height), ignored };
+ const merged = mergeNearby(spots);
+ const changed = merged.reduce((sum, area) => sum + area.pixels, 0);
+ const box = merged.length ? merged.map(({ pixels: _, ...area }) => area).reduce(union) : undefined;
+ // In a browser window, changes outside the web page (tab strip, address bar, spinner) are summarised, not listed.
+ const page = context.page, chromeAreas = page ? merged.filter(area => !intersects(area, page)) : [];
+ const areas = page ? merged.filter(area => intersects(area, page)) : merged;
+ const focus = context.focus;
+ for (const area of areas) if (focus && distance(area, focus) === 0) area.at_input = true;
+ // At or nearest to the input point first; otherwise the largest by extent, so a thin stroke across the canvas
+ // outranks a denser but small repainted label.
+ areas.sort((x, y) => (focus ? distance(x, focus) - distance(y, focus) : 0) || y.width * y.height - x.width * x.height || y.pixels - x.pixels);
+ const chrome = chromeAreas.length ? { box: chromeAreas.map(({ pixels: _, ...area }) => area).reduce(union), regions: chromeAreas.length } : undefined;
+ return { width, height, ...(box ? { box } : {}), areas: areas.slice(0, AREA_LIMIT), regions: areas.length, ...(chrome ? { chrome } : {}),
+ changed_pixels: changed, share: changed / (width * height), ignored };
}
export const pixelsChanged = (diff: PixelDiff) => diff.size_changed !== undefined || diff.box !== undefined;
@@ -112,6 +125,13 @@ function mergeNearby(spots: PixelArea[]): PixelArea[] {
return areas;
}
+const intersects = (a: PixelBox, b: PixelBox) => a.x < b.x + b.width && b.x < a.x + a.width && a.y < b.y + b.height && b.y < a.y + a.height;
+/** How far a point lies from a box, 0 inside it. */
+function distance(box: PixelBox, point: { x: number; y: number }): number {
+ const dx = Math.max(box.x - point.x, 0, point.x - (box.x + box.width - 1)), dy = Math.max(box.y - point.y, 0, point.y - (box.y + box.height - 1));
+ return Math.hypot(dx, dy);
+}
+
function union(a: PixelBox, b: PixelBox): PixelBox {
const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y);
return { x, y, width: Math.max(a.x + a.width, b.x + b.width) - x, height: Math.max(a.y + a.height, b.y + b.height) - y };
@@ -119,21 +139,63 @@ function union(a: PixelBox, b: PixelBox): PixelBox {
/** Each crop's long edge is at most this many pixels, so a few crops cost little next to a full overview. */
const CROP_EDGE = 512;
+/** A small area is enlarged until its crop's long edge reaches this, so a checkbox or a tile is easy to see. */
+const CROP_MIN_EDGE = 192;
+/** Enlargement never exceeds this factor. */
+const CROP_MAX_SCALE = 4;
/** Context kept around a changed area on each side, in overview pixels. */
const CROP_MARGIN = 12;
-/** Only the largest areas get crops; the rest are listed by their boxes. */
+/** Only the first areas get crops; the rest are listed by their boxes. */
export const CROPPED_AREAS = 3;
+/** When at least this share of the window changed, the whole window is shown instead of crops. */
+export const MOST_CHANGED = 0.5;
+/** Long edge of the overview that outlines and numbers the changed areas. */
+const OVERVIEW_EDGE = 640;
-/** Before and after PNG crops of the largest changed areas with a small margin, scaled down when large. */
-export async function changeCrops(before: Buffer, after: Buffer, diff: PixelDiff): Promise<{ region: PixelBox; before: Buffer; after: Buffer }[]> {
+/** Before and after PNG crops of the first changed areas with a small margin, enlarged when small and scaled down when large. */
+export async function changeCrops(before: Buffer, after: Buffer, diff: PixelDiff): Promise<{ region: PixelBox; scale: number; before: Buffer; after: Buffer }[]> {
if (diff.size_changed) return [];
- const crop = (png: Buffer, region: PixelBox) => sharp(png, { limitInputPixels: 32 * 1024 * 1024 })
- .extract({ left: region.x, top: region.y, width: region.width, height: region.height })
- .resize(CROP_EDGE, CROP_EDGE, { fit: "inside", withoutEnlargement: true }).png().toBuffer();
return Promise.all(diff.areas.slice(0, CROPPED_AREAS).map(async area => {
const x = Math.max(0, area.x - CROP_MARGIN), y = Math.max(0, area.y - CROP_MARGIN);
const region = { x, y, width: Math.min(diff.width, area.x + area.width + CROP_MARGIN) - x, height: Math.min(diff.height, area.y + area.height + CROP_MARGIN) - y };
- const [a, b] = await Promise.all([crop(before, region), crop(after, region)]);
- return { region, before: a, after: b };
+ const long = Math.max(region.width, region.height);
+ const scale = long < CROP_MIN_EDGE ? Math.min(CROP_MAX_SCALE, Math.floor(CROP_MIN_EDGE / long) || 1) : Math.min(1, CROP_EDGE / long);
+ const width = Math.max(1, Math.round(region.width * scale)), height = Math.max(1, Math.round(region.height * scale));
+ // Enlarged pixels stay square, so a checkmark is not blurred into its box.
+ const crop = (png: Buffer) => sharp(png, { limitInputPixels: 32 * 1024 * 1024 })
+ .extract({ left: region.x, top: region.y, width: region.width, height: region.height })
+ .resize(width, height, { kernel: scale > 1 ? "nearest" : "lanczos3" }).png().toBuffer();
+ const [b, a] = await Promise.all([crop(before), crop(after)]);
+ return { region, scale: Math.round(scale * 100) / 100, before: b, after: a };
}));
}
+
+/** Digits as 3x5 cells, so labels need no font. */
+const DIGITS = ["111101101101111", "010110010010111", "111001111100111", "111001111001111", "101101111001001", "111100111001111", "111100111101111", "111001001001001", "111101111101111", "111101111001111"];
+
+/** SVG rectangles drawing a number at (x, y), each cell `cell` pixels. */
+function digits(value: number, x: number, y: number, cell: number): string {
+ return [...String(value)].map((digit, index) => [...DIGITS[Number(digit)]].map((on, i) => on === "1"
+ ? `<rect x="${x + index * 4 * cell + (i % 3) * cell}" y="${y + Math.floor(i / 3) * cell}" width="${cell}" height="${cell}" fill="#fff"/>` : "").join("")).join("");
+}
+
+/**
+ * The window after the action, scaled to OVERVIEW_EDGE, with each listed changed area outlined and numbered in list
+ * order, and the browser chrome changes outlined without a number.
+ */
+export async function changeOverview(after: Buffer, diff: PixelDiff): Promise<Buffer> {
+ const scale = Math.min(1, OVERVIEW_EDGE / Math.max(diff.width, diff.height));
+ const width = Math.max(1, Math.round(diff.width * scale)), height = Math.max(1, Math.round(diff.height * scale));
+ const box = (b: PixelBox) => ({ x: Math.round(b.x * scale), y: Math.round(b.y * scale), w: Math.max(2, Math.round(b.width * scale)), h: Math.max(2, Math.round(b.height * scale)) });
+ const cell = 2, label = (n: number) => String(n).length * 4 * cell + 2 * cell;
+ const marks = diff.areas.map((area, i) => {
+ const r = box(area), w = label(i + 1), h = 7 * cell;
+ // The number sits above the outline, or inside it at the top edge of the image.
+ const ly = r.y - h >= 0 ? r.y - h : r.y;
+ return `<rect x="${r.x - 1}" y="${r.y - 1}" width="${r.w + 2}" height="${r.h + 2}" fill="none" stroke="#ff1f8f" stroke-width="2"/>`
+ + `<rect x="${r.x - 1}" y="${ly}" width="${w}" height="${h}" fill="#ff1f8f"/>${digits(i + 1, r.x - 1 + cell, ly + cell, cell)}`;
+ });
+ const chrome = diff.chrome ? (() => { const r = box(diff.chrome!.box); return `<rect x="${r.x}" y="${r.y}" width="${r.w}" height="${r.h}" fill="none" stroke="#ffb000" stroke-width="1" stroke-dasharray="4 3"/>`; })() : "";
+ const svg = Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">${chrome}${marks.join("")}</svg>`);
+ return sharp(after, { limitInputPixels: 32 * 1024 * 1024 }).resize(width, height).composite([{ input: svg, top: 0, left: 0 }]).png().toBuffer();
+}
diff --git a/src/provider.ts b/src/provider.ts
index a0bafed..bc0ad1d 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -222,12 +222,20 @@ function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {
const since = pixels.since ? ` since ${computerSource("observation", pixels.since)}` : "";
if (pixels.size_changed) return [`The window's overview size changed${since} from ${pixels.size_changed.width}x${pixels.size_changed.height} to ${pixels.width}x${pixels.height}; pixels were not compared.`];
if (!pixels.box) return uiChanged ? [`No visible pixel change${since}.`] : [];
- const share = pixels.share * 100, crops = pixels.crops ?? [];
+ const share = pixels.share * 100, percent = share < 0.1 ? share.toFixed(2) : share.toFixed(1), crops = pixels.crops ?? [];
const rect = (b: PixelBox) => `x=${b.x} y=${b.y} w=${b.width} h=${b.height}`;
- const lines = [`Pixels changed${since} in ${pixels.regions} ${pixels.regions === 1 ? "area" : "areas"} (overview pixels; ${share < 0.1 ? share.toFixed(2) : share.toFixed(1)}% of the window):`,
- ...pixels.areas.map((area, i) => ` ${rect(area)}${crops[i] ? `; images ${2 * i + 1} and ${2 * i + 2} show ${rect(crops[i])} before and after` : ""}`)];
- if (pixels.regions > pixels.areas.length) lines.push(` …and ${pixels.regions - pixels.areas.length} smaller areas.`);
- return lines;
+ const chrome = pixels.chrome ? [`Browser chrome outside the page also changed (${rect(pixels.chrome.box)}, ${pixels.chrome.regions} ${pixels.chrome.regions === 1 ? "area" : "areas"}): tab strip, address bar or loading state; not listed.`] : [];
+ if (pixels.full) return [`Most of the window changed${since} (${percent}% of its overview pixels): image 1 shows the whole window after the action.`, ...chrome];
+ if (!pixels.areas.length) return chrome;
+ // The overview, when attached, is image 1 and numbers the areas as listed; each crop pair follows it.
+ const first = pixels.overview ? 2 : 1;
+ const lines = [`Pixels changed${since} in ${pixels.regions} ${pixels.regions === 1 ? "area" : "areas"} (overview pixels; ${percent}% of the window)${pixels.overview ? "; image 1 outlines and numbers them on the window" : ""}:`,
+ ...pixels.areas.map((area, i) => {
+ const crop = crops[i], zoom = crop && crop.scale > 1 ? `, enlarged ${crop.scale}x` : "";
+ return ` ${i + 1}. ${rect(area)}${area.at_input ? " (at the input point)" : ""}${crop ? `; images ${first + 2 * i} and ${first + 2 * i + 1} show ${rect(crop)} before and after${zoom}` : ""}`;
+ })];
+ if (pixels.regions > pixels.areas.length) lines.push(` …and ${pixels.regions - pixels.areas.length} more areas, not listed; Read the window for all of it.`);
+ return [...lines, ...chrome];
}
/** Whole lines from the start that fit a byte budget, never splitting Unicode. */
diff --git a/src/successor.ts b/src/successor.ts
index 19a7a55..8cce465 100644
--- a/src/successor.ts
+++ b/src/successor.ts
@@ -1,9 +1,13 @@
import type { UiElement, UiResource } from "./ui-resources.ts";
import type { WebDom } from "./session.ts";
-import { diffFrames, pixelsChanged, type PixelBox, type PixelDiff, type PixelFrame } from "./pixel-diff.ts";
+import { diffFrames, pixelsChanged, type DiffContext, type PixelBox, type PixelDiff, type PixelFrame } from "./pixel-diff.ts";
-/** How a window's overview pixels changed; `crops` are the regions the attached before/after image pairs show, in order. */
-export type PixelReport = PixelDiff & { crops?: PixelBox[]; since?: string };
+/**
+ * How a window's overview pixels changed and what the attached images show: `overview` when the first image outlines
+ * and numbers the listed areas on the window, `crops` the regions of the before/after pairs after it with their
+ * scale, and `full` when most of the window changed and the only image is the whole window after the action.
+ */
+export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; since?: string };
/** Compact change list between an action's input observation and its successor. Bounded; full successors stay readable. */
export interface StateDiff {
@@ -138,8 +142,8 @@ export const diffIsEmpty = (diff: StateDiff): boolean =>
/** One native capture of a window: its UI walk and its decoded overview pixels, either of which may be missing. */
export interface Capture { ui?: UiResource; frame?: PixelFrame }
/** Compare two native captures of a window: UI elements when both have UI, pixels when both have pixels. */
-export function compareCaptures(before: Capture, after: Capture): StateDiff | undefined {
- const pixels = before.frame && after.frame ? diffFrames(before.frame, after.frame) : undefined;
+export function compareCaptures(before: Capture, after: Capture, context: DiffContext = {}): StateDiff | undefined {
+ const pixels = before.frame && after.frame ? diffFrames(before.frame, after.frame, context) : undefined;
const ui = before.ui && after.ui ? diffUi(before.ui, after.ui) : undefined;
if (ui) return pixels ? { ...ui, pixels } : ui;
return pixels ? { kind: "pixels", changed: [], added: [], removed: [], counts: { changed: 0, added: 0, removed: 0 }, pixels } : undefined;
diff --git a/src/ui-resources.ts b/src/ui-resources.ts
index 72b9694..84f1001 100644
--- a/src/ui-resources.ts
+++ b/src/ui-resources.ts
@@ -15,6 +15,8 @@ export interface UiElement {
uia_name?: string; automation_id?: string; class_name?: string; help_text?: string;
checked?: "on" | "off" | "mixed"; expanded?: boolean; offscreen?: boolean; focused?: boolean;
password?: boolean; read_only?: boolean; required?: boolean; actionable?: boolean;
+ /** Set on elements of a web page in a browser or an embedded browser view. */
+ in_web_content?: boolean;
range?: { value: number; min: number; max: number };
/** Whether the hint label is drawn on the overview; `read_only` marks static text and containers, which get no label. */
hint_display: "pending" | "shown" | "disabled" | "read_only" | "no_bounds" | "outside_image" | "collision" | "off";
@@ -72,7 +74,7 @@ function structural(e: Record<string, unknown>): Partial<UiElement> {
for (const [from, to] of [["name", "uia_name"], ["automation_id", "automation_id"], ["class_name", "class_name"], ["help_text", "help_text"]] as const)
if (typeof e[from] === "string" && e[from]) out[to] = (e[from] as string).slice(0, 1024);
if (e.checked === "on" || e.checked === "off" || e.checked === "mixed") out.checked = e.checked;
- for (const key of ["expanded", "offscreen", "focused", "password", "read_only", "required", "actionable"] as const)
+ for (const key of ["expanded", "offscreen", "focused", "password", "read_only", "required", "actionable", "in_web_content"] as const)
if (typeof e[key] === "boolean") out[key] = e[key] as boolean;
const range = e.range as { value?: unknown; min?: unknown; max?: unknown } | undefined;
if (range && [range.value, range.min, range.max].every(Number.isFinite)) out.range = { value: range.value as number, min: range.min as number, max: range.max as number };
@@ -204,3 +206,16 @@ export async function annotateUi(image: Content, resource: UiResource, width: nu
const marked = await sharp(bytes).composite([{ input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}">${svg.join("")}</svg>`) }]).png().toBuffer();
return { image: { type: "image", mimeType: "image/png", data: marked.toString("base64") }, annotation };
}
+
+/**
+ * The part of a window image that shows web page content: the bounds of its largest web content element, such as a
+ * browser's document. Undefined when the window shows no web content.
+ */
+export function webContentBox(ui: UiResource | undefined): ScreenRect | undefined {
+ let best: ScreenRect | undefined;
+ for (const element of ui?.elements ?? []) {
+ const box = element.bounds.window_image;
+ if (element.in_web_content && box && (!best || box.width * box.height > best.width * best.height)) best = box;
+ }
+ return best;
+}
diff --git a/test/action-changes.test.ts b/test/action-changes.test.ts
index adb0e24..fec19c3 100644
--- a/test/action-changes.test.ts
+++ b/test/action-changes.test.ts
@@ -63,9 +63,11 @@ const images = (result: ComputerResult) => result.content.filter((block): block
test("an action on an observation with pixels reports the changed area with before and after crops", async () => {
const f = await fixture(), { at } = await f.read(["full"]);
const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
- expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, crops: [{ x: 8, y: 28, width: 34, height: 34 }] });
- expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in 1 area \(overview pixels; 0\.06% of the window\):\n x=20 y=40 w=10 h=10; images 1 and 2 show x=8 y=28 w=34 h=34 before and after\n/);
- const [before, after] = images(result);
+ expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, overview: true, crops: [{ x: 8, y: 28, width: 34, height: 34, scale: 4 }] });
+ expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in 1 area \(overview pixels; 0\.06% of the window\); image 1 outlines and numbers them on the window:\n 1\. x=20 y=40 w=10 h=10( \(at the input point\))?; images 2 and 3 show x=8 y=28 w=34 h=34 before and after, enlarged 4x\n/);
+ // Image 1 is the window with the area outlined; the crops follow it.
+ const [overview, before, after] = images(result);
+ expect(await sharp(Buffer.from(overview.data, "base64")).metadata()).toMatchObject({ width: expect.any(Number) });
expect((await sharp(Buffer.from(before.data, "base64")).stats()).channels[0].min).toBe(255);
expect((await sharp(Buffer.from(after.data, "base64")).stats()).channels[0].min).toBe(0);
});
@@ -74,7 +76,9 @@ test("pixel input on a canvas reports only its pixel change, and empty space rep
const f = await fixture(), { observation } = await f.read(["full"]);
const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }));
expect(stroke.details.successor?.diff).toMatchObject({ counts: { changed: 0, added: 0, removed: 0 }, pixels: { box: { x: 297, y: 197, width: 6, height: 6 } } });
- expect(images(stroke)).toHaveLength(2);
+ // The overview and one before/after pair; the stroke is at the input point, so it is area 1.
+ expect(images(stroke)).toHaveLength(3);
+ expect(stroke.details.successor?.diff?.pixels?.areas[0]).toMatchObject({ at_input: true });
const next = address(computerSource("observation", stroke.details.successor!.observation_id!));
const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }));
expect(presentComputerResult(empty)).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);
@@ -97,9 +101,9 @@ test("a changes Read compares the window's pixels with its previous observation"
f.paint(200, 150);
const changes = (await f.resources.read(address(f.window), cwd, ["changes"]))!;
expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 197, y: 147, width: 6, height: 6 });
- expect(presentComputerResult(changes)).toMatch(/Pixels changed in 1 area .*\n x=197 y=147 w=6 h=6;/);
- // Only the two crops, not the full overview.
- expect(images(changes)).toHaveLength(2);
+ expect(presentComputerResult(changes)).toMatch(/Pixels changed in 1 area .*\n 1\. x=197 y=147 w=6 h=6;/);
+ // The outlined overview and the two crops, not the full-size window.
+ expect(images(changes)).toHaveLength(3);
});
test("an action reports what it changed by default, and its successor is the current observation", async () => {
diff --git a/test/pixel-diff.test.ts b/test/pixel-diff.test.ts
index 657648d..db9641d 100644
--- a/test/pixel-diff.test.ts
+++ b/test/pixel-diff.test.ts
@@ -1,6 +1,6 @@
import sharp from "sharp";
import { describe, expect, test } from "vitest";
-import { changeCrops, decodeFrame, diffFrames, pixelsChanged, type PixelFrame } from "../src/pixel-diff.ts";
+import { changeCrops, changeOverview, decodeFrame, diffFrames, pixelsChanged, type PixelFrame } from "../src/pixel-diff.ts";
const blank = (width = 200, height = 100): PixelFrame => ({ width, height, data: Buffer.alloc(width * height * 4, 255) });
function paint(frame: PixelFrame, x: number, y: number, width: number, height: number, value = 0): PixelFrame {
@@ -65,14 +65,55 @@ describe("diffFrames", () => {
});
});
+// LPT-928: the change at the input comes first, and browser chrome changes are summarised.
+describe("diffFrames with an action context", () => {
+ test("the area at the input point comes first even when another area is larger", () => {
+ const before = blank(400, 200);
+ const after = paint(paint(before, 10, 10, 200, 60), 300, 150, 12, 12);
+ expect(diffFrames(before, after).areas.map(area => area.x)).toEqual([10, 300]);
+ const focused = diffFrames(before, after, { focus: { x: 305, y: 155 } });
+ expect(focused.areas.map(area => [area.x, area.at_input])).toEqual([[300, true], [10, undefined]]);
+ });
+
+ test("without an area under the input point, the nearest one comes first", () => {
+ const before = blank(400, 200);
+ const after = paint(paint(before, 10, 10, 200, 60), 300, 150, 12, 12);
+ expect(diffFrames(before, after, { focus: { x: 330, y: 190 } }).areas.map(area => area.x)).toEqual([300, 10]);
+ });
+
+ test("changes outside the web page of a browser window are summarised as chrome", () => {
+ const before = blank(400, 300);
+ const after = paint(paint(before, 20, 5, 60, 20), 100, 150, 15, 15);
+ const diff = diffFrames(before, after, { page: { x: 0, y: 60, width: 400, height: 240 } });
+ expect(diff.areas.map(area => area.x)).toEqual([100]);
+ expect(diff.regions).toBe(1);
+ expect(diff.chrome).toEqual({ box: { x: 20, y: 5, width: 60, height: 20 }, regions: 1 });
+ });
+});
+
+describe("changeOverview", () => {
+ test("outlines each listed area on a scaled copy of the window", async () => {
+ const before = blank(1280, 640), after = paint(before, 600, 300, 40, 40);
+ const diff = diffFrames(before, after);
+ const overview = await changeOverview(await png(after), diff);
+ const meta = await sharp(overview).metadata();
+ expect([meta.width, meta.height]).toEqual([640, 320]);
+ // The outline is drawn in its own colour just outside the area, at half scale.
+ const frame = await decodeFrame(overview), at = (x: number, y: number) => [...frame.data.subarray((y * frame.width + x) * 4, (y * frame.width + x) * 4 + 3)];
+ expect(at(299, 170)).toEqual([255, 31, 143]);
+ expect(at(100, 100)).toEqual([255, 255, 255]);
+ });
+});
+
describe("changeCrops", () => {
- test("crops each changed area with a margin from both captures", async () => {
+ test("crops each changed area with a margin from both captures, enlarging a small one with square pixels", async () => {
const before = blank(), after = paint(blank(), 50, 30, 20, 10);
const [crop] = await changeCrops(await png(before), await png(after), diffFrames(before, after));
expect(crop.region).toEqual({ x: 38, y: 18, width: 44, height: 34 });
+ expect(crop.scale).toBe(4);
const a = await decodeFrame(crop.before), b = await decodeFrame(crop.after);
- expect([a.width, a.height, b.width, b.height]).toEqual([44, 34, 44, 34]);
- expect(diffFrames(a, b).box).toEqual({ x: 12, y: 12, width: 20, height: 10 });
+ expect([a.width, a.height, b.width, b.height]).toEqual([176, 136, 176, 136]);
+ expect(diffFrames(a, b).box).toEqual({ x: 48, y: 48, width: 80, height: 40 });
});
test("only the three largest areas get crops", async () => {