Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

WIP: actions report the changed area of the window's pixels (LPT-915)

alexshpunt-clanker[bot]04e3a541367c
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 0078047..e29f9e3 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -11,8 +11,10 @@ import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./se
import { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";
import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";
import { combine, evaluateNative, type WaitSpec } from "./ui-conditions.ts";
-import { diffDom, diffIsEmpty, diffUi, type StateDiff } from "./successor.ts";
-import type { UiElement, UiResource } from "./ui-resources.ts";
+import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";
+import { changeCrops, decodeFrame } from "./pixel-diff.ts";
+import type { WindowFrame } from "./session.ts";
+import type { UiCaptureOptions, UiElement, UiResource } from "./ui-resources.ts";
import { displayMode } from "./display-mode.ts";
import { isSemanticAction } from "./semantic-actions.ts";
import { accessibleName, ariaRole, checkedState, isVisible, nativeAriaSnapshot, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";
@@ -27,19 +29,20 @@ export function machineGuide(source: string): string[] {
}
/** A rejected request whose validation completed before native dispatch. */
export class ComputerRequestError extends Error {}
+/** Before and after PNG crops of a diff's changed pixels, in that order; records the cropped region on the diff. */
+async function cropImages(before: WindowFrame, after: WindowFrame, diff: StateDiff): Promise<ComputerResult["content"]> {
+ if (!diff.pixels?.box) return [];
+ const cropped = await changeCrops(before.png, after.png, diff.pixels).catch(() => undefined);
+ if (!cropped) return [];
+ diff.pixels.crop = cropped.region;
+ return [{ type: "image", data: cropped.before.toString("base64"), mimeType: "image/png" },
+ { type: "image", data: cropped.after.toString("base64"), mimeType: "image/png" }];
+}
function commandObject(value: string): Record<string, unknown> {
try { return object(JSON.parse(value)); }
catch { throw new ComputerRequestError("Use one valid JSON command object."); }
}
-/** Whether an action is aimed at a UI element (a hint, a token, a semantic action or a menu), not only at pixels or keys. */
-function elementAddressed(address: Extract<ComputerAddress, { kind: "observation" }>, tool: "write" | "insert", input: string): boolean {
- if (address.hint !== undefined) return true;
- if (tool === "write") return false;
- let command: Record<string, unknown>;
- try { command = commandObject(input); } catch { return false; }
- const steps = Array.isArray(command.sequence) ? command.sequence as Record<string, unknown>[] : [command];
- return steps.some(step => step && typeof step === "object" && ("hint" in step || "element_token" in step || step.action === "menu" || isSemanticAction(String(step.action))));
-}
+
/** Steps of one mixed batch; each native step after the first walks the UI again, so batches stay shorter than sequences. */
const BATCH_STEPS = 200;
/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */
@@ -51,11 +54,15 @@ const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "do
/**
* What follows an action: its optional postcondition, and the successor with its change list. The successor is on
* by default; `explicit` means the agent asked for it (`successor: true` or `expect`), which also overrides the
- * defaults that skip it, such as delivered pixel-only input that keeps its observation.
+ * defaults that skip it. `frame` holds the window's pixels before the action, when the input observation had them.
*/
-interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean }
-/** How long a native successor keeps looking for a UI change before reporting that nothing changed. */
+interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean; frame?: WindowFrame }
+/** How long a native successor keeps looking for a change before reporting that nothing changed. */
const SETTLE_MS = 300;
+/** How long a native successor waits for a changing window to show two identical captures in a row. */
+const STEADY_MS = 1_000;
+/** Pause between the captures that check whether a changed window has settled. */
+const STEADY_GAP_MS = 30;
type ObservationTarget = { window_ref: string; display?: never } | { display: string; window_ref?: never };
interface Snapshot { target: ObservationTarget; result: ComputerResult }
/** A running sandbox with its own guest session. */
@@ -445,16 +452,22 @@ export class ComputerResources {
const snapshot = address.kind === "observation" ? this.snapshots.get(address.ref) : undefined;
const windowRef = address.kind === "window" ? address.ref : snapshot?.target.window_ref;
if (view.ocr && windowRef === undefined) throw new ComputerRequestError("ocr reads window observations; display observations have no kept overview pixels.");
- if (view.changes && address.kind !== "window") throw new ComputerRequestError("changes reads a window address: it takes a fresh UI walk and compares it with the window's previous observation.");
+ if (view.changes && address.kind !== "window") throw new ComputerRequestError("changes reads a window address: it takes a fresh capture and compares it with the window's previous observation.");
const baseline = view.changes && windowRef !== undefined ? this.lastUi.get(windowRef) : undefined;
+ // The pixel baseline is the window's latest capture with pixels; the fresh capture keeps its overview size.
+ const frame = view.changes && windowRef !== undefined && view.kind === "overview" && view.capture.granularity !== "ui" ? session.windowFrame(windowRef) : undefined;
+ if (frame && view.kind === "overview" && view.capture.maxLongEdge === undefined && view.capture.scale === undefined) Object.assign(view.capture, frame.size);
const target = windowRef === undefined ? undefined : session.windowTarget(windowRef);
if (view.regions && windowRef === undefined) throw new ComputerRequestError("regions reads a window or one of its observations.");
// A region of a stored observation is read from its own pixels: exact boxes, no native call, input authority kept.
- const result = view.ocr && view.kind === "region" && snapshot && windowRef !== undefined
+ let result = view.ocr && view.kind === "region" && snapshot && windowRef !== undefined
? await this.recognize(session, snapshot.result, windowRef, view.region, signal)
// An observation keeps its own capture: regions parses it without another native read.
: view.regions && snapshot ? snapshot.result
: await this.readNative(address, cwd, views, view, signal);
+ // A window whose pixels cannot be captured still lists its UI changes, unless pixels were asked for explicitly.
+ if (view.changes && view.kind === "overview" && result && result.details.status !== "observed" && !views?.some(item => item === "full" || item === "image"))
+ result = await this.readNative(address, cwd, views, { ...view, capture: { ...view.capture, granularity: "ui" } }, signal);
if (!result) return result;
if (view.regions && result.details.status === "observed") {
try { return { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }
@@ -464,10 +477,17 @@ export class ComputerResources {
try { return { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }
catch (error) { throw new ComputerRequestError(`The aria snapshot could not be rendered: ${error instanceof Error ? error.message : error}`); }
}
- if (view.changes && result.details.status === "observed" && result.details.ui) {
- const changes = baseline ? { since: baseline.observation_id, diff: diffUi(baseline.ui, result.details.ui) }
- : { reason: "No earlier UI observation of this window in this session, so this is the full UI." };
- return { ...result, details: { ...result.details, changes } };
+ if (view.changes && result.details.status === "observed") {
+ const captured = session.windowFrame(windowRef!);
+ const latest = frame && captured?.observation_id === result.details.observation_id ? captured : undefined;
+ const diff = compareCaptures({ ui: baseline?.ui, frame: latest ? await decodeFrame(frame!.png) : undefined },
+ { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined });
+ if (!diff) return { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } };
+ const since = baseline?.observation_id ?? frame!.observation_id;
+ if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;
+ // The change list replaces the full overview: only crops of the changed area are shown.
+ const crops = latest ? await cropImages(frame!, latest, diff) : [];
+ return { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } };
}
if (view.kind === "state" && result.details.status === "observed") {
const health = await session.processHealth(target);
@@ -748,7 +768,7 @@ export class ComputerResources {
let follow: FollowUp | undefined;
if (tool === "insert" && window) {
const command = commandObject(value);
- follow = await this.followUpSpec(window, command, cwd);
+ follow = await this.withFrame(await this.followUpSpec(window, command, cwd), window, address.ref, cwd);
// Was the postcondition already true before anything is sent? Browsers sample now; native uses the input walk.
if (follow.expect) {
const session = await this.sessionFor(cwd, { window_ref: window });
@@ -769,16 +789,9 @@ export class ComputerResources {
this.snapshots.delete(address.ref);
return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), command, cwd, signal, follow);
}
- } else if (tool === "write" && window) follow = { successor: true, explicit: false };
+ } else if (tool === "write" && window) follow = await this.withFrame({ successor: true, explicit: false }, window, address.ref, cwd);
const result = await this.act(address, snapshot, input, tool, cwd, signal);
if (!follow || !window) return result;
- // The default UI change list follows input aimed at a UI element. Input by pixels or keys alone is meant for
- // surfaces the UI tree does not describe (canvases, games); a walk would cost time and, for delivered
- // pixel-only input, replace the observation it keeps for more pixel input.
- if (!follow.explicit && !snapshot.result.details.dom && !elementAddressed(address, tool, input)) {
- if (result.details.status !== "delivered") return result;
- return { ...result, content: [...result.content, { type: "text", text: "Input by pixels or keys alone is not followed by a UI walk. Read the window, or add \"successor\":true, to see what changed." }] };
- }
return this.followUp(window, snapshot, result, follow, cwd, signal);
}
@@ -827,6 +840,13 @@ export class ComputerResources {
return result;
}
+ /** Keep the input observation's own pixels as the baseline of the pixel comparison; they must be taken before the action. */
+ private async withFrame(follow: FollowUp, window: string, observation: string | undefined, cwd: string): Promise<FollowUp> {
+ if (!follow.successor) return follow;
+ const frame = (await this.sessionFor(cwd, { window_ref: window })).windowFrame(window);
+ return frame && frame.observation_id === observation ? { ...follow, frame } : follow;
+ }
+
/** Validate an action's optional `expect` postcondition and `successor` request before anything is dispatched. */
private async followUpSpec(window: string, command: Record<string, unknown>, cwd: string): Promise<FollowUp> {
if (command.successor !== undefined && typeof command.successor !== "boolean") throw new ComputerRequestError("successor must be boolean.");
@@ -852,7 +872,7 @@ export class ComputerResources {
if (!follow.explicit && (status === "refused" || (before.result.details.dom && !result.details.dom))) return result;
const session = await this.sessionFor(cwd, { window_ref: window });
const preexisting = follow.preexisting === true;
- let successor: ComputerResult | undefined, settled: StateDiff | undefined;
+ let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];
let postcondition: ComputerResult["details"]["postcondition"];
if (follow.expect && status !== "refused") {
// Unknown delivery is sampled once for information, but never reported as verified.
@@ -868,19 +888,7 @@ export class ComputerResources {
postcondition = { status: preexisting ? "preexisting" : "failed", reason: "The action was refused; nothing was sent." };
} else if (status !== "refused") {
if (result.details.dom) successor = result;
- else {
- // An app may repaint a little after input is delivered: when the first walk shows no change, look again
- // for a short while before saying that nothing changed.
- const started = performance.now();
- for (let attempt = 0; ; attempt++) {
- this.retireWindow(window);
- successor = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
- const waited = Math.round(performance.now() - started);
- settled = before.result.details.ui && successor.details.ui ? { ...diffUi(before.result.details.ui, successor.details.ui), waited_ms: waited } : undefined;
- if (!settled || !diffIsEmpty(settled) || successor.details.status !== "observed" || signal?.aborted || waited >= SETTLE_MS) break;
- await new Promise(resolve => setTimeout(resolve, Math.min([30, 60, 100][attempt] ?? 150, SETTLE_MS - waited)));
- }
- }
+ else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal));
}
const after = successor?.details;
const diff = before.result.details.dom && after?.dom ? diffDom(before.result.details.dom, after.dom)
@@ -890,7 +898,49 @@ export class ComputerResources {
...(postcondition ? { postcondition } : {}),
successor: after?.observation_id ? { observation_id: after.observation_id, ...(diff ? { diff } : { reason: "The input observation had no comparable UI or DOM." }) }
: { reason: status === "refused" ? "The action was refused; the input observation is unchanged but consumed." : "No successor observation could be taken." } };
- return { ...result, details };
+ return { ...result, content: [...result.content, ...crops], details };
+ }
+
+ /**
+ * Capture the window after native input until its state is known: when nothing changed yet, look again for up to
+ * SETTLE_MS before saying so; once something changed, capture until two captures in a row are identical, for up
+ * to STEADY_MS, so the change list shows the settled window rather than a frame in the middle of a repaint.
+ * Captures keep the input observation's overview size so their pixels compare with it.
+ */
+ private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal):
+ Promise<{ successor: ComputerResult; diff?: StateDiff; crops: ComputerResult["content"] }> {
+ const started = performance.now();
+ let options: UiCaptureOptions = { granularity: "full", ...(frame ? frame.size : {}) };
+ const baseline: Capture = { ui: before.result.details.ui, frame: frame ? await decodeFrame(frame.png) : undefined };
+ let successor: ComputerResult, diff: StateDiff | undefined, previous: Capture | undefined, latest: WindowFrame | undefined;
+ for (let attempt = 0; ; attempt++) {
+ this.retireWindow(window);
+ successor = this.remember({ window_ref: window }, await session.observeQuiet(window, signal, options));
+ const waited = Math.round(performance.now() - started);
+ // A window whose pixels cannot be captured (minimized, or a failed screenshot) still gets its UI compared.
+ if (successor.details.status !== "observed" && options.granularity === "full" && !signal?.aborted) {
+ options = { granularity: "ui" }; baseline.frame = undefined; attempt--; continue;
+ }
+ if (successor.details.status !== "observed" || signal?.aborted) break;
+ const captured = session.windowFrame(window);
+ latest = captured?.observation_id === successor.details.observation_id ? captured : undefined;
+ const current: Capture = { ui: successor.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined };
+ diff = compareCaptures(baseline, current);
+ if (!diff) break;
+ diff.waited_ms = waited;
+ if (diffIsEmpty(diff)) {
+ if (waited >= SETTLE_MS) break;
+ previous = undefined;
+ await new Promise(resolve => setTimeout(resolve, Math.min([30, 60, 100][attempt] ?? 150, SETTLE_MS - waited)));
+ continue;
+ }
+ const steady = previous && compareCaptures(previous, current);
+ if (steady && diffIsEmpty(steady)) break;
+ if (waited >= STEADY_MS) { diff.unsettled = true; break; }
+ previous = current;
+ await new Promise(resolve => setTimeout(resolve, STEADY_GAP_MS));
+ }
+ return { successor, diff, crops: diff && frame && latest ? await cropImages(frame, latest, diff) : [] };
}
/**
@@ -898,14 +948,14 @@ export class ComputerResources {
* matches and is ready (enabled for input; for pointer input also on screen and at the same place in two walks), or
* until the timeout. Several matches refuse at once with each match's locator, like Playwright's strict mode.
*/
- private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; visible?: boolean; describe: string }, signal?: AbortSignal):
+ private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; visible?: boolean; describe: string; pixels?: boolean }, signal?: AbortSignal):
Promise<{ element: UiElement; observation: ComputerResult } | { refused: string; observation?: ComputerResult }> {
const deadline = performance.now() + options.timeout_ms;
let previous: UiElement | undefined, reason = "";
for (let attempt = 0; ; attempt++) {
if (signal?.aborted) return { refused: "Cancelled while waiting for the target; nothing was sent." };
this.retireWindow(window);
- const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));
+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal, options.pixels ? { granularity: "full" } : undefined));
if (fresh.details.status !== "observed" || !fresh.details.ui) return { refused: `No current UI sample of the window: ${computerResultText(fresh).split("\n")[0]}`, observation: fresh };
const ui = fresh.details.ui;
let found: UiElement[];
@@ -945,7 +995,8 @@ export class ComputerResources {
if (!NATIVE_LOCATOR_ACTIONS.includes(action)) throw new ComputerRequestError(`A target takes one of ${[...NATIVE_LOCATOR_ACTIONS, ...Object.keys(LOCATOR_VERBS)].join(", ")}.`);
const describe = typeof target === "string" ? target : JSON.stringify(target);
const pointer = ["click", "double_click", "right_click", "hover", "check", "uncheck"].includes(action);
- const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, visible: pointer, describe }, signal);
+ // With a successor, the resolving walks also capture pixels: the last one is the pixel baseline of the action.
+ const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, visible: pointer, describe, pixels: follow?.successor }, signal);
const mode = this.getMode();
if ("refused" in resolved) return { content: [{ type: "text", text: resolved.refused }], details: { status: "refused", mode, window_ref: window }, isError: true };
const { element, observation } = resolved;
@@ -961,6 +1012,7 @@ export class ComputerResources {
// The walk that resolved the target is the state before the action: the change list compares with it.
const before: Snapshot = { target: { window_ref: window }, result: observation };
if (follow?.expect && observation.details.ui) follow.preexisting = combine(evaluateNative(observation.details.ui, follow.expect.conditions)) === "satisfied";
+ if (follow) follow = await this.withFrame(follow, window, observation.details.observation_id, cwd);
const finish = (result: ComputerResult) => follow ? this.followUp(window, before, result, follow, cwd, signal) : Promise.resolve(result);
this.snapshots.delete(observation.details.observation_id!);
const result = await session.act(nativeCommand as Action, signal);
diff --git a/src/computer-views.ts b/src/computer-views.ts
index eaf2bc9..457583c 100644
--- a/src/computer-views.ts
+++ b/src/computer-views.ts
@@ -24,7 +24,7 @@ function keys(value: Record<string, unknown>, allowed: readonly string[]): void
/** Validate owned views before capturing; text offsets are never image coordinates.
* `ocr` adds Windows text recognition to a window overview, an observation or a region detail.
- * `changes` reads a window's UI and lists only what changed since its previous observation.
+ * `changes` reads a window's UI and pixels and lists only what changed since its previous observation.
* `regions` adds icons and text areas found by Cua perception to a window overview or its current observation.
* `aria` reads a window's UI as a Playwright aria snapshot whose refs are the observation's hints. */
export function parseComputerViews(views: readonly string[] = []): ComputerViews & { ocr?: true; changes?: true; regions?: true; aria?: true } {
@@ -42,9 +42,9 @@ export function parseComputerViews(views: readonly string[] = []): ComputerViews
}
if (views.includes("changes")) {
const view = parseView(views.filter(view => view !== "changes"));
- if (view.kind !== "overview" || (view.capture.granularity ?? "ui") !== "ui")
- throw Error("Use changes alone or with ui capture options such as max_elements; it compares UI, not pixels.");
- return { kind: "overview", capture: { ...view.capture, granularity: "ui" }, changes: true };
+ if (view.kind !== "overview")
+ throw Error("Use changes alone (UI and pixels), or with ui, image or full and their capture options; it compares a fresh capture with the window's previous observation.");
+ return { kind: "overview", capture: { ...view.capture, granularity: view.capture.granularity ?? "full" }, changes: true };
}
if (!views.includes("ocr")) return parseView(views);
const view = parseView(views.filter(view => view !== "ocr"));
diff --git a/src/pixel-diff.ts b/src/pixel-diff.ts
new file mode 100644
index 0000000..d7b4f14
--- /dev/null
+++ b/src/pixel-diff.ts
@@ -0,0 +1,107 @@
+import sharp from "sharp";
+
+/** Decoded RGBA pixels of one window overview capture. */
+export interface PixelFrame { width: number; height: number; data: Buffer }
+export interface PixelBox { x: number; y: number; width: number; height: number }
+/** How a window's overview pixels changed between two captures, in overview pixels. */
+export interface PixelDiff {
+ /** Overview size of both captures; when they differ, `size_changed` holds the earlier size and nothing else is compared. */
+ width: number; height: number;
+ size_changed?: { width: number; height: number };
+ /** Box around every counted change; absent when nothing counted changed. */
+ box?: PixelBox;
+ /** Changed pixels that count, and their share of the whole overview. */
+ changed_pixels: number; share: number;
+ /** Separate changed spots that count. */
+ regions: number;
+ /** Spots too small to count, such as a blinking caret. */
+ ignored: number;
+}
+
+/** A channel must move by more than this (of 255) for its pixel to count as changed, so compression noise does not. */
+const CHANNEL_THRESHOLD = 26;
+/** Changed pixels are grouped into spots of touching cells this large. */
+const CELL = 8;
+/** A spot with fewer changed pixels than this does not count. */
+const MIN_SPOT_PIXELS = 12;
+/** A spot at most this wide and this tall is caret-shaped (a text cursor blinking) and does not count. */
+const CARET_WIDTH = 3, CARET_HEIGHT = 64;
+
+export async function decodeFrame(png: Buffer): Promise<PixelFrame> {
+ const { data, info } = await sharp(png, { limitInputPixels: 32 * 1024 * 1024 }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
+ return { width: info.width, height: info.height, data };
+}
+
+/** Compare two captures of the same window: which pixels changed, where, and how much, ignoring caret-sized spots. */
+export function diffFrames(before: PixelFrame, after: PixelFrame): PixelDiff {
+ const { width, height } = after;
+ if (before.width !== width || before.height !== height)
+ return { width, height, size_changed: { width: before.width, height: before.height }, changed_pixels: 0, share: 0, regions: 0, ignored: 0 };
+ const columns = Math.ceil(width / CELL), rows = Math.ceil(height / CELL);
+ // Per cell: changed pixel count and the exact box of its changed pixels.
+ const count = new Uint32Array(columns * rows);
+ const left = new Int32Array(columns * rows).fill(width), top = new Int32Array(columns * rows).fill(height);
+ const right = new Int32Array(columns * rows).fill(-1), bottom = new Int32Array(columns * rows).fill(-1);
+ const a = before.data, b = after.data;
+ for (let y = 0; y < height; y++) {
+ const row = y * width * 4, cellRow = Math.floor(y / CELL) * columns;
+ for (let x = 0; x < width; x++) {
+ const i = row + x * 4;
+ if (Math.abs(a[i] - b[i]) <= CHANNEL_THRESHOLD && Math.abs(a[i + 1] - b[i + 1]) <= CHANNEL_THRESHOLD && Math.abs(a[i + 2] - b[i + 2]) <= CHANNEL_THRESHOLD) continue;
+ const cell = cellRow + Math.floor(x / CELL);
+ count[cell]++;
+ if (x < left[cell]) left[cell] = x;
+ if (x > right[cell]) right[cell] = x;
+ if (y < top[cell]) top[cell] = y;
+ if (y > bottom[cell]) bottom[cell] = y;
+ }
+ }
+ // Spots are groups of changed cells that touch, including diagonally.
+ const seen = new Uint8Array(columns * rows);
+ let box: PixelBox | undefined, changed = 0, regions = 0, ignored = 0;
+ for (let start = 0; start < count.length; start++) {
+ if (!count[start] || seen[start]) continue;
+ const stack = [start]; seen[start] = 1;
+ let pixels = 0, x0 = width, y0 = height, x1 = -1, y1 = -1;
+ while (stack.length) {
+ const cell = stack.pop()!;
+ pixels += count[cell];
+ x0 = Math.min(x0, left[cell]); y0 = Math.min(y0, top[cell]); x1 = Math.max(x1, right[cell]); y1 = Math.max(y1, bottom[cell]);
+ const cx = cell % columns, cy = Math.floor(cell / columns);
+ for (let dy = -1; dy <= 1; dy++) for (let dx = -1; dx <= 1; dx++) {
+ const nx = cx + dx, ny = cy + dy, next = ny * columns + nx;
+ if (nx < 0 || ny < 0 || nx >= columns || ny >= rows || seen[next] || !count[next]) continue;
+ seen[next] = 1; stack.push(next);
+ }
+ }
+ const spot = { x: x0, y: y0, width: x1 - x0 + 1, height: y1 - y0 + 1 };
+ if (pixels < MIN_SPOT_PIXELS || (spot.width <= CARET_WIDTH && spot.height <= CARET_HEIGHT)) { ignored++; continue; }
+ regions++; changed += pixels;
+ box = box ? union(box, spot) : spot;
+ }
+ return { width, height, ...(box ? { box } : {}), changed_pixels: changed, share: changed / (width * height), regions, ignored };
+}
+
+export const pixelsChanged = (diff: PixelDiff) => diff.size_changed !== undefined || diff.box !== undefined;
+
+function union(a: PixelBox, b: PixelBox): PixelBox {
+ const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y);
+ return { x, y, width: Math.max(a.x + a.width, b.x + b.width) - x, height: Math.max(a.y + a.height, b.y + b.height) - y };
+}
+
+/** Each crop's long edge is at most this many pixels, so two crops cost little next to a full overview. */
+const CROP_EDGE = 512;
+/** Context kept around the changed box on each side, in overview pixels. */
+const CROP_MARGIN = 12;
+
+/** Before and after PNG crops of the changed box with a small margin, scaled down when large. */
+export async function changeCrops(before: Buffer, after: Buffer, diff: PixelDiff): Promise<{ region: PixelBox; before: Buffer; after: Buffer } | undefined> {
+ if (!diff.box || diff.size_changed) return undefined;
+ const x = Math.max(0, diff.box.x - CROP_MARGIN), y = Math.max(0, diff.box.y - CROP_MARGIN);
+ const region = { x, y, width: Math.min(diff.width, diff.box.x + diff.box.width + CROP_MARGIN) - x, height: Math.min(diff.height, diff.box.y + diff.box.height + CROP_MARGIN) - y };
+ const crop = (png: Buffer) => sharp(png, { limitInputPixels: 32 * 1024 * 1024 })
+ .extract({ left: region.x, top: region.y, width: region.width, height: region.height })
+ .resize(CROP_EDGE, CROP_EDGE, { fit: "inside", withoutEnlargement: true }).png().toBuffer();
+ const [a, b] = await Promise.all([crop(before), crop(after)]);
+ return { region, before: a, after: b };
+}
diff --git a/src/provider.ts b/src/provider.ts
index aa7f8af..c60d02c 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -8,7 +8,7 @@ import { readFile } from "node:fs/promises";
import { ComputerResources, requireComputerAddress, computerResultText, ComputerRequestError, type SandboxFactory, type MachineFactory } from "./computer-resources.ts";
import { computerSource } from "./computer-address.ts";
import type { WebLogEntry } from "./web-browser.ts";
-import type { StateDiff } from "./successor.ts";
+import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";
import { computerActionRenderer } from "./computer-action-renderer.ts";
import { computerReadRenderer } from "./computer-read-renderer.ts";
import type { ComputerSession, ComputerResult, ComputerMode, Content } from "./session.ts";
@@ -189,21 +189,37 @@ export function presentComputerResult(result: ComputerResult, full = false): str
const changeCount = (c: { changed: number; added: number; removed: number }) =>
c.changed + c.added + c.removed ? `${c.changed} changed, ${c.added} added, ${c.removed} removed` : "nothing changed";
-/** Lines of a compact UI or DOM change list; changed and added rows carry the new observation's hints. */
+/** Lines of a compact UI, DOM or pixel change list; changed and added rows carry the new observation's hints. */
function diffLines(diff: StateDiff, afterAction = false): string[] {
const page = diff.page ?? {};
const total = diff.counts.changed + diff.counts.added + diff.counts.removed;
const omitted = total - diff.changed.length - diff.added.length - diff.removed.length;
+ const pixels = diff.pixels;
+ const subject = diff.kind === "dom" ? "on the page" : diff.kind === "pixels" ? "in the window's pixels" : pixels ? "in the UI or the window's pixels" : "in the UI";
+ const nothing = !afterAction ? `Nothing changed ${subject} since that observation.`
+ : `Nothing changed ${subject}${diff.waited_ms !== undefined ? ` within ${diff.waited_ms} ms` : ""} after the action. Check that the target was right before acting again; do not replay blindly.`;
+ const uiChanged = total > 0 || Object.keys(page).length > 0;
return [...Object.entries(page).map(([key, change]) => `Changed ${key}: ${JSON.stringify(change![0])} → ${JSON.stringify(change![1])}`),
- total === 0 && !Object.keys(page).length
- ? (!afterAction ? `Nothing changed in the ${diff.kind === "ui" ? "UI" : "page"} since that observation.`
- : diff.kind === "ui" ? `Nothing changed in the UI${diff.waited_ms !== undefined ? ` within ${diff.waited_ms} ms` : ""} after the action. Check that the target was right before acting again; do not replay blindly.` : "Nothing changed on the page after the action. Check that the target was right before acting again; do not replay blindly.")
- : `Changes: ${diff.counts.changed} changed, ${diff.counts.added} added, ${diff.counts.removed} removed`,
+ ...(diffIsEmpty(diff) ? [nothing] : diff.kind === "pixels" ? [] : [`Changes: ${diff.counts.changed} changed, ${diff.counts.added} added, ${diff.counts.removed} removed`]),
...diff.changed.map(line => `~ ${label(line)}`), ...diff.added.map(line => `+ ${label(line)}`), ...diff.removed.map(line => `- ${label(line)}`),
...(omitted > 0 ? [`${omitted} more changes are not listed; Search or Read the new observation for them.`] : []),
+ ...(pixels && !diffIsEmpty(diff) ? pixelLines(pixels, uiChanged) : []),
+ ...(afterAction && !pixels && diff.kind === "ui" ? ["Pixels were not compared: the input observation had no pixels."] : []),
+ ...(diff.unsettled ? [`The window was still changing ${diff.waited_ms} ms after the action; the successor shows it at that moment.`] : []),
...(diff.note ? [diff.note] : [])];
}
+/** How the window's pixels changed, in overview pixels, and what the attached crops show. */
+function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {
+ const since = pixels.since ? ` since ${computerSource("observation", pixels.since)}` : "";
+ if (pixels.size_changed) return [`The window's overview size changed${since} from ${pixels.size_changed.width}x${pixels.size_changed.height} to ${pixels.width}x${pixels.height}; pixels were not compared.`];
+ if (!pixels.box) return uiChanged ? [`No visible pixel change${since}.`] : [];
+ const box = pixels.box, share = pixels.share * 100;
+ const crop = pixels.crop;
+ return [`Pixels changed${since} in x=${box.x} y=${box.y} w=${box.width} h=${box.height} (overview pixels; ${share < 0.1 ? share.toFixed(2) : share.toFixed(1)}% of the window, ${pixels.regions} ${pixels.regions === 1 ? "spot" : "spots"}).`,
+ ...(crop ? [`The two images crop x=${crop.x} y=${crop.y} w=${crop.width} h=${crop.height}: before, then after.`] : [])];
+}
+
/** Whole lines from the start that fit a byte budget, never splitting Unicode. */
function fittingLines(lines: readonly string[], budget: number): string[] {
const kept: string[] = [];
@@ -344,7 +360,9 @@ export async function performComputer(resources: ComputerResources, tool: "write
// Completion of the native request is separate from application-effect certainty.
const ok = !result.isError && ["delivered", "observed", "closed"].includes(result.details.status);
const summary = presentComputerResult(result);
- return { source, summary, data: { kind: "computer-action", ok,
+ // Images of an action result, such as before/after crops of its pixel change, follow the summary for the model.
+ const images = result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
+ return { source, summary, ...(images.length ? { images } : {}), data: { kind: "computer-action", ok,
effect, ...result.details,
receipt: computerResultText(result),
...(!ok ? { error: { code: typeof result.details.state?.code === "string" ? result.details.state.code : `COMPUTER_${effect.toUpperCase().replace("-", "_")}`,
diff --git a/src/session.ts b/src/session.ts
index c9148c9..3c49df6 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -141,7 +141,9 @@ interface Observation { texts?: Map<string, string>; id: string; tokens: Set<str
/** Kept after pixel-only input: coordinates stay valid until a freshness check finds a changed size or state; UI tokens and the native capture are spent. */
pixelsOnly?: boolean }
interface WindowState { id: string; nativeId: string; kind: "restore" | "management" }
-interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark }
+/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */
+export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }
+interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }
interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }
/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */
export function textMatcher(query?: string): (text: string) => boolean {
@@ -1029,6 +1031,7 @@ export class ComputerSession {
const raw = images[0];
if (raw?.type === "image" && Buffer.byteLength(raw.data, "base64") <= 12 * 1024 * 1024) {
observed.rawImage = raw;
+ entry.frame = { observation_id: id, png: Buffer.from(raw.data, "base64"), size: { maxLongEdge: settings.maxLongEdge, scale: settings.scale } };
const active = new Set([...this.entries.values()].map(e => e.observation));
this.uiImages = [...this.uiImages.filter(old => active.has(old)), observed];
for (const old of this.uiImages.splice(0, Math.max(0, this.uiImages.length - 3))) old.rawImage = undefined;
@@ -1244,15 +1247,21 @@ export class ComputerSession {
});
}
- /** A fresh UI-only observation without the human frame, for successors and batch steps. */
- observeQuiet(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
+ /** The window's latest unannotated capture with pixels, the baseline for comparing its pixels; undefined when none is kept. */
+ windowFrame(ref: string): WindowFrame | undefined { return this.entries.get(ref)?.frame; }
+
+ /**
+ * A fresh observation without the human frame, for successors and batch steps: UI-only by default, or with
+ * pixels at a given overview size so they compare with an earlier capture.
+ */
+ observeQuiet(ref: string, signal?: AbortSignal, options: UiCaptureOptions = { granularity: "ui" }): Promise<ComputerResult> {
return this.queue(async () => {
if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
try {
const entry = await this.current(ref, signal);
if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
if (!this.allowed(entry)) return this.denied(entry.ref);
- return await this.capture(entry, signal, undefined, { granularity: "ui" }, true);
+ return await this.capture(entry, signal, undefined, options, true);
} catch (error) { return this.failed(error, false); }
});
}
diff --git a/src/standalone-tools.ts b/src/standalone-tools.ts
index 4cf491d..6cd14b1 100644
--- a/src/standalone-tools.ts
+++ b/src/standalone-tools.ts
@@ -97,7 +97,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerRes
const action = await performComputer(resources, tool, String(input.path), input, ctx.cwd, signal);
// The receipt is plain data with a few untyped native fields; a JSON copy gives scripts exactly that.
const receipt: JsonValue = JSON.parse(JSON.stringify(action.script));
- return { content: [{ type: "text", text: action.summary }], details: { results: [], metadata: { semanticAction: action.data } },
+ return { content: [{ type: "text", text: action.summary }, ...("images" in action ? action.images ?? [] : [])], details: { results: [], metadata: { semanticAction: action.data } },
structuredContent: receipt, ...(action.data.ok ? {} : { isError: true }) };
},
});
diff --git a/src/successor.ts b/src/successor.ts
index 252a9f4..a2d26f5 100644
--- a/src/successor.ts
+++ b/src/successor.ts
@@ -1,9 +1,14 @@
import type { UiElement, UiResource } from "./ui-resources.ts";
import type { WebDom } from "./session.ts";
+import { diffFrames, pixelsChanged, type PixelBox, type PixelDiff, type PixelFrame } from "./pixel-diff.ts";
+
+/** How a window's overview pixels changed; `crop` is the region the before/after crops show. */
+export type PixelReport = PixelDiff & { crop?: PixelBox; since?: string };
/** Compact change list between an action's input observation and its successor. Bounded; full successors stay readable. */
export interface StateDiff {
- kind: "dom" | "ui";
+ /** What was compared: page lines, UI elements, or only pixels when the earlier observation had no UI. */
+ kind: "dom" | "ui" | "pixels";
/** Lines or elements that changed, appeared or disappeared, at most DIFF_LIMIT each. */
changed: string[]; added: string[]; removed: string[];
/** Totals before bounding. */
@@ -11,8 +16,12 @@ export interface StateDiff {
page?: { url?: [string, string]; title?: [string, string]; tab?: [string | null, string | null] };
/** Why the diff may be incomplete, such as a partial native walk. */
note?: string;
- /** How long after the action the native walks kept looking for a change before reporting none. */
+ /** How long after the action the native captures kept looking for a change, or waited for the window to settle. */
waited_ms?: number;
+ /** The window's overview pixels compared with the earlier capture; absent when either side had no pixels. */
+ pixels?: PixelReport;
+ /** Set when the window was still changing when the wait ended; the successor shows it at that moment. */
+ unsettled?: true;
}
const DIFF_LIMIT = 30;
const bounded = (items: string[]) => items.slice(0, DIFF_LIMIT);
@@ -124,4 +133,14 @@ export function diffUi(before: UiResource, after: UiResource): StateDiff {
/** Whether a diff found any difference at all. */
export const diffIsEmpty = (diff: StateDiff): boolean =>
- diff.counts.changed + diff.counts.added + diff.counts.removed === 0 && !Object.keys(diff.page ?? {}).length;
+ diff.counts.changed + diff.counts.added + diff.counts.removed === 0 && !Object.keys(diff.page ?? {}).length && !(diff.pixels && pixelsChanged(diff.pixels));
+
+/** One native capture of a window: its UI walk and its decoded overview pixels, either of which may be missing. */
+export interface Capture { ui?: UiResource; frame?: PixelFrame }
+/** Compare two native captures of a window: UI elements when both have UI, pixels when both have pixels. */
+export function compareCaptures(before: Capture, after: Capture): StateDiff | undefined {
+ const pixels = before.frame && after.frame ? diffFrames(before.frame, after.frame) : undefined;
+ const ui = before.ui && after.ui ? diffUi(before.ui, after.ui) : undefined;
+ if (ui) return pixels ? { ...ui, pixels } : ui;
+ return pixels ? { kind: "pixels", changed: [], added: [], removed: [], counts: { changed: 0, added: 0, removed: 0 }, pixels } : undefined;
+}
\ No newline at end of file
diff --git a/test/action-changes.test.ts b/test/action-changes.test.ts
index 6023618..b60fd2d 100644
--- a/test/action-changes.test.ts
+++ b/test/action-changes.test.ts
@@ -1,28 +1,46 @@
+import sharp from "sharp";
import { expect, test, vi } from "vitest";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
import { presentComputerResult } from "../src/provider.ts";
-import { ComputerSession, type Driver } from "../src/session.ts";
+import { ComputerSession, type ComputerResult, type Content, type Driver } from "../src/session.ts";
const cwd = "/synthetic-action-changes";
const address = requireComputerAddress;
// LPT-914: every action reports what changed in the window, like Playwright MCP's incremental snapshots.
+// LPT-915: and how the window's pixels changed, with before/after crops of the changed area.
async function fixture() {
let count = 0, serial = 0;
+ const dots: { x: number; y: number }[] = [];
+ // The window paints one bar per count under the button and a dot wherever a pixel click lands on the canvas.
+ const frame = async () => {
+ const shapes = [...Array.from({ length: count }, (_, i) => `<rect x="${20 + i * 12}" y="40" width="10" height="10" fill="#000"/>`),
+ ...dots.map(dot => `<rect x="${dot.x - 3}" y="${dot.y - 3}" width="6" height="6" fill="#c00"/>`)];
+ const png = await sharp({ create: { width: 500, height: 320, channels: 3, background: "#fff" } })
+ .composite([{ input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="500" height="320">${shapes.join("")}</svg>`) }]).png().toBuffer();
+ return { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
+ };
const element = (index: number, role: string, label: string) => ({ element_token: `s${serial}:${index}`, role, label, enabled: true, actions: role === "Button" ? ["invoke"] : [],
frame: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });
const driver: Driver = {
identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),
call: vi.fn<Driver["call"]>(async (name, args) => {
+ const input = args as { element_token?: string; x?: number; y?: number; include_screenshot?: boolean };
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };
if (name === "get_window_state") {
serial++;
- return { content: [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },
+ const pixels = input.include_screenshot !== false;
+ return { content: pixels ? [await frame()] : [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },
+ ...(pixels ? { capture_id: `c${serial}`, screenshot_width: 500, screenshot_height: 320 } : {}),
elements_complete: true, elements: [element(0, "Button", `Count ${count}`), element(1, "Text", "Status")] } };
}
- // Clicking the button counts; clicking the status label does nothing.
- if (name === "click") { if (String((args as { element_token?: string }).element_token).endsWith(":0")) count++; return { content: [], structuredContent: { effect: "confirmed" } }; }
+ // Clicking the button counts; clicking the status label does nothing; a pixel click paints a dot there.
+ if (name === "click") {
+ if (input.element_token?.endsWith(":0")) count++;
+ if (input.x !== undefined && input.y !== undefined && input.y > 100) dots.push({ x: input.x, y: input.y });
+ return { content: [], structuredContent: { effect: "confirmed" } };
+ }
throw Error(`Unexpected native call ${name}`);
}),
};
@@ -30,13 +48,56 @@ async function fixture() {
const resources = new ComputerResources(async () => session, async () => session.close());
const ref = ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
const window = computerSource("window", ref);
- const read = async () => {
- const view = (await resources.read(address(window), cwd, ["ui"]))!;
- return { view, at: (index: number) => address(`${computerSource("observation", view.details.observation_id!)}?hint=${encodeURIComponent(view.details.ui!.elements[index].hint)}`) };
+ const read = async (views = ["ui"]) => {
+ const view = (await resources.read(address(window), cwd, views))!;
+ const observation = computerSource("observation", view.details.observation_id!);
+ return { view, observation: address(observation), at: (index: number) => address(`${observation}?hint=${encodeURIComponent(view.details.ui!.elements[index].hint)}`) };
};
const walks = () => vi.mocked(driver.call).mock.calls.filter(([name]) => name === "get_window_state").length;
return { resources, window, read, walks };
}
+const images = (result: ComputerResult) => result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
+
+test("an action on an observation with pixels reports the changed area with before and after crops", async () => {
+ const f = await fixture(), { at } = await f.read(["full"]);
+ const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");
+ expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, crop: { x: 8, y: 28, width: 34, height: 34 } });
+ expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in x=20 y=40 w=10 h=10 \(overview pixels; 0\.06% of the window, 1 spot\)\.\nThe two images crop x=8 y=28 w=34 h=34: before, then after\./);
+ const [before, after] = images(result);
+ expect((await sharp(Buffer.from(before.data, "base64")).stats()).channels[0].min).toBe(255);
+ expect((await sharp(Buffer.from(after.data, "base64")).stats()).channels[0].min).toBe(0);
+});
+
+test("pixel input on a canvas reports only its pixel change, and empty space reports nothing", async () => {
+ const f = await fixture(), { observation } = await f.read(["full"]);
+ const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }), "insert");
+ expect(stroke.details.successor?.diff).toMatchObject({ counts: { changed: 0, added: 0, removed: 0 }, pixels: { box: { x: 297, y: 197, width: 6, height: 6 } } });
+ expect(images(stroke)).toHaveLength(2);
+ const next = address(computerSource("observation", stroke.details.successor!.observation_id!));
+ const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }), "insert");
+ expect(presentComputerResult(empty)).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);
+ expect(images(empty)).toHaveLength(0);
+});
+
+test("an action on a UI-only observation says that pixels were not compared, and its successor has them", async () => {
+ const f = await fixture(), { at } = await f.read();
+ const first = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");
+ expect(presentComputerResult(first)).toMatch(/Pixels were not compared: the input observation had no pixels\./);
+ const view = (await f.resources.read(address(computerSource("observation", first.details.successor!.observation_id!)), cwd, []))!;
+ const hint = view.details.ui!.elements[0].hint;
+ const second = await f.resources.perform(address(`${computerSource("observation", first.details.successor!.observation_id!)}?hint=${hint}`), cwd, JSON.stringify({ action: "click" }), "insert");
+ expect(second.details.successor?.diff?.pixels?.box).toEqual({ x: 32, y: 40, width: 10, height: 10 });
+});
+
+test("a changes Read compares the window's pixels with its previous observation", async () => {
+ const f = await fixture(), { observation } = await f.read(["full"]);
+ await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200, successor: false }), "insert");
+ const changes = (await f.resources.read(address(f.window), cwd, ["changes"]))!;
+ expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 297, y: 197, width: 6, height: 6 });
+ expect(presentComputerResult(changes)).toMatch(/Pixels changed in x=297 y=197 w=6 h=6/);
+ // Only the two crops, not the full overview.
+ expect(images(changes)).toHaveLength(2);
+});
test("an action reports what it changed by default, and its successor is the current observation", async () => {
const f = await fixture(), { at } = await f.read();
diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.ts
index 877c9c6..29ea722 100644
--- a/test/computer-resources.test.ts
+++ b/test/computer-resources.test.ts
@@ -136,8 +136,10 @@ test("a new overview retires the prior resource instead of rebinding its hint",
expect(f.inputs()).toHaveLength(0);
expect((await f.resources.perform(address(`${latest.source}?hint=${encodeURIComponent(latest.hint!)}`), cwd, "current", "write")).details.status).toBe("delivered");
expect(f.inputs()[0][1]).toMatchObject({ element_token: "snapshot-2:0", text: "current" });
- // Two overviews, then only the successor's UI walks after the input.
- expect(f.captures().slice(2).every(([, args]) => (args as { include_screenshot?: boolean }).include_screenshot === false)).toBe(true);
+ // Two overviews, then the input, then the successor's captures.
+ const order = f.native().map(([name]) => name).filter(name => name !== "list_windows" && name !== "get_window_control_state");
+ expect(order.slice(0, 3)).toEqual(["get_window_state", "get_window_state", f.inputs()[0][0]]);
+ expect(order.slice(3).every(name => name === "get_window_state")).toBe(true);
});
test.each(["safe", "closed", "reused"] as const)("%s targets refuse cached reads and input without another capture", async change => {
@@ -246,11 +248,12 @@ test("a display overview accepts a size for that Read only", async () => {
await expect(f.resources.read(display, cwd, ['capture:{"max_elements":5}'])).rejects.toThrow(/accepts only capture/);
});
-test("pixel-only input keeps the observation until the window size changes; UI hints and other input spend it", async () => {
+test("pixel-only input without a successor keeps the observation until the window size changes; UI hints and other input spend it", async () => {
const f = await fixture();
(f.driver as Driver & { windowManagement: boolean }).windowManagement = true;
const view = await f.read();
- const click = (x: number) => f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x, y: 40 }), "insert");
+ // A default successor would replace the observation; without one, pixel-only input keeps it for more pixel input.
+ const click = (x: number) => f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x, y: 40, successor: false }), "insert");
for (const x of [10, 20, 30]) expect((await click(x)).details.status).toBe("delivered");
expect(f.captures()).toHaveLength(1);
// The native capture is single-use; kept observations send window-local pixels without it.
@@ -261,9 +264,9 @@ test("pixel-only input keeps the observation until the window size changes; UI h
expect(hinted.content[0]).toMatchObject({ text: expect.stringContaining("UI hints of this observation expired") });
const again = await f.read();
f.setBounds({ x: 300, y: 90, width: 500, height: 320 });
- expect((await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert")).details.status).toBe("delivered");
+ expect((await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert")).details.status).toBe("delivered");
f.setBounds({ x: 300, y: 90, width: 640, height: 320 });
- const resized = await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert");
+ const resized = await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert");
expect(resized.details.status).toBe("refused");
expect(resized.content[0]).toMatchObject({ text: expect.stringContaining("resized from 500x320 to 640x320") });
await expect(f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert")).rejects.toThrow(/expired/i);
diff --git a/test/pixel-diff.test.ts b/test/pixel-diff.test.ts
new file mode 100644
index 0000000..da0dfbf
--- /dev/null
+++ b/test/pixel-diff.test.ts
@@ -0,0 +1,78 @@
+import sharp from "sharp";
+import { describe, expect, test } from "vitest";
+import { changeCrops, decodeFrame, diffFrames, pixelsChanged, type PixelFrame } from "../src/pixel-diff.ts";
+
+const blank = (width = 200, height = 100): PixelFrame => ({ width, height, data: Buffer.alloc(width * height * 4, 255) });
+function paint(frame: PixelFrame, x: number, y: number, width: number, height: number, value = 0): PixelFrame {
+ const data = Buffer.from(frame.data);
+ for (let row = y; row < y + height; row++) for (let column = x; column < x + width; column++) {
+ const i = (row * frame.width + column) * 4; data[i] = data[i + 1] = data[i + 2] = value;
+ }
+ return { ...frame, data };
+}
+const png = (frame: PixelFrame) => sharp(frame.data, { raw: { width: frame.width, height: frame.height, channels: 4 } }).png().toBuffer();
+
+describe("diffFrames", () => {
+ test("identical captures report no change", () => {
+ const diff = diffFrames(blank(), blank());
+ expect(pixelsChanged(diff)).toBe(false);
+ expect(diff).toMatchObject({ changed_pixels: 0, share: 0, regions: 0 });
+ });
+
+ test("a stroke reports its box, its pixels and their share", () => {
+ const diff = diffFrames(blank(), paint(blank(), 20, 40, 100, 2));
+ expect(diff.box).toEqual({ x: 20, y: 40, width: 100, height: 2 });
+ expect(diff.changed_pixels).toBe(200);
+ expect(diff.share).toBeCloseTo(200 / 20_000);
+ expect(diff.regions).toBe(1);
+ });
+
+ test("separate spots count as regions inside one box", () => {
+ const after = paint(paint(blank(), 10, 10, 10, 10), 150, 70, 20, 20);
+ const diff = diffFrames(blank(), after);
+ expect(diff.regions).toBe(2);
+ expect(diff.box).toEqual({ x: 10, y: 10, width: 160, height: 80 });
+ });
+
+ test("a blinking caret and a few stray pixels do not count", () => {
+ const after = paint(paint(blank(), 50, 20, 2, 18), 120, 60, 3, 3);
+ const diff = diffFrames(blank(), after);
+ expect(pixelsChanged(diff)).toBe(false);
+ expect(diff.ignored).toBe(2);
+ });
+
+ test("a caret next to typed text is part of that change", () => {
+ const after = paint(paint(blank(), 20, 20, 40, 12), 61, 18, 2, 16);
+ const diff = diffFrames(blank(), after);
+ expect(diff.box).toEqual({ x: 20, y: 18, width: 43, height: 16 });
+ });
+
+ test("small colour noise below the threshold does not count", () => {
+ const diff = diffFrames(blank(), paint(blank(), 0, 0, 200, 100, 240));
+ expect(pixelsChanged(diff)).toBe(false);
+ });
+
+ test("a different capture size is reported instead of compared", () => {
+ const diff = diffFrames(blank(200, 100), blank(300, 100));
+ expect(diff.size_changed).toEqual({ width: 200, height: 100 });
+ expect(pixelsChanged(diff)).toBe(true);
+ });
+});
+
+describe("changeCrops", () => {
+ test("crops the changed box with a margin from both captures", async () => {
+ const before = blank(), after = paint(blank(), 50, 30, 20, 10);
+ const crops = (await changeCrops(await png(before), await png(after), diffFrames(before, after)))!;
+ expect(crops.region).toEqual({ x: 38, y: 18, width: 44, height: 34 });
+ const a = await decodeFrame(crops.before), b = await decodeFrame(crops.after);
+ expect([a.width, a.height, b.width, b.height]).toEqual([44, 34, 44, 34]);
+ expect(diffFrames(a, b).box).toEqual({ x: 12, y: 12, width: 20, height: 10 });
+ });
+
+ test("large changed areas are scaled down to a bounded crop", async () => {
+ const before = blank(1600, 900), after = paint(blank(1600, 900), 0, 0, 1600, 900, 0);
+ const crops = (await changeCrops(await png(before), await png(after), diffFrames(before, after)))!;
+ const meta = await sharp(crops.after).metadata();
+ expect(Math.max(meta.width!, meta.height!)).toBe(512);
+ });
+});
diff --git a/test/remote-machine.test.ts b/test/remote-machine.test.ts
index 31832f8..1f4ed04 100644
--- a/test/remote-machine.test.ts
+++ b/test/remote-machine.test.ts
@@ -39,12 +39,14 @@ test("machines come only from validated configuration", async () => {
function backend(pid: number) {
const driver: Driver = {
identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}), selectedDisplays: true,
- call: vi.fn<Driver["call"]>(async (name) => {
+ call: vi.fn<Driver["call"]>(async (name, args) => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };
if (name === "list_displays") return { content: [], structuredContent: { platform: "windows", topology_id: `t${pid}`, displays: [{ display_id: `D${pid}`, primary: true,
monitor_handle: `0x${pid}`, monitor_device_paths: [`\\\\?\\DISPLAY#${pid}`], bounds: { x: 0, y: 0, width: 1920, height: 1080 },
work_area: { x: 0, y: 0, width: 1920, height: 1040 }, scale_factor: 1, dpi: 96 }] } };
+ // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.
if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],
+ ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),
window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };
if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };
if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: `App ${pid}`, launch_path: `C:\\app${pid}.exe` }] } };
@@ -84,7 +86,8 @@ test("a machine starts on first use; its windows, apps and displays come only fr
const apps = await f.resources.search(address(`${f.machineSource}/apps`), cwd, "App");
const app = (apps.details.state!.apps as { app_ref: string }[])[0].app_ref;
await f.resources.perform(address(computerSource("app", app)), cwd, JSON.stringify({ action: "open" }), "insert");
- expect(f.names(f.remoteDriver)).toEqual(["list_windows", "get_window_state", "type_text", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);
+ // The Write's successor walks the window again; repeated walks are collapsed here.
+ expect(f.names(f.remoteDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);
expect(f.names(f.hostDriver)).toEqual([]);
// Every use after the first reuses the started machine.
expect(f.start).toHaveBeenCalledOnce();
diff --git a/test/sandbox.test.ts b/test/sandbox.test.ts
index f04a86f..3f8a8d5 100644
--- a/test/sandbox.test.ts
+++ b/test/sandbox.test.ts
@@ -99,9 +99,11 @@ function backend(pid: number, domain?: "windows-sandbox") {
const driver: Driver = {
...(domain ? { sourceDomain: domain } : {}),
identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}),
- call: vi.fn<Driver["call"]>(async (name) => {
+ call: vi.fn<Driver["call"]>(async (name, args) => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };
+ // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.
if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],
+ ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),
window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };
if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };
throw new Error(`Unexpected native call ${name}`);
@@ -130,7 +132,8 @@ test("guest windows are found, read and operated only through the sandbox's own
expect(view?.details.status).toBe("observed");
await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "hello", "write");
const names = (driver: Driver) => vi.mocked(driver.call).mock.calls.map(([name]) => name);
- expect(names(f.guestDriver)).toEqual(["list_windows", "get_window_state", "type_text"]);
+ // The Write's successor walks the window again; repeated walks are collapsed here.
+ expect(names(f.guestDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state"]);
expect(names(f.hostDriver)).toEqual([]);
expect(f.resources.descriptor(address(f.sandboxSource))).toContain(f.sandboxSource);
});
diff --git a/test/ui-changes.test.ts b/test/ui-changes.test.ts
index 3d7f8b9..c847be1 100644
--- a/test/ui-changes.test.ts
+++ b/test/ui-changes.test.ts
@@ -12,11 +12,14 @@ test("a changes Read compares the window's UI with its previous observation", as
frame: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });
const driver: Driver = {
identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),
- call: vi.fn<Driver["call"]>(async name => {
+ call: vi.fn<Driver["call"]>(async (name, args) => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };
if (name === "get_window_state") {
serial++;
+ // This window cannot be captured, like a minimized one: a capture with pixels reports why.
+ const pixels = (args as { include_screenshot?: boolean }).include_screenshot !== false;
return { content: [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },
+ ...(pixels ? { screenshot_error: "The window has no visible surface." } : {}),
elements_complete: true, elements: [element(0, "Button", "Save"), element(1, "Edit", "Text editor", text), element(2, "Button", "Close")] } };
}
throw Error(`Unexpected native call ${name}`);
@@ -28,7 +31,8 @@ test("a changes Read compares the window's UI with its previous observation", as
const window = requireComputerAddress(computerSource("window", ref));
const first = await resources.read(window, cwd, ["changes"]);
- expect(first!.details.changes).toEqual({ reason: expect.stringMatching(/full UI/) });
+ expect(first!.details.changes).toEqual({ reason: expect.stringMatching(/full observation/) });
+ // This window gives no screenshot, so changes falls back to its UI alone.
expect(first!.details.captured).toEqual({ image: false, ui: true });
const same = await resources.read(window, cwd, ["changes"]);
@@ -43,8 +47,9 @@ test("a changes Read compares the window's UI with its previous observation", as
expect(diff.changed[0]).toBe(`${editor.hint} Edit "Text editor": value="" → "hello"`);
expect(resources.grantsAuthority(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)))).toBe(true);
- for (const views of [["changes", "image"], ["changes", "full"], ["changes", "ocr"]])
- await expect(resources.read(window, cwd, views)).rejects.toThrow();
+ await expect(resources.read(window, cwd, ["changes", "ocr"])).rejects.toThrow();
+ // Pixels asked for explicitly are not replaced by a UI-only walk.
+ expect((await resources.read(window, cwd, ["changes", "full"]))!.details.status).toBe("refused");
await expect(resources.read(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)), cwd, ["changes"]))
.rejects.toThrow(/window address/);
});