Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
WIP: actions report the changed area of the window's pixels (LPT-915)
alexshpunt-clanker[bot]04e3a541367c
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 0078047..e29f9e3 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -11,8 +11,10 @@ import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./seimport { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";import { combine, evaluateNative, type WaitSpec } from "./ui-conditions.ts";-import { diffDom, diffIsEmpty, diffUi, type StateDiff } from "./successor.ts";-import type { UiElement, UiResource } from "./ui-resources.ts";+import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";+import { changeCrops, decodeFrame } from "./pixel-diff.ts";+import type { WindowFrame } from "./session.ts";+import type { UiCaptureOptions, UiElement, UiResource } from "./ui-resources.ts";import { displayMode } from "./display-mode.ts";import { isSemanticAction } from "./semantic-actions.ts";import { accessibleName, ariaRole, checkedState, isVisible, nativeAriaSnapshot, nativeSelector, queryNative, suggestLocator, type ParsedSelector } from "./native-locator.ts";@@ -27,19 +29,20 @@ export function machineGuide(source: string): string[] {}/** A rejected request whose validation completed before native dispatch. */export class ComputerRequestError extends Error {}+/** Before and after PNG crops of a diff's changed pixels, in that order; records the cropped region on the diff. */+async function cropImages(before: WindowFrame, after: WindowFrame, diff: StateDiff): Promise<ComputerResult["content"]> {+ if (!diff.pixels?.box) return [];+ const cropped = await changeCrops(before.png, after.png, diff.pixels).catch(() => undefined);+ if (!cropped) return [];+ diff.pixels.crop = cropped.region;+ return [{ type: "image", data: cropped.before.toString("base64"), mimeType: "image/png" },+ { type: "image", data: cropped.after.toString("base64"), mimeType: "image/png" }];+}function commandObject(value: string): Record<string, unknown> {try { return object(JSON.parse(value)); }catch { throw new ComputerRequestError("Use one valid JSON command object."); }}-/** Whether an action is aimed at a UI element (a hint, a token, a semantic action or a menu), not only at pixels or keys. */-function elementAddressed(address: Extract<ComputerAddress, { kind: "observation" }>, tool: "write" | "insert", input: string): boolean {- if (address.hint !== undefined) return true;- if (tool === "write") return false;- let command: Record<string, unknown>;- try { command = commandObject(input); } catch { return false; }- const steps = Array.isArray(command.sequence) ? command.sequence as Record<string, unknown>[] : [command];- return steps.some(step => step && typeof step === "object" && ("hint" in step || "element_token" in step || step.action === "menu" || isSemanticAction(String(step.action))));-}+/** Steps of one mixed batch; each native step after the first walks the UI again, so batches stay shorter than sequences. */const BATCH_STEPS = 200;/** Native actions a batch may run; each later step re-resolves its target in a fresh UI walk. */@@ -51,11 +54,15 @@ const LOCATOR_VERBS: Record<string, string> = { fill: "set_value", dblclick: "do/*** What follows an action: its optional postcondition, and the successor with its change list. The successor is on* by default; `explicit` means the agent asked for it (`successor: true` or `expect`), which also overrides the- * defaults that skip it, such as delivered pixel-only input that keeps its observation.+ * defaults that skip it. `frame` holds the window's pixels before the action, when the input observation had them.*/-interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean }-/** How long a native successor keeps looking for a UI change before reporting that nothing changed. */+interface FollowUp { expect?: WaitSpec; successor: boolean; explicit: boolean; preexisting?: boolean; frame?: WindowFrame }+/** How long a native successor keeps looking for a change before reporting that nothing changed. */const SETTLE_MS = 300;+/** How long a native successor waits for a changing window to show two identical captures in a row. */+const STEADY_MS = 1_000;+/** Pause between the captures that check whether a changed window has settled. */+const STEADY_GAP_MS = 30;type ObservationTarget = { window_ref: string; display?: never } | { display: string; window_ref?: never };interface Snapshot { target: ObservationTarget; result: ComputerResult }/** A running sandbox with its own guest session. */@@ -445,16 +452,22 @@ export class ComputerResources {const snapshot = address.kind === "observation" ? this.snapshots.get(address.ref) : undefined;const windowRef = address.kind === "window" ? address.ref : snapshot?.target.window_ref;if (view.ocr && windowRef === undefined) throw new ComputerRequestError("ocr reads window observations; display observations have no kept overview pixels.");- if (view.changes && address.kind !== "window") throw new ComputerRequestError("changes reads a window address: it takes a fresh UI walk and compares it with the window's previous observation.");+ if (view.changes && address.kind !== "window") throw new ComputerRequestError("changes reads a window address: it takes a fresh capture and compares it with the window's previous observation.");const baseline = view.changes && windowRef !== undefined ? this.lastUi.get(windowRef) : undefined;+ // The pixel baseline is the window's latest capture with pixels; the fresh capture keeps its overview size.+ const frame = view.changes && windowRef !== undefined && view.kind === "overview" && view.capture.granularity !== "ui" ? session.windowFrame(windowRef) : undefined;+ if (frame && view.kind === "overview" && view.capture.maxLongEdge === undefined && view.capture.scale === undefined) Object.assign(view.capture, frame.size);const target = windowRef === undefined ? undefined : session.windowTarget(windowRef);if (view.regions && windowRef === undefined) throw new ComputerRequestError("regions reads a window or one of its observations.");// A region of a stored observation is read from its own pixels: exact boxes, no native call, input authority kept.- const result = view.ocr && view.kind === "region" && snapshot && windowRef !== undefined+ let result = view.ocr && view.kind === "region" && snapshot && windowRef !== undefined? await this.recognize(session, snapshot.result, windowRef, view.region, signal)// An observation keeps its own capture: regions parses it without another native read.: view.regions && snapshot ? snapshot.result: await this.readNative(address, cwd, views, view, signal);+ // A window whose pixels cannot be captured still lists its UI changes, unless pixels were asked for explicitly.+ if (view.changes && view.kind === "overview" && result && result.details.status !== "observed" && !views?.some(item => item === "full" || item === "image"))+ result = await this.readNative(address, cwd, views, { ...view, capture: { ...view.capture, granularity: "ui" } }, signal);if (!result) return result;if (view.regions && result.details.status === "observed") {try { return { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }@@ -464,10 +477,17 @@ export class ComputerResources {try { return { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }catch (error) { throw new ComputerRequestError(`The aria snapshot could not be rendered: ${error instanceof Error ? error.message : error}`); }}- if (view.changes && result.details.status === "observed" && result.details.ui) {- const changes = baseline ? { since: baseline.observation_id, diff: diffUi(baseline.ui, result.details.ui) }- : { reason: "No earlier UI observation of this window in this session, so this is the full UI." };- return { ...result, details: { ...result.details, changes } };+ if (view.changes && result.details.status === "observed") {+ const captured = session.windowFrame(windowRef!);+ const latest = frame && captured?.observation_id === result.details.observation_id ? captured : undefined;+ const diff = compareCaptures({ ui: baseline?.ui, frame: latest ? await decodeFrame(frame!.png) : undefined },+ { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined });+ if (!diff) return { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } };+ const since = baseline?.observation_id ?? frame!.observation_id;+ if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;+ // The change list replaces the full overview: only crops of the changed area are shown.+ const crops = latest ? await cropImages(frame!, latest, diff) : [];+ return { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } };}if (view.kind === "state" && result.details.status === "observed") {const health = await session.processHealth(target);@@ -748,7 +768,7 @@ export class ComputerResources {let follow: FollowUp | undefined;if (tool === "insert" && window) {const command = commandObject(value);- follow = await this.followUpSpec(window, command, cwd);+ follow = await this.withFrame(await this.followUpSpec(window, command, cwd), window, address.ref, cwd);// Was the postcondition already true before anything is sent? Browsers sample now; native uses the input walk.if (follow.expect) {const session = await this.sessionFor(cwd, { window_ref: window });@@ -769,16 +789,9 @@ export class ComputerResources {this.snapshots.delete(address.ref);return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), command, cwd, signal, follow);}- } else if (tool === "write" && window) follow = { successor: true, explicit: false };+ } else if (tool === "write" && window) follow = await this.withFrame({ successor: true, explicit: false }, window, address.ref, cwd);const result = await this.act(address, snapshot, input, tool, cwd, signal);if (!follow || !window) return result;- // The default UI change list follows input aimed at a UI element. Input by pixels or keys alone is meant for- // surfaces the UI tree does not describe (canvases, games); a walk would cost time and, for delivered- // pixel-only input, replace the observation it keeps for more pixel input.- if (!follow.explicit && !snapshot.result.details.dom && !elementAddressed(address, tool, input)) {- if (result.details.status !== "delivered") return result;- return { ...result, content: [...result.content, { type: "text", text: "Input by pixels or keys alone is not followed by a UI walk. Read the window, or add \"successor\":true, to see what changed." }] };- }return this.followUp(window, snapshot, result, follow, cwd, signal);}@@ -827,6 +840,13 @@ export class ComputerResources {return result;}+ /** Keep the input observation's own pixels as the baseline of the pixel comparison; they must be taken before the action. */+ private async withFrame(follow: FollowUp, window: string, observation: string | undefined, cwd: string): Promise<FollowUp> {+ if (!follow.successor) return follow;+ const frame = (await this.sessionFor(cwd, { window_ref: window })).windowFrame(window);+ return frame && frame.observation_id === observation ? { ...follow, frame } : follow;+ }+/** Validate an action's optional `expect` postcondition and `successor` request before anything is dispatched. */private async followUpSpec(window: string, command: Record<string, unknown>, cwd: string): Promise<FollowUp> {if (command.successor !== undefined && typeof command.successor !== "boolean") throw new ComputerRequestError("successor must be boolean.");@@ -852,7 +872,7 @@ export class ComputerResources {if (!follow.explicit && (status === "refused" || (before.result.details.dom && !result.details.dom))) return result;const session = await this.sessionFor(cwd, { window_ref: window });const preexisting = follow.preexisting === true;- let successor: ComputerResult | undefined, settled: StateDiff | undefined;+ let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];let postcondition: ComputerResult["details"]["postcondition"];if (follow.expect && status !== "refused") {// Unknown delivery is sampled once for information, but never reported as verified.@@ -868,19 +888,7 @@ export class ComputerResources {postcondition = { status: preexisting ? "preexisting" : "failed", reason: "The action was refused; nothing was sent." };} else if (status !== "refused") {if (result.details.dom) successor = result;- else {- // An app may repaint a little after input is delivered: when the first walk shows no change, look again- // for a short while before saying that nothing changed.- const started = performance.now();- for (let attempt = 0; ; attempt++) {- this.retireWindow(window);- successor = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));- const waited = Math.round(performance.now() - started);- settled = before.result.details.ui && successor.details.ui ? { ...diffUi(before.result.details.ui, successor.details.ui), waited_ms: waited } : undefined;- if (!settled || !diffIsEmpty(settled) || successor.details.status !== "observed" || signal?.aborted || waited >= SETTLE_MS) break;- await new Promise(resolve => setTimeout(resolve, Math.min([30, 60, 100][attempt] ?? 150, SETTLE_MS - waited)));- }- }+ else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal));}const after = successor?.details;const diff = before.result.details.dom && after?.dom ? diffDom(before.result.details.dom, after.dom)@@ -890,7 +898,49 @@ export class ComputerResources {...(postcondition ? { postcondition } : {}),successor: after?.observation_id ? { observation_id: after.observation_id, ...(diff ? { diff } : { reason: "The input observation had no comparable UI or DOM." }) }: { reason: status === "refused" ? "The action was refused; the input observation is unchanged but consumed." : "No successor observation could be taken." } };- return { ...result, details };+ return { ...result, content: [...result.content, ...crops], details };+ }++ /**+ * Capture the window after native input until its state is known: when nothing changed yet, look again for up to+ * SETTLE_MS before saying so; once something changed, capture until two captures in a row are identical, for up+ * to STEADY_MS, so the change list shows the settled window rather than a frame in the middle of a repaint.+ * Captures keep the input observation's overview size so their pixels compare with it.+ */+ private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal):+ Promise<{ successor: ComputerResult; diff?: StateDiff; crops: ComputerResult["content"] }> {+ const started = performance.now();+ let options: UiCaptureOptions = { granularity: "full", ...(frame ? frame.size : {}) };+ const baseline: Capture = { ui: before.result.details.ui, frame: frame ? await decodeFrame(frame.png) : undefined };+ let successor: ComputerResult, diff: StateDiff | undefined, previous: Capture | undefined, latest: WindowFrame | undefined;+ for (let attempt = 0; ; attempt++) {+ this.retireWindow(window);+ successor = this.remember({ window_ref: window }, await session.observeQuiet(window, signal, options));+ const waited = Math.round(performance.now() - started);+ // A window whose pixels cannot be captured (minimized, or a failed screenshot) still gets its UI compared.+ if (successor.details.status !== "observed" && options.granularity === "full" && !signal?.aborted) {+ options = { granularity: "ui" }; baseline.frame = undefined; attempt--; continue;+ }+ if (successor.details.status !== "observed" || signal?.aborted) break;+ const captured = session.windowFrame(window);+ latest = captured?.observation_id === successor.details.observation_id ? captured : undefined;+ const current: Capture = { ui: successor.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined };+ diff = compareCaptures(baseline, current);+ if (!diff) break;+ diff.waited_ms = waited;+ if (diffIsEmpty(diff)) {+ if (waited >= SETTLE_MS) break;+ previous = undefined;+ await new Promise(resolve => setTimeout(resolve, Math.min([30, 60, 100][attempt] ?? 150, SETTLE_MS - waited)));+ continue;+ }+ const steady = previous && compareCaptures(previous, current);+ if (steady && diffIsEmpty(steady)) break;+ if (waited >= STEADY_MS) { diff.unsettled = true; break; }+ previous = current;+ await new Promise(resolve => setTimeout(resolve, STEADY_GAP_MS));+ }+ return { successor, diff, crops: diff && frame && latest ? await cropImages(frame, latest, diff) : [] };}/**@@ -898,14 +948,14 @@ export class ComputerResources {* matches and is ready (enabled for input; for pointer input also on screen and at the same place in two walks), or* until the timeout. Several matches refuse at once with each match's locator, like Playwright's strict mode.*/- private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; visible?: boolean; describe: string }, signal?: AbortSignal):+ private async resolveNative(window: string, session: ComputerSession, selector: ParsedSelector, options: { timeout_ms: number; enabled: boolean; stable: boolean; visible?: boolean; describe: string; pixels?: boolean }, signal?: AbortSignal):Promise<{ element: UiElement; observation: ComputerResult } | { refused: string; observation?: ComputerResult }> {const deadline = performance.now() + options.timeout_ms;let previous: UiElement | undefined, reason = "";for (let attempt = 0; ; attempt++) {if (signal?.aborted) return { refused: "Cancelled while waiting for the target; nothing was sent." };this.retireWindow(window);- const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal));+ const fresh = this.remember({ window_ref: window }, await session.observeQuiet(window, signal, options.pixels ? { granularity: "full" } : undefined));if (fresh.details.status !== "observed" || !fresh.details.ui) return { refused: `No current UI sample of the window: ${computerResultText(fresh).split("\n")[0]}`, observation: fresh };const ui = fresh.details.ui;let found: UiElement[];@@ -945,7 +995,8 @@ export class ComputerResources {if (!NATIVE_LOCATOR_ACTIONS.includes(action)) throw new ComputerRequestError(`A target takes one of ${[...NATIVE_LOCATOR_ACTIONS, ...Object.keys(LOCATOR_VERBS)].join(", ")}.`);const describe = typeof target === "string" ? target : JSON.stringify(target);const pointer = ["click", "double_click", "right_click", "hover", "check", "uncheck"].includes(action);- const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, visible: pointer, describe }, signal);+ // With a successor, the resolving walks also capture pixels: the last one is the pixel baseline of the action.+ const resolved = await this.resolveNative(window, session, selector, { timeout_ms: (timeout as number | undefined) ?? 5_000, enabled: action !== "hover", stable: pointer, visible: pointer, describe, pixels: follow?.successor }, signal);const mode = this.getMode();if ("refused" in resolved) return { content: [{ type: "text", text: resolved.refused }], details: { status: "refused", mode, window_ref: window }, isError: true };const { element, observation } = resolved;@@ -961,6 +1012,7 @@ export class ComputerResources {// The walk that resolved the target is the state before the action: the change list compares with it.const before: Snapshot = { target: { window_ref: window }, result: observation };if (follow?.expect && observation.details.ui) follow.preexisting = combine(evaluateNative(observation.details.ui, follow.expect.conditions)) === "satisfied";+ if (follow) follow = await this.withFrame(follow, window, observation.details.observation_id, cwd);const finish = (result: ComputerResult) => follow ? this.followUp(window, before, result, follow, cwd, signal) : Promise.resolve(result);this.snapshots.delete(observation.details.observation_id!);const result = await session.act(nativeCommand as Action, signal);diff --git a/src/computer-views.ts b/src/computer-views.tsindex eaf2bc9..457583c 100644--- a/src/computer-views.ts+++ b/src/computer-views.ts@@ -24,7 +24,7 @@ function keys(value: Record<string, unknown>, allowed: readonly string[]): void/** Validate owned views before capturing; text offsets are never image coordinates.* `ocr` adds Windows text recognition to a window overview, an observation or a region detail.- * `changes` reads a window's UI and lists only what changed since its previous observation.+ * `changes` reads a window's UI and pixels and lists only what changed since its previous observation.* `regions` adds icons and text areas found by Cua perception to a window overview or its current observation.* `aria` reads a window's UI as a Playwright aria snapshot whose refs are the observation's hints. */export function parseComputerViews(views: readonly string[] = []): ComputerViews & { ocr?: true; changes?: true; regions?: true; aria?: true } {@@ -42,9 +42,9 @@ export function parseComputerViews(views: readonly string[] = []): ComputerViews}if (views.includes("changes")) {const view = parseView(views.filter(view => view !== "changes"));- if (view.kind !== "overview" || (view.capture.granularity ?? "ui") !== "ui")- throw Error("Use changes alone or with ui capture options such as max_elements; it compares UI, not pixels.");- return { kind: "overview", capture: { ...view.capture, granularity: "ui" }, changes: true };+ if (view.kind !== "overview")+ throw Error("Use changes alone (UI and pixels), or with ui, image or full and their capture options; it compares a fresh capture with the window's previous observation.");+ return { kind: "overview", capture: { ...view.capture, granularity: view.capture.granularity ?? "full" }, changes: true };}if (!views.includes("ocr")) return parseView(views);const view = parseView(views.filter(view => view !== "ocr"));diff --git a/src/pixel-diff.ts b/src/pixel-diff.tsnew file mode 100644index 0000000..d7b4f14--- /dev/null+++ b/src/pixel-diff.ts@@ -0,0 +1,107 @@+import sharp from "sharp";++/** Decoded RGBA pixels of one window overview capture. */+export interface PixelFrame { width: number; height: number; data: Buffer }+export interface PixelBox { x: number; y: number; width: number; height: number }+/** How a window's overview pixels changed between two captures, in overview pixels. */+export interface PixelDiff {+ /** Overview size of both captures; when they differ, `size_changed` holds the earlier size and nothing else is compared. */+ width: number; height: number;+ size_changed?: { width: number; height: number };+ /** Box around every counted change; absent when nothing counted changed. */+ box?: PixelBox;+ /** Changed pixels that count, and their share of the whole overview. */+ changed_pixels: number; share: number;+ /** Separate changed spots that count. */+ regions: number;+ /** Spots too small to count, such as a blinking caret. */+ ignored: number;+}++/** A channel must move by more than this (of 255) for its pixel to count as changed, so compression noise does not. */+const CHANNEL_THRESHOLD = 26;+/** Changed pixels are grouped into spots of touching cells this large. */+const CELL = 8;+/** A spot with fewer changed pixels than this does not count. */+const MIN_SPOT_PIXELS = 12;+/** A spot at most this wide and this tall is caret-shaped (a text cursor blinking) and does not count. */+const CARET_WIDTH = 3, CARET_HEIGHT = 64;++export async function decodeFrame(png: Buffer): Promise<PixelFrame> {+ const { data, info } = await sharp(png, { limitInputPixels: 32 * 1024 * 1024 }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });+ return { width: info.width, height: info.height, data };+}++/** Compare two captures of the same window: which pixels changed, where, and how much, ignoring caret-sized spots. */+export function diffFrames(before: PixelFrame, after: PixelFrame): PixelDiff {+ const { width, height } = after;+ if (before.width !== width || before.height !== height)+ return { width, height, size_changed: { width: before.width, height: before.height }, changed_pixels: 0, share: 0, regions: 0, ignored: 0 };+ const columns = Math.ceil(width / CELL), rows = Math.ceil(height / CELL);+ // Per cell: changed pixel count and the exact box of its changed pixels.+ const count = new Uint32Array(columns * rows);+ const left = new Int32Array(columns * rows).fill(width), top = new Int32Array(columns * rows).fill(height);+ const right = new Int32Array(columns * rows).fill(-1), bottom = new Int32Array(columns * rows).fill(-1);+ const a = before.data, b = after.data;+ for (let y = 0; y < height; y++) {+ const row = y * width * 4, cellRow = Math.floor(y / CELL) * columns;+ for (let x = 0; x < width; x++) {+ const i = row + x * 4;+ if (Math.abs(a[i] - b[i]) <= CHANNEL_THRESHOLD && Math.abs(a[i + 1] - b[i + 1]) <= CHANNEL_THRESHOLD && Math.abs(a[i + 2] - b[i + 2]) <= CHANNEL_THRESHOLD) continue;+ const cell = cellRow + Math.floor(x / CELL);+ count[cell]++;+ if (x < left[cell]) left[cell] = x;+ if (x > right[cell]) right[cell] = x;+ if (y < top[cell]) top[cell] = y;+ if (y > bottom[cell]) bottom[cell] = y;+ }+ }+ // Spots are groups of changed cells that touch, including diagonally.+ const seen = new Uint8Array(columns * rows);+ let box: PixelBox | undefined, changed = 0, regions = 0, ignored = 0;+ for (let start = 0; start < count.length; start++) {+ if (!count[start] || seen[start]) continue;+ const stack = [start]; seen[start] = 1;+ let pixels = 0, x0 = width, y0 = height, x1 = -1, y1 = -1;+ while (stack.length) {+ const cell = stack.pop()!;+ pixels += count[cell];+ x0 = Math.min(x0, left[cell]); y0 = Math.min(y0, top[cell]); x1 = Math.max(x1, right[cell]); y1 = Math.max(y1, bottom[cell]);+ const cx = cell % columns, cy = Math.floor(cell / columns);+ for (let dy = -1; dy <= 1; dy++) for (let dx = -1; dx <= 1; dx++) {+ const nx = cx + dx, ny = cy + dy, next = ny * columns + nx;+ if (nx < 0 || ny < 0 || nx >= columns || ny >= rows || seen[next] || !count[next]) continue;+ seen[next] = 1; stack.push(next);+ }+ }+ const spot = { x: x0, y: y0, width: x1 - x0 + 1, height: y1 - y0 + 1 };+ if (pixels < MIN_SPOT_PIXELS || (spot.width <= CARET_WIDTH && spot.height <= CARET_HEIGHT)) { ignored++; continue; }+ regions++; changed += pixels;+ box = box ? union(box, spot) : spot;+ }+ return { width, height, ...(box ? { box } : {}), changed_pixels: changed, share: changed / (width * height), regions, ignored };+}++export const pixelsChanged = (diff: PixelDiff) => diff.size_changed !== undefined || diff.box !== undefined;++function union(a: PixelBox, b: PixelBox): PixelBox {+ const x = Math.min(a.x, b.x), y = Math.min(a.y, b.y);+ return { x, y, width: Math.max(a.x + a.width, b.x + b.width) - x, height: Math.max(a.y + a.height, b.y + b.height) - y };+}++/** Each crop's long edge is at most this many pixels, so two crops cost little next to a full overview. */+const CROP_EDGE = 512;+/** Context kept around the changed box on each side, in overview pixels. */+const CROP_MARGIN = 12;++/** Before and after PNG crops of the changed box with a small margin, scaled down when large. */+export async function changeCrops(before: Buffer, after: Buffer, diff: PixelDiff): Promise<{ region: PixelBox; before: Buffer; after: Buffer } | undefined> {+ if (!diff.box || diff.size_changed) return undefined;+ const x = Math.max(0, diff.box.x - CROP_MARGIN), y = Math.max(0, diff.box.y - CROP_MARGIN);+ const region = { x, y, width: Math.min(diff.width, diff.box.x + diff.box.width + CROP_MARGIN) - x, height: Math.min(diff.height, diff.box.y + diff.box.height + CROP_MARGIN) - y };+ const crop = (png: Buffer) => sharp(png, { limitInputPixels: 32 * 1024 * 1024 })+ .extract({ left: region.x, top: region.y, width: region.width, height: region.height })+ .resize(CROP_EDGE, CROP_EDGE, { fit: "inside", withoutEnlargement: true }).png().toBuffer();+ const [a, b] = await Promise.all([crop(before), crop(after)]);+ return { region, before: a, after: b };+}diff --git a/src/provider.ts b/src/provider.tsindex aa7f8af..c60d02c 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -8,7 +8,7 @@ import { readFile } from "node:fs/promises";import { ComputerResources, requireComputerAddress, computerResultText, ComputerRequestError, type SandboxFactory, type MachineFactory } from "./computer-resources.ts";import { computerSource } from "./computer-address.ts";import type { WebLogEntry } from "./web-browser.ts";-import type { StateDiff } from "./successor.ts";+import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";import { computerActionRenderer } from "./computer-action-renderer.ts";import { computerReadRenderer } from "./computer-read-renderer.ts";import type { ComputerSession, ComputerResult, ComputerMode, Content } from "./session.ts";@@ -189,21 +189,37 @@ export function presentComputerResult(result: ComputerResult, full = false): strconst changeCount = (c: { changed: number; added: number; removed: number }) =>c.changed + c.added + c.removed ? `${c.changed} changed, ${c.added} added, ${c.removed} removed` : "nothing changed";-/** Lines of a compact UI or DOM change list; changed and added rows carry the new observation's hints. */+/** Lines of a compact UI, DOM or pixel change list; changed and added rows carry the new observation's hints. */function diffLines(diff: StateDiff, afterAction = false): string[] {const page = diff.page ?? {};const total = diff.counts.changed + diff.counts.added + diff.counts.removed;const omitted = total - diff.changed.length - diff.added.length - diff.removed.length;+ const pixels = diff.pixels;+ const subject = diff.kind === "dom" ? "on the page" : diff.kind === "pixels" ? "in the window's pixels" : pixels ? "in the UI or the window's pixels" : "in the UI";+ const nothing = !afterAction ? `Nothing changed ${subject} since that observation.`+ : `Nothing changed ${subject}${diff.waited_ms !== undefined ? ` within ${diff.waited_ms} ms` : ""} after the action. Check that the target was right before acting again; do not replay blindly.`;+ const uiChanged = total > 0 || Object.keys(page).length > 0;return [...Object.entries(page).map(([key, change]) => `Changed ${key}: ${JSON.stringify(change![0])} → ${JSON.stringify(change![1])}`),- total === 0 && !Object.keys(page).length- ? (!afterAction ? `Nothing changed in the ${diff.kind === "ui" ? "UI" : "page"} since that observation.`- : diff.kind === "ui" ? `Nothing changed in the UI${diff.waited_ms !== undefined ? ` within ${diff.waited_ms} ms` : ""} after the action. Check that the target was right before acting again; do not replay blindly.` : "Nothing changed on the page after the action. Check that the target was right before acting again; do not replay blindly.")- : `Changes: ${diff.counts.changed} changed, ${diff.counts.added} added, ${diff.counts.removed} removed`,+ ...(diffIsEmpty(diff) ? [nothing] : diff.kind === "pixels" ? [] : [`Changes: ${diff.counts.changed} changed, ${diff.counts.added} added, ${diff.counts.removed} removed`]),...diff.changed.map(line => `~ ${label(line)}`), ...diff.added.map(line => `+ ${label(line)}`), ...diff.removed.map(line => `- ${label(line)}`),...(omitted > 0 ? [`${omitted} more changes are not listed; Search or Read the new observation for them.`] : []),+ ...(pixels && !diffIsEmpty(diff) ? pixelLines(pixels, uiChanged) : []),+ ...(afterAction && !pixels && diff.kind === "ui" ? ["Pixels were not compared: the input observation had no pixels."] : []),+ ...(diff.unsettled ? [`The window was still changing ${diff.waited_ms} ms after the action; the successor shows it at that moment.`] : []),...(diff.note ? [diff.note] : [])];}+/** How the window's pixels changed, in overview pixels, and what the attached crops show. */+function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {+ const since = pixels.since ? ` since ${computerSource("observation", pixels.since)}` : "";+ if (pixels.size_changed) return [`The window's overview size changed${since} from ${pixels.size_changed.width}x${pixels.size_changed.height} to ${pixels.width}x${pixels.height}; pixels were not compared.`];+ if (!pixels.box) return uiChanged ? [`No visible pixel change${since}.`] : [];+ const box = pixels.box, share = pixels.share * 100;+ const crop = pixels.crop;+ return [`Pixels changed${since} in x=${box.x} y=${box.y} w=${box.width} h=${box.height} (overview pixels; ${share < 0.1 ? share.toFixed(2) : share.toFixed(1)}% of the window, ${pixels.regions} ${pixels.regions === 1 ? "spot" : "spots"}).`,+ ...(crop ? [`The two images crop x=${crop.x} y=${crop.y} w=${crop.width} h=${crop.height}: before, then after.`] : [])];+}+/** Whole lines from the start that fit a byte budget, never splitting Unicode. */function fittingLines(lines: readonly string[], budget: number): string[] {const kept: string[] = [];@@ -344,7 +360,9 @@ export async function performComputer(resources: ComputerResources, tool: "write// Completion of the native request is separate from application-effect certainty.const ok = !result.isError && ["delivered", "observed", "closed"].includes(result.details.status);const summary = presentComputerResult(result);- return { source, summary, data: { kind: "computer-action", ok,+ // Images of an action result, such as before/after crops of its pixel change, follow the summary for the model.+ const images = result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");+ return { source, summary, ...(images.length ? { images } : {}), data: { kind: "computer-action", ok,effect, ...result.details,receipt: computerResultText(result),...(!ok ? { error: { code: typeof result.details.state?.code === "string" ? result.details.state.code : `COMPUTER_${effect.toUpperCase().replace("-", "_")}`,diff --git a/src/session.ts b/src/session.tsindex c9148c9..3c49df6 100644--- a/src/session.ts+++ b/src/session.ts@@ -141,7 +141,9 @@ interface Observation { texts?: Map<string, string>; id: string; tokens: Set<str/** Kept after pixel-only input: coordinates stay valid until a freshness check finds a changed size or state; UI tokens and the native capture are spent. */pixelsOnly?: boolean }interface WindowState { id: string; nativeId: string; kind: "restore" | "management" }-interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark }+/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */+export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }+interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; observation?: Observation; windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */export function textMatcher(query?: string): (text: string) => boolean {@@ -1029,6 +1031,7 @@ export class ComputerSession {const raw = images[0];if (raw?.type === "image" && Buffer.byteLength(raw.data, "base64") <= 12 * 1024 * 1024) {observed.rawImage = raw;+ entry.frame = { observation_id: id, png: Buffer.from(raw.data, "base64"), size: { maxLongEdge: settings.maxLongEdge, scale: settings.scale } };const active = new Set([...this.entries.values()].map(e => e.observation));this.uiImages = [...this.uiImages.filter(old => active.has(old)), observed];for (const old of this.uiImages.splice(0, Math.max(0, this.uiImages.length - 3))) old.rawImage = undefined;@@ -1244,15 +1247,21 @@ export class ComputerSession {});}- /** A fresh UI-only observation without the human frame, for successors and batch steps. */- observeQuiet(ref: string, signal?: AbortSignal): Promise<ComputerResult> {+ /** The window's latest unannotated capture with pixels, the baseline for comparing its pixels; undefined when none is kept. */+ windowFrame(ref: string): WindowFrame | undefined { return this.entries.get(ref)?.frame; }++ /**+ * A fresh observation without the human frame, for successors and batch steps: UI-only by default, or with+ * pixels at a given overview size so they compare with an earlier capture.+ */+ observeQuiet(ref: string, signal?: AbortSignal, options: UiCaptureOptions = { granularity: "ui" }): Promise<ComputerResult> {return this.queue(async () => {if (signal?.aborted) return this.result("refused", "Cancelled before observation.");try {const entry = await this.current(ref, signal);if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");if (!this.allowed(entry)) return this.denied(entry.ref);- return await this.capture(entry, signal, undefined, { granularity: "ui" }, true);+ return await this.capture(entry, signal, undefined, options, true);} catch (error) { return this.failed(error, false); }});}diff --git a/src/standalone-tools.ts b/src/standalone-tools.tsindex 4cf491d..6cd14b1 100644--- a/src/standalone-tools.ts+++ b/src/standalone-tools.ts@@ -97,7 +97,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerResconst action = await performComputer(resources, tool, String(input.path), input, ctx.cwd, signal);// The receipt is plain data with a few untyped native fields; a JSON copy gives scripts exactly that.const receipt: JsonValue = JSON.parse(JSON.stringify(action.script));- return { content: [{ type: "text", text: action.summary }], details: { results: [], metadata: { semanticAction: action.data } },+ return { content: [{ type: "text", text: action.summary }, ...("images" in action ? action.images ?? [] : [])], details: { results: [], metadata: { semanticAction: action.data } },structuredContent: receipt, ...(action.data.ok ? {} : { isError: true }) };},});diff --git a/src/successor.ts b/src/successor.tsindex 252a9f4..a2d26f5 100644--- a/src/successor.ts+++ b/src/successor.ts@@ -1,9 +1,14 @@import type { UiElement, UiResource } from "./ui-resources.ts";import type { WebDom } from "./session.ts";+import { diffFrames, pixelsChanged, type PixelBox, type PixelDiff, type PixelFrame } from "./pixel-diff.ts";++/** How a window's overview pixels changed; `crop` is the region the before/after crops show. */+export type PixelReport = PixelDiff & { crop?: PixelBox; since?: string };/** Compact change list between an action's input observation and its successor. Bounded; full successors stay readable. */export interface StateDiff {- kind: "dom" | "ui";+ /** What was compared: page lines, UI elements, or only pixels when the earlier observation had no UI. */+ kind: "dom" | "ui" | "pixels";/** Lines or elements that changed, appeared or disappeared, at most DIFF_LIMIT each. */changed: string[]; added: string[]; removed: string[];/** Totals before bounding. */@@ -11,8 +16,12 @@ export interface StateDiff {page?: { url?: [string, string]; title?: [string, string]; tab?: [string | null, string | null] };/** Why the diff may be incomplete, such as a partial native walk. */note?: string;- /** How long after the action the native walks kept looking for a change before reporting none. */+ /** How long after the action the native captures kept looking for a change, or waited for the window to settle. */waited_ms?: number;+ /** The window's overview pixels compared with the earlier capture; absent when either side had no pixels. */+ pixels?: PixelReport;+ /** Set when the window was still changing when the wait ended; the successor shows it at that moment. */+ unsettled?: true;}const DIFF_LIMIT = 30;const bounded = (items: string[]) => items.slice(0, DIFF_LIMIT);@@ -124,4 +133,14 @@ export function diffUi(before: UiResource, after: UiResource): StateDiff {/** Whether a diff found any difference at all. */export const diffIsEmpty = (diff: StateDiff): boolean =>- diff.counts.changed + diff.counts.added + diff.counts.removed === 0 && !Object.keys(diff.page ?? {}).length;+ diff.counts.changed + diff.counts.added + diff.counts.removed === 0 && !Object.keys(diff.page ?? {}).length && !(diff.pixels && pixelsChanged(diff.pixels));++/** One native capture of a window: its UI walk and its decoded overview pixels, either of which may be missing. */+export interface Capture { ui?: UiResource; frame?: PixelFrame }+/** Compare two native captures of a window: UI elements when both have UI, pixels when both have pixels. */+export function compareCaptures(before: Capture, after: Capture): StateDiff | undefined {+ const pixels = before.frame && after.frame ? diffFrames(before.frame, after.frame) : undefined;+ const ui = before.ui && after.ui ? diffUi(before.ui, after.ui) : undefined;+ if (ui) return pixels ? { ...ui, pixels } : ui;+ return pixels ? { kind: "pixels", changed: [], added: [], removed: [], counts: { changed: 0, added: 0, removed: 0 }, pixels } : undefined;+}\ No newline at end of filediff --git a/test/action-changes.test.ts b/test/action-changes.test.tsindex 6023618..b60fd2d 100644--- a/test/action-changes.test.ts+++ b/test/action-changes.test.ts@@ -1,28 +1,46 @@+import sharp from "sharp";import { expect, test, vi } from "vitest";import { computerSource } from "../src/computer-address.ts";import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";import { presentComputerResult } from "../src/provider.ts";-import { ComputerSession, type Driver } from "../src/session.ts";+import { ComputerSession, type ComputerResult, type Content, type Driver } from "../src/session.ts";const cwd = "/synthetic-action-changes";const address = requireComputerAddress;// LPT-914: every action reports what changed in the window, like Playwright MCP's incremental snapshots.+// LPT-915: and how the window's pixels changed, with before/after crops of the changed area.async function fixture() {let count = 0, serial = 0;+ const dots: { x: number; y: number }[] = [];+ // The window paints one bar per count under the button and a dot wherever a pixel click lands on the canvas.+ const frame = async () => {+ const shapes = [...Array.from({ length: count }, (_, i) => `<rect x="${20 + i * 12}" y="40" width="10" height="10" fill="#000"/>`),+ ...dots.map(dot => `<rect x="${dot.x - 3}" y="${dot.y - 3}" width="6" height="6" fill="#c00"/>`)];+ const png = await sharp({ create: { width: 500, height: 320, channels: 3, background: "#fff" } })+ .composite([{ input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="500" height="320">${shapes.join("")}</svg>`) }]).png().toBuffer();+ return { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };+ };const element = (index: number, role: string, label: string) => ({ element_token: `s${serial}:${index}`, role, label, enabled: true, actions: role === "Button" ? ["invoke"] : [],frame: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });const driver: Driver = {identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),call: vi.fn<Driver["call"]>(async (name, args) => {+ const input = args as { element_token?: string; x?: number; y?: number; include_screenshot?: boolean };if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };if (name === "get_window_state") {serial++;- return { content: [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },+ const pixels = input.include_screenshot !== false;+ return { content: pixels ? [await frame()] : [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },+ ...(pixels ? { capture_id: `c${serial}`, screenshot_width: 500, screenshot_height: 320 } : {}),elements_complete: true, elements: [element(0, "Button", `Count ${count}`), element(1, "Text", "Status")] } };}- // Clicking the button counts; clicking the status label does nothing.- if (name === "click") { if (String((args as { element_token?: string }).element_token).endsWith(":0")) count++; return { content: [], structuredContent: { effect: "confirmed" } }; }+ // Clicking the button counts; clicking the status label does nothing; a pixel click paints a dot there.+ if (name === "click") {+ if (input.element_token?.endsWith(":0")) count++;+ if (input.x !== undefined && input.y !== undefined && input.y > 100) dots.push({ x: input.x, y: input.y });+ return { content: [], structuredContent: { effect: "confirmed" } };+ }throw Error(`Unexpected native call ${name}`);}),};@@ -30,13 +48,56 @@ async function fixture() {const resources = new ComputerResources(async () => session, async () => session.close());const ref = ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;const window = computerSource("window", ref);- const read = async () => {- const view = (await resources.read(address(window), cwd, ["ui"]))!;- return { view, at: (index: number) => address(`${computerSource("observation", view.details.observation_id!)}?hint=${encodeURIComponent(view.details.ui!.elements[index].hint)}`) };+ const read = async (views = ["ui"]) => {+ const view = (await resources.read(address(window), cwd, views))!;+ const observation = computerSource("observation", view.details.observation_id!);+ return { view, observation: address(observation), at: (index: number) => address(`${observation}?hint=${encodeURIComponent(view.details.ui!.elements[index].hint)}`) };};const walks = () => vi.mocked(driver.call).mock.calls.filter(([name]) => name === "get_window_state").length;return { resources, window, read, walks };}+const images = (result: ComputerResult) => result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");++test("an action on an observation with pixels reports the changed area with before and after crops", async () => {+ const f = await fixture(), { at } = await f.read(["full"]);+ const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");+ expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, crop: { x: 8, y: 28, width: 34, height: 34 } });+ expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in x=20 y=40 w=10 h=10 \(overview pixels; 0\.06% of the window, 1 spot\)\.\nThe two images crop x=8 y=28 w=34 h=34: before, then after\./);+ const [before, after] = images(result);+ expect((await sharp(Buffer.from(before.data, "base64")).stats()).channels[0].min).toBe(255);+ expect((await sharp(Buffer.from(after.data, "base64")).stats()).channels[0].min).toBe(0);+});++test("pixel input on a canvas reports only its pixel change, and empty space reports nothing", async () => {+ const f = await fixture(), { observation } = await f.read(["full"]);+ const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }), "insert");+ expect(stroke.details.successor?.diff).toMatchObject({ counts: { changed: 0, added: 0, removed: 0 }, pixels: { box: { x: 297, y: 197, width: 6, height: 6 } } });+ expect(images(stroke)).toHaveLength(2);+ const next = address(computerSource("observation", stroke.details.successor!.observation_id!));+ const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }), "insert");+ expect(presentComputerResult(empty)).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);+ expect(images(empty)).toHaveLength(0);+});++test("an action on a UI-only observation says that pixels were not compared, and its successor has them", async () => {+ const f = await fixture(), { at } = await f.read();+ const first = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }), "insert");+ expect(presentComputerResult(first)).toMatch(/Pixels were not compared: the input observation had no pixels\./);+ const view = (await f.resources.read(address(computerSource("observation", first.details.successor!.observation_id!)), cwd, []))!;+ const hint = view.details.ui!.elements[0].hint;+ const second = await f.resources.perform(address(`${computerSource("observation", first.details.successor!.observation_id!)}?hint=${hint}`), cwd, JSON.stringify({ action: "click" }), "insert");+ expect(second.details.successor?.diff?.pixels?.box).toEqual({ x: 32, y: 40, width: 10, height: 10 });+});++test("a changes Read compares the window's pixels with its previous observation", async () => {+ const f = await fixture(), { observation } = await f.read(["full"]);+ await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200, successor: false }), "insert");+ const changes = (await f.resources.read(address(f.window), cwd, ["changes"]))!;+ expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 297, y: 197, width: 6, height: 6 });+ expect(presentComputerResult(changes)).toMatch(/Pixels changed in x=297 y=197 w=6 h=6/);+ // Only the two crops, not the full overview.+ expect(images(changes)).toHaveLength(2);+});test("an action reports what it changed by default, and its successor is the current observation", async () => {const f = await fixture(), { at } = await f.read();diff --git a/test/computer-resources.test.ts b/test/computer-resources.test.tsindex 877c9c6..29ea722 100644--- a/test/computer-resources.test.ts+++ b/test/computer-resources.test.ts@@ -136,8 +136,10 @@ test("a new overview retires the prior resource instead of rebinding its hint",expect(f.inputs()).toHaveLength(0);expect((await f.resources.perform(address(`${latest.source}?hint=${encodeURIComponent(latest.hint!)}`), cwd, "current", "write")).details.status).toBe("delivered");expect(f.inputs()[0][1]).toMatchObject({ element_token: "snapshot-2:0", text: "current" });- // Two overviews, then only the successor's UI walks after the input.- expect(f.captures().slice(2).every(([, args]) => (args as { include_screenshot?: boolean }).include_screenshot === false)).toBe(true);+ // Two overviews, then the input, then the successor's captures.+ const order = f.native().map(([name]) => name).filter(name => name !== "list_windows" && name !== "get_window_control_state");+ expect(order.slice(0, 3)).toEqual(["get_window_state", "get_window_state", f.inputs()[0][0]]);+ expect(order.slice(3).every(name => name === "get_window_state")).toBe(true);});test.each(["safe", "closed", "reused"] as const)("%s targets refuse cached reads and input without another capture", async change => {@@ -246,11 +248,12 @@ test("a display overview accepts a size for that Read only", async () => {await expect(f.resources.read(display, cwd, ['capture:{"max_elements":5}'])).rejects.toThrow(/accepts only capture/);});-test("pixel-only input keeps the observation until the window size changes; UI hints and other input spend it", async () => {+test("pixel-only input without a successor keeps the observation until the window size changes; UI hints and other input spend it", async () => {const f = await fixture();(f.driver as Driver & { windowManagement: boolean }).windowManagement = true;const view = await f.read();- const click = (x: number) => f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x, y: 40 }), "insert");+ // A default successor would replace the observation; without one, pixel-only input keeps it for more pixel input.+ const click = (x: number) => f.resources.perform(view.observation, cwd, JSON.stringify({ action: "click", x, y: 40, successor: false }), "insert");for (const x of [10, 20, 30]) expect((await click(x)).details.status).toBe("delivered");expect(f.captures()).toHaveLength(1);// The native capture is single-use; kept observations send window-local pixels without it.@@ -261,9 +264,9 @@ test("pixel-only input keeps the observation until the window size changes; UI hexpect(hinted.content[0]).toMatchObject({ text: expect.stringContaining("UI hints of this observation expired") });const again = await f.read();f.setBounds({ x: 300, y: 90, width: 500, height: 320 });- expect((await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert")).details.status).toBe("delivered");+ expect((await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert")).details.status).toBe("delivered");f.setBounds({ x: 300, y: 90, width: 640, height: 320 });- const resized = await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert");+ const resized = await f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5, successor: false }), "insert");expect(resized.details.status).toBe("refused");expect(resized.content[0]).toMatchObject({ text: expect.stringContaining("resized from 500x320 to 640x320") });await expect(f.resources.perform(again.observation, cwd, JSON.stringify({ action: "click", x: 5, y: 5 }), "insert")).rejects.toThrow(/expired/i);diff --git a/test/pixel-diff.test.ts b/test/pixel-diff.test.tsnew file mode 100644index 0000000..da0dfbf--- /dev/null+++ b/test/pixel-diff.test.ts@@ -0,0 +1,78 @@+import sharp from "sharp";+import { describe, expect, test } from "vitest";+import { changeCrops, decodeFrame, diffFrames, pixelsChanged, type PixelFrame } from "../src/pixel-diff.ts";++const blank = (width = 200, height = 100): PixelFrame => ({ width, height, data: Buffer.alloc(width * height * 4, 255) });+function paint(frame: PixelFrame, x: number, y: number, width: number, height: number, value = 0): PixelFrame {+ const data = Buffer.from(frame.data);+ for (let row = y; row < y + height; row++) for (let column = x; column < x + width; column++) {+ const i = (row * frame.width + column) * 4; data[i] = data[i + 1] = data[i + 2] = value;+ }+ return { ...frame, data };+}+const png = (frame: PixelFrame) => sharp(frame.data, { raw: { width: frame.width, height: frame.height, channels: 4 } }).png().toBuffer();++describe("diffFrames", () => {+ test("identical captures report no change", () => {+ const diff = diffFrames(blank(), blank());+ expect(pixelsChanged(diff)).toBe(false);+ expect(diff).toMatchObject({ changed_pixels: 0, share: 0, regions: 0 });+ });++ test("a stroke reports its box, its pixels and their share", () => {+ const diff = diffFrames(blank(), paint(blank(), 20, 40, 100, 2));+ expect(diff.box).toEqual({ x: 20, y: 40, width: 100, height: 2 });+ expect(diff.changed_pixels).toBe(200);+ expect(diff.share).toBeCloseTo(200 / 20_000);+ expect(diff.regions).toBe(1);+ });++ test("separate spots count as regions inside one box", () => {+ const after = paint(paint(blank(), 10, 10, 10, 10), 150, 70, 20, 20);+ const diff = diffFrames(blank(), after);+ expect(diff.regions).toBe(2);+ expect(diff.box).toEqual({ x: 10, y: 10, width: 160, height: 80 });+ });++ test("a blinking caret and a few stray pixels do not count", () => {+ const after = paint(paint(blank(), 50, 20, 2, 18), 120, 60, 3, 3);+ const diff = diffFrames(blank(), after);+ expect(pixelsChanged(diff)).toBe(false);+ expect(diff.ignored).toBe(2);+ });++ test("a caret next to typed text is part of that change", () => {+ const after = paint(paint(blank(), 20, 20, 40, 12), 61, 18, 2, 16);+ const diff = diffFrames(blank(), after);+ expect(diff.box).toEqual({ x: 20, y: 18, width: 43, height: 16 });+ });++ test("small colour noise below the threshold does not count", () => {+ const diff = diffFrames(blank(), paint(blank(), 0, 0, 200, 100, 240));+ expect(pixelsChanged(diff)).toBe(false);+ });++ test("a different capture size is reported instead of compared", () => {+ const diff = diffFrames(blank(200, 100), blank(300, 100));+ expect(diff.size_changed).toEqual({ width: 200, height: 100 });+ expect(pixelsChanged(diff)).toBe(true);+ });+});++describe("changeCrops", () => {+ test("crops the changed box with a margin from both captures", async () => {+ const before = blank(), after = paint(blank(), 50, 30, 20, 10);+ const crops = (await changeCrops(await png(before), await png(after), diffFrames(before, after)))!;+ expect(crops.region).toEqual({ x: 38, y: 18, width: 44, height: 34 });+ const a = await decodeFrame(crops.before), b = await decodeFrame(crops.after);+ expect([a.width, a.height, b.width, b.height]).toEqual([44, 34, 44, 34]);+ expect(diffFrames(a, b).box).toEqual({ x: 12, y: 12, width: 20, height: 10 });+ });++ test("large changed areas are scaled down to a bounded crop", async () => {+ const before = blank(1600, 900), after = paint(blank(1600, 900), 0, 0, 1600, 900, 0);+ const crops = (await changeCrops(await png(before), await png(after), diffFrames(before, after)))!;+ const meta = await sharp(crops.after).metadata();+ expect(Math.max(meta.width!, meta.height!)).toBe(512);+ });+});diff --git a/test/remote-machine.test.ts b/test/remote-machine.test.tsindex 31832f8..1f4ed04 100644--- a/test/remote-machine.test.ts+++ b/test/remote-machine.test.ts@@ -39,12 +39,14 @@ test("machines come only from validated configuration", async () => {function backend(pid: number) {const driver: Driver = {identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}), selectedDisplays: true,- call: vi.fn<Driver["call"]>(async (name) => {+ call: vi.fn<Driver["call"]>(async (name, args) => {if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };if (name === "list_displays") return { content: [], structuredContent: { platform: "windows", topology_id: `t${pid}`, displays: [{ display_id: `D${pid}`, primary: true,monitor_handle: `0x${pid}`, monitor_device_paths: [`\\\\?\\DISPLAY#${pid}`], bounds: { x: 0, y: 0, width: 1920, height: 1080 },work_area: { x: 0, y: 0, width: 1920, height: 1040 }, scale_factor: 1, dpi: 96 }] } };+ // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],+ ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: `App ${pid}`, launch_path: `C:\\app${pid}.exe` }] } };@@ -84,7 +86,8 @@ test("a machine starts on first use; its windows, apps and displays come only frconst apps = await f.resources.search(address(`${f.machineSource}/apps`), cwd, "App");const app = (apps.details.state!.apps as { app_ref: string }[])[0].app_ref;await f.resources.perform(address(computerSource("app", app)), cwd, JSON.stringify({ action: "open" }), "insert");- expect(f.names(f.remoteDriver)).toEqual(["list_windows", "get_window_state", "type_text", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);+ // The Write's successor walks the window again; repeated walks are collapsed here.+ expect(f.names(f.remoteDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);expect(f.names(f.hostDriver)).toEqual([]);// Every use after the first reuses the started machine.expect(f.start).toHaveBeenCalledOnce();diff --git a/test/sandbox.test.ts b/test/sandbox.test.tsindex f04a86f..3f8a8d5 100644--- a/test/sandbox.test.ts+++ b/test/sandbox.test.ts@@ -99,9 +99,11 @@ function backend(pid: number, domain?: "windows-sandbox") {const driver: Driver = {...(domain ? { sourceDomain: domain } : {}),identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}),- call: vi.fn<Driver["call"]>(async (name) => {+ call: vi.fn<Driver["call"]>(async (name, args) => {if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };+ // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],+ ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };throw new Error(`Unexpected native call ${name}`);@@ -130,7 +132,8 @@ test("guest windows are found, read and operated only through the sandbox's ownexpect(view?.details.status).toBe("observed");await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, "hello", "write");const names = (driver: Driver) => vi.mocked(driver.call).mock.calls.map(([name]) => name);- expect(names(f.guestDriver)).toEqual(["list_windows", "get_window_state", "type_text"]);+ // The Write's successor walks the window again; repeated walks are collapsed here.+ expect(names(f.guestDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state"]);expect(names(f.hostDriver)).toEqual([]);expect(f.resources.descriptor(address(f.sandboxSource))).toContain(f.sandboxSource);});diff --git a/test/ui-changes.test.ts b/test/ui-changes.test.tsindex 3d7f8b9..c847be1 100644--- a/test/ui-changes.test.ts+++ b/test/ui-changes.test.ts@@ -12,11 +12,14 @@ test("a changes Read compares the window's UI with its previous observation", asframe: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });const driver: Driver = {identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),- call: vi.fn<Driver["call"]>(async name => {+ call: vi.fn<Driver["call"]>(async (name, args) => {if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };if (name === "get_window_state") {serial++;+ // This window cannot be captured, like a minimized one: a capture with pixels reports why.+ const pixels = (args as { include_screenshot?: boolean }).include_screenshot !== false;return { content: [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },+ ...(pixels ? { screenshot_error: "The window has no visible surface." } : {}),elements_complete: true, elements: [element(0, "Button", "Save"), element(1, "Edit", "Text editor", text), element(2, "Button", "Close")] } };}throw Error(`Unexpected native call ${name}`);@@ -28,7 +31,8 @@ test("a changes Read compares the window's UI with its previous observation", asconst window = requireComputerAddress(computerSource("window", ref));const first = await resources.read(window, cwd, ["changes"]);- expect(first!.details.changes).toEqual({ reason: expect.stringMatching(/full UI/) });+ expect(first!.details.changes).toEqual({ reason: expect.stringMatching(/full observation/) });+ // This window gives no screenshot, so changes falls back to its UI alone.expect(first!.details.captured).toEqual({ image: false, ui: true });const same = await resources.read(window, cwd, ["changes"]);@@ -43,8 +47,9 @@ test("a changes Read compares the window's UI with its previous observation", asexpect(diff.changed[0]).toBe(`${editor.hint} Edit "Text editor": value="" → "hello"`);expect(resources.grantsAuthority(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)))).toBe(true);- for (const views of [["changes", "image"], ["changes", "full"], ["changes", "ocr"]])- await expect(resources.read(window, cwd, views)).rejects.toThrow();+ await expect(resources.read(window, cwd, ["changes", "ocr"])).rejects.toThrow();+ // Pixels asked for explicitly are not replaced by a UI-only walk.+ expect((await resources.read(window, cwd, ["changes", "full"]))!.details.status).toBe("refused");await expect(resources.read(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)), cwd, ["changes"])).rejects.toThrow(/window address/);});