Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
feat: present final browser viewport and temporal results
alexshpunt-clanker[bot]a2daaba8628d
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 2a4e4b8..f0983d2 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -5,14 +5,14 @@ import { FULL_RESOLUTION } from "./cua-image-settings.ts";import { WATCH_DEFAULTS } from "./watch.ts";import type { PixelBox } from "./pixel-diff.ts";import { ShortAddresses, SHORT_REFERENCE } from "./short-addresses.ts";-import { parseComputerViews, type ComputerViews } from "./computer-views.ts";+import { parseComputerViews } from "./computer-views.ts";import { checkedName, type ProcessHealth } from "./machine-host.ts";import { mkdir, readFile, rm, writeFile } from "node:fs/promises";import { execFile } from "node:child_process";import { promisify } from "node:util";import { fileURLToPath } from "node:url";import { basename, dirname, resolve } from "node:path";-import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./session.ts";+import type { ComputerSession, ComputerResult, ComputerMode, Action, Content } from "./session.ts";import { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";@@ -684,11 +684,11 @@ export class ComputerResources {result = await this.readNative(address, cwd, views, { ...view, capture: { ...view.capture, granularity: "ui" } }, signal);if (!result) return result;if (view.regions && result.details.status === "observed") {- try { return { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }+ try { result = { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }catch (error) { throw new ComputerRequestError(`Visual region detection failed: ${error instanceof Error ? error.message : error}`); }}if (view.aria && result.details.status === "observed" && result.details.ui) {- try { return { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }+ try { result = { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }catch (error) { throw new ComputerRequestError(`The aria snapshot could not be rendered: ${error instanceof Error ? error.message : error}`); }}if (view.changes && result.details.status === "observed") {@@ -697,12 +697,12 @@ export class ComputerResources {const earlier = latest ? await decodeFrame(frame!.png) : undefined;const diff = compareCaptures({ ui: baseline?.ui, frame: earlier }, { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined },diffContext(result.details.ui, earlier, undefined));- if (!diff) return { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } };+ if (!diff) return session.presentObservation(windowRef!, { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } }, signal);const since = baseline?.observation_id ?? frame!.observation_id;if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;// The change list replaces the full overview: only crops of the changed area are shown.const crops = latest ? await cropImages(frame!.png, latest.png, diff) : [];- return { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } };+ return session.presentObservation(windowRef!, { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } }, signal);}if (view.kind === "state" && result.details.status === "observed") {const health = await session.processHealth(target);@@ -728,7 +728,7 @@ export class ComputerResources {return { ...result, content: [...result.content, { type: "text", text: `${header}\n${listing}` }], details: { ...result.details, text_lines: lines } };}- private async readNative(address: ComputerAddress, cwd: string, views: readonly string[] | undefined, view: ComputerViews, signal?: AbortSignal): Promise<ComputerResult | undefined> {+ private async readNative(address: ComputerAddress, cwd: string, views: readonly string[] | undefined, view: ReturnType<typeof parseComputerViews>, signal?: AbortSignal): Promise<ComputerResult | undefined> {if (address.kind === "machine") {if (views?.length) throw new ComputerRequestError("A machine address has no image views. Read a returned window or display address.");if (address.part === "windows" || address.part === "apps") throw new ComputerRequestError(`Search ${address.source} with a name or regex:<pattern>.`);@@ -767,7 +767,7 @@ export class ComputerResources {this.retireWindow(address.ref, view.kind === "dom" ? "dom" : "native");if (view.kind === "dom") return this.remember({ window_ref: address.ref }, await session.observeDom(address.ref, view.options, signal));if (view.kind !== "overview") throw new ComputerRequestError("Read an overview of the window, then element views on its returned observation address.");- return this.remember({ window_ref: address.ref }, await session.observe(address.ref, signal, undefined, undefined, undefined, view.capture));+ return this.remember({ window_ref: address.ref }, await session.observe(address.ref, signal, undefined, undefined, undefined, view.capture, view.changes));}if (address.kind === "display") {if (view.kind !== "overview" || Object.keys(view.capture).some(key => key !== "maxLongEdge" && key !== "scale"))@@ -929,14 +929,17 @@ export class ComputerResources {return { source: viewportSource(viewport.window_ref, viewport.name), name: viewport.name, window_ref: viewport.window_ref, rect: clipped, image: viewportImageSize(clipped) };}/** A viewport's image cut from a full-resolution window capture, with where it lies in the window's latest image. */- private async showViewport(viewport: Viewport, clipped: PixelBox, png: Buffer, captured: ComputerResult, scale: { x: number; y: number }, note = ""): Promise<ComputerResult> {+ private async showViewport(viewport: Viewport, clipped: PixelBox, png: Buffer, captured: ComputerResult, scale: { x: number; y: number }, session: ComputerSession, signal?: AbortSignal, note = ""): Promise<ComputerResult> {const image = await viewportImage(png, clipped), details = ComputerResources.viewportDetails(viewport, clipped), size = details.image;const shown = { x: Math.round(clipped.x / scale.x), y: Math.round(clipped.y / scale.y), width: Math.round(clipped.width / scale.x), height: Math.round(clipped.height / scale.y) };const text = `${details.source} is ${shown.width}x${shown.height} at ${shown.x},${shown.y} of the window's latest image. Its image is ${size.width}x${size.height}` +(size.width === clipped.width ? ", the window's own resolution." : `: ${clipped.width}x${clipped.height} window pixels scaled down to fit ${VIEWPORT_CAP} px.`) +` Points of the computer_* input tools on ${details.source} are pixels of this image.${note}`;- return { content: [{ type: "text", text }, { type: "image", data: image.toString("base64"), mimeType: "image/png" }],- details: { ...captured.details, status: "observed", viewport: details } };+ const output: Extract<Content, { type: "image" }> = { type: "image", data: image.toString("base64"), mimeType: "image/png" };+ const presented = await session.presentViewport(viewport.window_ref, output, captured, clipped, signal);+ if (presented.details.status !== "observed") return presented;+ return { content: [{ type: "text", text }, output],+ details: { ...captured.details, ...presented.details, status: "observed", viewport: details } };}/*** Make, move or resize a viewport of a window and show it. The rectangle is in pixels of the window's latest image,@@ -961,7 +964,7 @@ export class ComputerResources {const viewport: Viewport = { window_ref: address.ref, name: address.name, rect: clipped };this.viewports.set(viewportSource(address.ref, address.name), viewport);const unseen = session.inputView(address.ref).seen ? "" : " The window had no Read with an image yet, so the rectangle was taken in its full-resolution pixels.";- return this.stamp(place, await this.showViewport(viewport, clipped, png, captured, scale, unseen));+ return this.stamp(place, await this.showViewport(viewport, clipped, png, captured, scale, session, signal, unseen));}/** Read a viewport: its image at full resolution up to VIEWPORT_CAP, and with ocr the text in it. Only looks. */private async readViewport(address: Extract<ComputerAddress, { kind: "viewport" }>, cwd: string, views: readonly string[] | undefined, signal?: AbortSignal): Promise<ComputerResult> {@@ -975,8 +978,8 @@ export class ComputerResources {if (!png) return captured;const full = await pngSize(png), clipped = clipViewport(viewport.rect, full);if (!clipped) throw new ComputerRequestError(`${viewportSource(viewport.window_ref, viewport.name)} lies outside its window now. Move it with computer_view.`);- const result = await this.showViewport(viewport, clipped, png, captured, this.imageScale(session, viewport.window_ref, full));- if (!view.ocr) return result;+ const result = await this.showViewport(viewport, clipped, png, captured, this.imageScale(session, viewport.window_ref, full), session, signal);+ if (!view.ocr || result.details.status !== "observed") return result;const image = result.content.find(block => block.type === "image")!;let lines;// The whole image as a region: Windows OCR then reads it enlarged, so small text of a small viewport reads too.diff --git a/src/driver.ts b/src/driver.tsindex ae9b8e7..08bd346 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -237,8 +237,13 @@ export class McpDriver implements Driver {throw new Error(`native_runtime_schema_missing: ${name}`);}if (this.nativeVisualsAvailable) {- const configured = CallToolResultSchema.parse(await client.callTool({ name: "set_agent_visuals", arguments: { ...this.visualSettings.current() } }, undefined, { signal, timeout: 15_000 }));- if (configured.isError) throw new Error("The driver refused its native visual configuration.");+ try {+ const configured = CallToolResultSchema.parse(await client.callTool({ name: "set_agent_visuals", arguments: { ...this.visualSettings.current() } }, undefined, { signal, timeout: 2_000 }));+ this.nativeVisualsAvailable = !configured.isError;+ } catch {+ // Presentation is optional. A renderer/configuration failure must not prevent native input.+ this.nativeVisualsAvailable = false;+ }} else await this.jumpCursor(client, signal);return client;} catch (error) {@@ -320,7 +325,9 @@ export class McpDriver implements Driver {* connection, so it has none.*/companion(): McpDriver | undefined {- return this.config.endpoint ? undefined : new McpDriver(this.config, this.version);+ if (this.config.endpoint) return undefined;+ const visuals = new VisualSettings(); visuals.update({ preset: "off" });+ return new McpDriver(this.config, this.version, visuals);}/** One cancellable native call. This method never retries input. */async call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult> {diff --git a/src/session.ts b/src/session.tsindex e3a04a1..f1fd73a 100644--- a/src/session.ts+++ b/src/session.ts@@ -640,6 +640,22 @@ export class ComputerSession {native: { capture_id: state.capture_id, agent_overlay_capture: state.agent_overlay_capture } };}+ /** A temporal cue represents one returned operation; background capture samples stay silent. */+ private async frameActivity(source: ObservationSource | undefined, image?: Content, signal?: AbortSignal, action = "frames"): Promise<void> {+ if (!source || signal?.aborted || !this.observer) return;+ const bounds = screenRect(source.image.native_bounds);+ if (source.kind === "window") {+ const entry = this.entries.get(String(source.identity.window_ref));+ if (entry && this.allowed(entry)) await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,+ { kind: "inspect" }, image, action);+ } else if (this.getMode() === "yolo") {+ const display = String(source.identity.display_id ?? "primary");+ const original = source.image.original_dimensions;+ const physical = bounds ?? (display === "primary" && original ? { x: 0, y: 0, ...original } : undefined);+ await notifyHighlight(this.observer, physical ? { display, bounds: physical, mark: { kind: "inspect" },+ ...(image?.type === "image" ? { image } : {}), action } : undefined, signal);+ }+ }/** Capture one authorized source into a private bounded resource; no frame grants input or detail authority. */captureSequence(target: { window_ref?: string; display?: string }, options: SequenceOptions, signal?: AbortSignal): Promise<ComputerResult> {return this.queue(async () => {@@ -696,6 +712,8 @@ export class ComputerSession {this.sequences.put(sequence);const metadata = sequenceMetadata(sequence);const complete = ["completed", "frame_limit"].includes(sequence.stop_reason);+ if (complete) await this.frameActivity(sequence.frames.at(-1)?.source, undefined, signal, "sequence");+ if (!permitted()) return denied();return this.result(complete ? "observed" : "refused", `Sequence ${sequence.stop_reason}: ${sequence.frames.length} frames, ${sequence.skipped_slots.length} skipped slots. No images forwarded; select frames explicitly. Read only; observe again before input.`,undefined, { sequence: metadata });} catch (error) {@@ -734,13 +752,17 @@ export class ComputerSession {layout.tiles.map((tile, i) => `Tile ${i + 1} (row ${Math.floor(i / layout.columns!) + 1}, column ${i % layout.columns! + 1}): frame ${tile.index} at ${tile.timestamp_ms.toFixed(1)} ms`).join("\n"): `Per-pixel ${rendering.as === "max" ? "maximum" : "minimum"} of ${frames.length} frames${crop} (frames ${frames.map(frame => frame.index).join(", ")}); image ${layout.width}x${layout.height}. ` +`${rendering.as === "max" ? "Bright" : "Dark"} detail from any frame shows at once, for example text revealed by a moving light.`;- return this.result("observed", `${summary}\nRead only; observe again before input.`, undefined, { selected_frames: metadata, rendered_frames: layout },- [{ type: "image", mimeType: "image/png", data: png.toString("base64") }]);+ const image: Content = { type: "image", mimeType: "image/png", data: png.toString("base64") };+ await this.frameActivity(frames.at(-1)?.source, image, signal);+ if (!permitted() || signal?.aborted) return denied();+ return this.result("observed", `${summary}\nRead only; observe again before input.`, undefined, { selected_frames: metadata, rendered_frames: layout }, [image]);}// Interleave frame labels with their images so both direct calls and Codemode preserve correspondence.const result = this.result("observed", `Selected ${frames.length} read-only sequence frames. Their indexes and measured timestamps label the image order.`, undefined,{ selected_frames: metadata });for (const frame of frames) result.content.push({ type: "text", text: `Sequence ${id} · frame ${frame.index} · ${frame.timestamp_ms.toFixed(1)} ms` }, frame.image);+ await this.frameActivity(frames.at(-1)?.source, frames.at(-1)?.image, signal);+ if (!permitted() || signal?.aborted) return denied();return result;} catch {if (!permitted()) return denied();@@ -841,6 +863,8 @@ export class ComputerSession {return this.result("refused", `The watch did not start: ${signal?.aborted ? "cancelled" : ended.stop_reason === "stopped" ? "no frame came within 30 s" : ended.stop_reason}.`);}const size = sequence.frames[0].source.image.dimensions;+ await this.frameActivity(sequence.frames[0].source, undefined, signal, "watch");+ if (!this.allowed(entry) || signal?.aborted) { watch.stop.abort(); return this.result("refused", "Watch access revoked or cancelled."); }return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +`Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +`It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,@@ -912,6 +936,8 @@ export class ComputerSession {{ sequence: sequenceMetadata(sequence), ...(rendered ? { rendered_frames: rendered } : {}),watch: { sequence_id: id, watching: watch.watching, running: false, marks: watch.marks, changed_frames: distinct, ...(written ? { files: written } : {}) } });result.content.push(...content);+ await this.frameActivity(frames.at(-1)?.source, content.find(block => block.type === "image"), undefined, "watch");+ if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") return this.result("refused", "Watch access revoked; pixels withheld.");return result;}/** Enumerate native Windows monitors in YOLO. References grant read access, never input. */@@ -1212,7 +1238,7 @@ export class ComputerSession {output: { width: state.width, height: state.height, bytes: bytes.length, mimeType: image.mimeType, hash: createHash("sha256").update(bytes).digest("hex") }, input: "none" } }, images);} catch { return this.result("refused", "Window detail failed; pixels and diagnostics withheld. Observe again."); }}- private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false): Promise<ComputerResult> {+ private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false, deferPresentation = false): Promise<ComputerResult> {validateUiCapture(options);// A native capture keeps the page's DOM snapshot: reading changes nothing, so both stay current until input.entry.windowState = undefined;@@ -1298,7 +1324,7 @@ export class ComputerSession {const extra = { ...this.addressed(entry, state.virtual_desktop), observation_id: id,source, granularity, captured: { image: hasImage, ui: hasUi } };if (!hasImage) {- highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });+ highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; data withheld.");return this.result("observed", `Fresh UI-only window observation: ${ui!.elements.length} UI elements. No pixels or image detail; current hints/tokens only.`, state, { ...extra, ...(highlight ? { highlight } : {}), ui: structuredClone(ui!) });}@@ -1316,7 +1342,7 @@ export class ComputerSession {const rendered = await annotateUi(raw, annotationUi, observed.width, observed.height, hasUi && (options.hints ?? settings.hints ?? true), undefined, annotationUi.elements,{ options: settings.guides, action: observed.actionMark });if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled during annotation; pixels withheld."); }- highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);+ highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled; pixels withheld."); }const metadata = this.imageMetadata([rendered.image], state, settings);const message = hasUi@@ -1337,9 +1363,10 @@ export class ComputerSession {return entry;}- private webFailure(entry: WindowEntry, error: unknown, acting = false): ComputerResult {+ private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal): Promise<ComputerResult> {const reason = error instanceof Error ? error.message : String(error);const uncertain = acting && error instanceof WebError && (error.timeout || error.transport);+ if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused");return this.result(uncertain ? "unknown" : "refused", uncertain? `The action may have reached the page. ${reason} Read the window with dom before deciding; never replay it blindly.`: `Playwright refused: ${reason}`, undefined, this.addressed(entry));@@ -1429,9 +1456,11 @@ export class ComputerSession {entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", action.action);+ if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");return this.result("delivered", `Playwright performed ${action.action}${action.target && action.action !== "evaluate" ? " after its actionability checks" : ""}. Check the fresh snapshot below for the application's response.`,undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });- } catch (error) { return this.webFailure(entry, error, true); }+ } catch (error) { return this.webFailure(entry, error, true, signal); }});}@@ -1448,9 +1477,11 @@ export class ComputerSession {entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;try {const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", action.action);+ if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");return this.result("delivered", `Playwright performed ${action.action}. Check the fresh snapshot below for the page's response.`,undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });- } catch (error) { return this.webFailure(entry, error, true); }+ } catch (error) { return this.webFailure(entry, error, true, signal); }});}@@ -1536,6 +1567,9 @@ export class ComputerSession {catch (error) { return this.result("refused", (error as Error).message, undefined, this.addressed(entry)); }const text = observed.texts?.get(hint) ?? [element.name, element.value].filter(item => item).join("\n");const cut = text.length >= ELEMENT_TEXT_LIMIT;+ const region = observed.frames.get(element.element_token);+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) });+ if (!this.allowed(entry)) return this.denied(ref);return this.result("observed", `Element ${hint} text from this observation: ${text.length} characters${cut ? ", cut at the 200,000-character limit" : ""}. Page it with offset/limit.`,undefined, { ...this.addressed(entry), observation_id: id, element_text: { hint, role: element.role, name: element.name, text, cut } });});@@ -1589,6 +1623,36 @@ export class ComputerSession {});}+ /** Present a post-processed observation without recapturing or changing its input authority. */+ presentObservation(ref: string, result: ComputerResult, signal?: AbortSignal): Promise<ComputerResult> {+ return this.queue(async () => {+ const entry = await this.current(ref, signal);+ if (!entry || !this.allowed(entry) || signal?.aborted || entry.observation?.id !== result.details.observation_id)+ return this.result("refused", "Observation source revoked or cancelled; data withheld.");+ const image = result.content.filter(block => block.type === "image").at(-1);+ const highlight = await this.activity(entry, { kind: "inspect" }, signal, image);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation source revoked or cancelled; data withheld.");+ return { ...result, details: { ...result.details, ...(highlight ? { highlight } : {}) } };+ });+ }+ /** Present a final viewport image without changing observation/input authority. */+ presentViewport(ref: string, image: Content, captured: ComputerResult, rect: PixelBox, signal?: AbortSignal): Promise<ComputerResult> {+ return this.queue(async () => {+ const entry = await this.current(ref, signal);+ if (!entry || !this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");+ const bounds = screenRect(captured.details.state?.window_bounds);+ const observed = entry.observation;+ const mapping = observed?.mapping;+ const sameBounds = bounds && observed?.bounds && (["x", "y", "width", "height"] as const).every(key => bounds[key] === observed.bounds![key]);+ // The earlier AX witnesses must prove the same full-resolution width and native source.+ const region = sameBounds && mapping && observed?.width && observed.height && mapping.scale * observed.width === captured.details.state?.screenshot_width && mapping.scale * observed.height === captured.details.state?.screenshot_height+ ? windowRegion({ image_id: "viewport", ...rect }, { ...mapping, scale: 1 }, bounds) : undefined;+ const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,+ region ? { kind: "detail", region } : { kind: "inspect" }, image);+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");+ return this.result("observed", "", undefined, highlight ? { highlight } : {});+ });+ }/*** A fresh observation without the human frame, for successors: UI-only by default, or with* pixels at a given overview size so they compare with an earlier capture.@@ -1821,7 +1885,7 @@ export class ComputerSession {});}/** Read fresh window parts for an existing reference. Each mode retires this window's previous observation. */- observe(ref: string, signal?: AbortSignal, query?: string, zoom?: DetailRequest, ui?: UiRequest, options?: UiCaptureOptions): Promise<ComputerResult> {+ observe(ref: string, signal?: AbortSignal, query?: string, zoom?: DetailRequest, ui?: UiRequest, options?: UiCaptureOptions, deferPresentation = false): Promise<ComputerResult> {return this.queue(async () => {if (signal?.aborted) return this.result("refused", "Cancelled before observation.");try {@@ -1835,7 +1899,7 @@ export class ComputerSession {catch (error) { return this.result("refused", (error as Error).message); }}if (zoom && !known?.detail) return this.result("refused", "Detail needs a current overview. Observe again.");- return entry ? ui ? await this.refineUi(entry, ui, signal) : zoom ? await this.windowDetail(entry, zoom, signal) : await this.capture(entry, signal, query, options) : this.result("refused", "Unknown or closed window_ref. Discover again.");+ return entry ? ui ? await this.refineUi(entry, ui, signal) : zoom ? await this.windowDetail(entry, zoom, signal) : await this.capture(entry, signal, query, options, false, deferPresentation) : this.result("refused", "Unknown or closed window_ref. Discover again.");} catch (error) { return this.failed(error, false); }});}diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.tsindex 5d5c6f4..d781c8e 100644--- a/test/agent-cursor.test.ts+++ b/test/agent-cursor.test.ts@@ -7,7 +7,7 @@ import { McpDriver } from "../src/driver.ts";const dirs: string[] = [];afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });-async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals") {+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused") {const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));dirs.push(dir);const log = path.join(dir, "calls.jsonl");@@ -39,6 +39,23 @@ test("a native visual driver receives presentation settings without forcing a onexpect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview", generation: 1 } });} finally { await driver.close(); }});+test("presentation configuration failure leaves input usable without replay", async () => {+ const { driver, calls } = await start("visuals-refused");+ try {+ expect((await driver.call("list_windows", {})).isError).toBeFalsy();+ expect((await calls()).map(call => call.name)).toEqual(["set_agent_visuals", "list_windows"]);+ } finally { await driver.close(); }+});++test("a capture companion disables its own native presentation", async () => {+ const { driver, calls } = await start("visuals"); const companion = driver.companion()!;+ try {+ await companion.call("list_windows", {});+ expect((await calls())[0]).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "off" } });+ await driver.call("list_windows", {});+ expect((await calls())[2]).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "full" } });+ } finally { await companion.close(); await driver.close(); }+});test("a refused cursor setting stops the driver before any input", async () => {const { driver, calls } = await start("refuses");await expect(driver.call("list_windows", {})).rejects.toThrow("refused the agent cursor glide setting");diff --git a/test/attach-browser.test.ts b/test/attach-browser.test.tsindex ee36758..79d2837 100644--- a/test/attach-browser.test.ts+++ b/test/attach-browser.test.ts@@ -1,12 +1,13 @@import { expect, test, vi } from "vitest";import { ComputerSession, type ComputerMode, type Driver, type DriverResult } from "../src/session.ts";import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";+import type { WindowObserver } from "../src/native-visuals.ts";// LPT-850: Playwright attaches to a running Chrome or Edge only in YOLO, binds it to the addressed window, and the// session's close detaches through the worker instead of leaving the browser to anyone else.const window = { pid: 4100, window_id: 77, title: "Attach target - Personal - Microsoft Edge", app_name: "msedge.exe", bounds: { x: 0, y: 0, width: 800, height: 600 } };-function fixture() {+function fixture(observer?: WindowObserver) {let mode: ComputerMode = "yolo";const call = vi.fn<Driver["call"]>(async (name): Promise<DriverResult> => {if (name === "list_windows") return { content: [], structuredContent: { windows: [window] } };@@ -15,11 +16,26 @@ function fixture() {const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({ url: "data:", title: "Attach target", snapshot: "- textbox \"Field\"", tabs: [], events: [] })) };const launch = vi.fn<WebLauncher>(async () => ({ browser: browser as unknown as WebBrowser,opened: { pid: window.pid, version: "155", url: "data:", title: "Attach target", via: "cdp" } }));- const session = new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }, () => mode, undefined, undefined, launch);+ const session = new ComputerSession({ call, identity: vi.fn(async () => `PiComputerUse.${"a".repeat(32)}:4100:1`), close: vi.fn(async () => {}) }, () => mode, undefined, observer, launch);const ref = async () => ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;return { session, launch, browser, ref, setMode(value: ComputerMode) { mode = value; } };}+test("Playwright activity stays source-associated without exposing text or inventing a desktop point", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = fixture(observer);+ const ref = await f.ref(); await f.session.attachBrowser(ref);+ const read = await f.session.observeDom(ref);+ expect(observer.mock.calls.at(-1)?.[0]).toMatchObject({ bounds: window.bounds, mark: { kind: "inspect" } });+ observer.mockClear();+ const result = await f.session.actDom(ref, read.details.observation_id!, { action: "fill", target: { ref: "e1" }, text: "synthetic-secret" });+ expect(result.details.status).toBe("delivered");+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "action" }, action: "fill", outcome: "delivered" });+ expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.point");+ expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.region");+ expect(JSON.stringify(observer.mock.calls)).not.toContain("synthetic-secret");+ await f.session.close();+});test("Safe mode refuses before any Playwright work", async () => {const f = fixture();const ref = await f.ref();diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjsindex cc35c99..5ab810f 100644--- a/test/fixtures/agent-cursor-driver.mjs+++ b/test/fixtures/agent-cursor-driver.mjs@@ -8,9 +8,10 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotconst [log, mode] = process.argv.slice(2);let glide = 0;const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });-server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode === "visuals" ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {appendFileSync(log, JSON.stringify({ name, arguments: args }) + "\n");+ if (name === "set_agent_visuals" && mode === "visuals-refused") return { content: [], isError: true };if (name === "set_agent_cursor_motion") {if (mode === "refuses") return { content: [{ type: "text", text: "refused" }], isError: true };if (mode === "applies") glide = args.glide_duration_ms;diff --git a/test/sequence-session.test.ts b/test/sequence-session.test.tsindex 79ea550..ddf2340 100644--- a/test/sequence-session.test.ts+++ b/test/sequence-session.test.ts@@ -1,22 +1,39 @@import sharp from "sharp";import { expect, test, vi } from "vitest";import { ComputerSession, type Driver, type ComputerMode } from "../src/session.ts";+import type { WindowObserver } from "../src/native-visuals.ts";-async function fixture() {+async function fixture(observer?: WindowObserver) {const bytes = await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer();const image = { type: "image" as const, mimeType: "image/png", data: bytes.toString("base64") };- let mode: ComputerMode = "yolo"; let lifetime = "initial";+ let mode: ComputerMode = "yolo"; let lifetime = `PiComputerUse.${"a".repeat(32)}:42:1`;const driver: Driver = {identity: vi.fn(async () => lifetime), close: vi.fn(async () => {}),call: vi.fn(async (name) => name === "list_windows" ? { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } } :{ content: [image], structuredContent: { pid: 42, window_id: 91, window_bounds: { x: 10, y: 20, width: 40, height: 30 },screenshot_width: 40, screenshot_height: 30, screenshot_original_width: 40, screenshot_original_height: 30, elements: [] } }),};- const session = new ComputerSession(driver, () => mode);+ const session = new ComputerSession(driver, () => mode, undefined, observer);const windows = await session.windows(); const ref = (windows.details.state!.windows as { window_ref: string }[])[0].window_ref;return { session, driver, ref, setMode: (value: ComputerMode) => { mode = value; }, replace: () => { lifetime = "replacement"; } };}+test("temporal playback emits a bounded completion cue and previews only explicitly returned images", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);+ const captured = await f.session.captureSequence({ window_ref: f.ref }, { duration_ms: 1 });+ expect(captured.details.status).toBe("observed");+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image");+ observer.mockClear();+ const before = vi.mocked(f.driver.call).mock.calls.length;+ const selected = await f.session.sequenceFrames(captured.details.sequence!.sequence_id, "all", undefined, { as: "sheet" });+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ image: selected.content.find(block => block.type === "image"), action: "frames" });+ expect(vi.mocked(f.driver.call).mock.calls).toHaveLength(before);+ f.setMode("safe"); observer.mockClear();+ expect((await f.session.sequenceFrames(captured.details.sequence!.sequence_id, "all")).details.status).toBe("refused");+ expect(observer).not.toHaveBeenCalled();+});test("sequence captures no context images, revokes old input, and explicit selection returns read-only frames", async () => {const f = await fixture(); const overview = await f.session.observe(f.ref);const capture = await f.session.captureSequence({ window_ref: f.ref }, { duration_ms: 1 });diff --git a/test/window-observer.test.ts b/test/window-observer.test.tsindex 6540cbc..5b96329 100644--- a/test/window-observer.test.ts+++ b/test/window-observer.test.ts@@ -2,6 +2,7 @@ import { expect, test, vi } from "vitest";import sharp from "sharp";import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";import type { WindowObserver } from "../src/native-visuals.ts";+import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";async function fixture(observer: WindowObserver) {let mode: ComputerMode = "yolo"; let marker = 1;@@ -19,7 +20,7 @@ async function fixture(observer: WindowObserver) {const created = { pid: 42, window_id: 101, app_name: "Calculator.exe", title: "Owned", bounds };windows.push(created); return { content: [], structuredContent: { pid: 42, windows: [created] } };}- if (name === "get_window_state") return { content: [png], structuredContent: { ...args, elements: [{ element_token: "root", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: 500, screenshot_height: 350, window_bounds: bounds } };+ if (name === "get_window_state") return { content: [args.max_image_dimension === 4096 ? raw : png], structuredContent: { ...args, elements: [{ element_token: "root", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: args.max_image_dimension === 4096 ? 1000 : 500, screenshot_height: args.max_image_dimension === 4096 ? 700 : 350, window_bounds: bounds } };if (name === "zoom") return { content: [jpeg], structuredContent: { width: 200, height: 100 } };if (name === "get_desktop_state") return { content: [args.max_image_dimension === 0 ? raw : png], structuredContent: { display: "primary", platform: "windows", screenshot_width: args.max_image_dimension === 0 ? 1000 : 500, screenshot_height: args.max_image_dimension === 0 ? 700 : 350, screenshot_original_width: 1000, screenshot_original_height: 700, screen_width: 1000, screen_height: 700, scale_factor: 1 } };if (name === "click" && args.x !== undefined) return { content: [{ type: "text", text: "✅ Posted click to Chromium pid 42 at (231,241) (background, no foreground swap)." }], structuredContent: { path: "post_message", effect: "unverifiable" } };@@ -35,6 +36,23 @@ async function outside(session: ComputerSession) {return (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;}+test("change reads present only images in the final returned result", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);+ const ref = await outside(f.session); const resources = new ComputerResources(async () => f.session, async () => f.session.close());+ const address = requireComputerAddress(`computer:window/${ref}`);+ await resources.read(address, process.cwd()); observer.mockClear();+ const original = f.driver.call;+ const changed = { type: "image" as const, mimeType: "image/png", data: (await sharp(Buffer.from(f.png.data, "base64"))+ .composite([{ input: { create: { width: 40, height: 30, channels: 3, background: "red" } }, left: 60, top: 60 }]).png().toBuffer()).toString("base64") };+ f.driver.call = vi.fn(async (name, args, signal) => { const reply = await original(name, args, signal);+ return name === "get_window_state" ? { ...reply, content: [changed] } : reply; });+ const result = await resources.read(address, process.cwd(), ["changes"]);+ const images = result!.content.filter(block => block.type === "image");+ expect(result!.details.status).toBe("observed"); expect(images.length).toBeGreaterThan(0);+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ image: images.at(-1) });+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image.data", changed.data);+});test("overview, requested window zoom and primary-display crop flash their own physical scopes", async () => {const observer = vi.fn<WindowObserver>(async () => ({ status: "shown" })); const f = await fixture(observer);f.setMode("safe"); const opened = await f.session.open("Calculator");@@ -136,6 +154,16 @@ test("snapshot refinement previews the final annotated image and verification emexpect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "verify" } });expect(observer.mock.calls[0][0]).not.toHaveProperty("image");});+test("a viewport previews its returned crop and marks only the proven physical region", async () => {+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" }));+ const f = await fixture(observer); const ref = await outside(f.session);+ await f.session.observe(ref); observer.mockClear();+ const resources = new ComputerResources(async () => f.session, async () => f.session.close());+ const read = await resources.view(requireComputerAddress(`computer:window/${ref}/pano`), process.cwd(), { x: 10, y: 10, width: 100, height: 70 });+ expect(read.details.status).toBe("observed");+ expect(observer).toHaveBeenCalledTimes(1);+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "detail", region: { x: 51, y: 61, width: 200, height: 140 } }, image: read.content.find(block => block.type === "image") });+});test("renderer failure does not corrupt or replay a valid desktop capture", async () => {const observer = vi.fn<WindowObserver>(async () => { throw new Error("render failure"); }); const f = await fixture(observer);const read = await f.session.observe(await outside(f.session), undefined, undefined, undefined, undefined, { hints: false });