Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: present final browser viewport and temporal results

alexshpunt-clanker[bot]a2daaba8628d
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 2a4e4b8..f0983d2 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -5,14 +5,14 @@ import { FULL_RESOLUTION } from "./cua-image-settings.ts";
import { WATCH_DEFAULTS } from "./watch.ts";
import type { PixelBox } from "./pixel-diff.ts";
import { ShortAddresses, SHORT_REFERENCE } from "./short-addresses.ts";
-import { parseComputerViews, type ComputerViews } from "./computer-views.ts";
+import { parseComputerViews } from "./computer-views.ts";
import { checkedName, type ProcessHealth } from "./machine-host.ts";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { basename, dirname, resolve } from "node:path";
-import type { ComputerSession, ComputerResult, ComputerMode, Action } from "./session.ts";
+import type { ComputerSession, ComputerResult, ComputerMode, Action, Content } from "./session.ts";
import { parseWindowManagement, type WindowManagementCommand } from "./window-management.ts";
import { validateWebAction, validateWebLaunch, type WebAction } from "./web-browser.ts";
import { compareCaptures, diffDom, diffIsEmpty, diffUi, type Capture, type StateDiff } from "./successor.ts";
@@ -684,11 +684,11 @@ export class ComputerResources {
result = await this.readNative(address, cwd, views, { ...view, capture: { ...view.capture, granularity: "ui" } }, signal);
if (!result) return result;
if (view.regions && result.details.status === "observed") {
- try { return { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }
+ try { result = { ...result, details: { ...result.details, visual_regions: await session.visualRegions(windowRef!, result.details.observation_id!, signal) } }; }
catch (error) { throw new ComputerRequestError(`Visual region detection failed: ${error instanceof Error ? error.message : error}`); }
}
if (view.aria && result.details.status === "observed" && result.details.ui) {
- try { return { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }
+ try { result = { ...result, details: { ...result.details, aria: nativeAriaSnapshot(result.details.ui) } }; }
catch (error) { throw new ComputerRequestError(`The aria snapshot could not be rendered: ${error instanceof Error ? error.message : error}`); }
}
if (view.changes && result.details.status === "observed") {
@@ -697,12 +697,12 @@ export class ComputerResources {
const earlier = latest ? await decodeFrame(frame!.png) : undefined;
const diff = compareCaptures({ ui: baseline?.ui, frame: earlier }, { ui: result.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined },
diffContext(result.details.ui, earlier, undefined));
- if (!diff) return { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } };
+ if (!diff) return session.presentObservation(windowRef!, { ...result, details: { ...result.details, changes: { reason: "No earlier observation of this window in this session to compare with, so this is the full observation." } } }, signal);
const since = baseline?.observation_id ?? frame!.observation_id;
if (diff.pixels && frame!.observation_id !== since) diff.pixels.since = frame!.observation_id;
// The change list replaces the full overview: only crops of the changed area are shown.
const crops = latest ? await cropImages(frame!.png, latest.png, diff) : [];
- return { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } };
+ return session.presentObservation(windowRef!, { ...result, content: [...result.content.filter(block => block.type !== "image"), ...crops], details: { ...result.details, changes: { since, diff } } }, signal);
}
if (view.kind === "state" && result.details.status === "observed") {
const health = await session.processHealth(target);
@@ -728,7 +728,7 @@ export class ComputerResources {
return { ...result, content: [...result.content, { type: "text", text: `${header}\n${listing}` }], details: { ...result.details, text_lines: lines } };
}
- private async readNative(address: ComputerAddress, cwd: string, views: readonly string[] | undefined, view: ComputerViews, signal?: AbortSignal): Promise<ComputerResult | undefined> {
+ private async readNative(address: ComputerAddress, cwd: string, views: readonly string[] | undefined, view: ReturnType<typeof parseComputerViews>, signal?: AbortSignal): Promise<ComputerResult | undefined> {
if (address.kind === "machine") {
if (views?.length) throw new ComputerRequestError("A machine address has no image views. Read a returned window or display address.");
if (address.part === "windows" || address.part === "apps") throw new ComputerRequestError(`Search ${address.source} with a name or regex:<pattern>.`);
@@ -767,7 +767,7 @@ export class ComputerResources {
this.retireWindow(address.ref, view.kind === "dom" ? "dom" : "native");
if (view.kind === "dom") return this.remember({ window_ref: address.ref }, await session.observeDom(address.ref, view.options, signal));
if (view.kind !== "overview") throw new ComputerRequestError("Read an overview of the window, then element views on its returned observation address.");
- return this.remember({ window_ref: address.ref }, await session.observe(address.ref, signal, undefined, undefined, undefined, view.capture));
+ return this.remember({ window_ref: address.ref }, await session.observe(address.ref, signal, undefined, undefined, undefined, view.capture, view.changes));
}
if (address.kind === "display") {
if (view.kind !== "overview" || Object.keys(view.capture).some(key => key !== "maxLongEdge" && key !== "scale"))
@@ -929,14 +929,17 @@ export class ComputerResources {
return { source: viewportSource(viewport.window_ref, viewport.name), name: viewport.name, window_ref: viewport.window_ref, rect: clipped, image: viewportImageSize(clipped) };
}
/** A viewport's image cut from a full-resolution window capture, with where it lies in the window's latest image. */
- private async showViewport(viewport: Viewport, clipped: PixelBox, png: Buffer, captured: ComputerResult, scale: { x: number; y: number }, note = ""): Promise<ComputerResult> {
+ private async showViewport(viewport: Viewport, clipped: PixelBox, png: Buffer, captured: ComputerResult, scale: { x: number; y: number }, session: ComputerSession, signal?: AbortSignal, note = ""): Promise<ComputerResult> {
const image = await viewportImage(png, clipped), details = ComputerResources.viewportDetails(viewport, clipped), size = details.image;
const shown = { x: Math.round(clipped.x / scale.x), y: Math.round(clipped.y / scale.y), width: Math.round(clipped.width / scale.x), height: Math.round(clipped.height / scale.y) };
const text = `${details.source} is ${shown.width}x${shown.height} at ${shown.x},${shown.y} of the window's latest image. Its image is ${size.width}x${size.height}` +
(size.width === clipped.width ? ", the window's own resolution." : `: ${clipped.width}x${clipped.height} window pixels scaled down to fit ${VIEWPORT_CAP} px.`) +
` Points of the computer_* input tools on ${details.source} are pixels of this image.${note}`;
- return { content: [{ type: "text", text }, { type: "image", data: image.toString("base64"), mimeType: "image/png" }],
- details: { ...captured.details, status: "observed", viewport: details } };
+ const output: Extract<Content, { type: "image" }> = { type: "image", data: image.toString("base64"), mimeType: "image/png" };
+ const presented = await session.presentViewport(viewport.window_ref, output, captured, clipped, signal);
+ if (presented.details.status !== "observed") return presented;
+ return { content: [{ type: "text", text }, output],
+ details: { ...captured.details, ...presented.details, status: "observed", viewport: details } };
}
/**
* Make, move or resize a viewport of a window and show it. The rectangle is in pixels of the window's latest image,
@@ -961,7 +964,7 @@ export class ComputerResources {
const viewport: Viewport = { window_ref: address.ref, name: address.name, rect: clipped };
this.viewports.set(viewportSource(address.ref, address.name), viewport);
const unseen = session.inputView(address.ref).seen ? "" : " The window had no Read with an image yet, so the rectangle was taken in its full-resolution pixels.";
- return this.stamp(place, await this.showViewport(viewport, clipped, png, captured, scale, unseen));
+ return this.stamp(place, await this.showViewport(viewport, clipped, png, captured, scale, session, signal, unseen));
}
/** Read a viewport: its image at full resolution up to VIEWPORT_CAP, and with ocr the text in it. Only looks. */
private async readViewport(address: Extract<ComputerAddress, { kind: "viewport" }>, cwd: string, views: readonly string[] | undefined, signal?: AbortSignal): Promise<ComputerResult> {
@@ -975,8 +978,8 @@ export class ComputerResources {
if (!png) return captured;
const full = await pngSize(png), clipped = clipViewport(viewport.rect, full);
if (!clipped) throw new ComputerRequestError(`${viewportSource(viewport.window_ref, viewport.name)} lies outside its window now. Move it with computer_view.`);
- const result = await this.showViewport(viewport, clipped, png, captured, this.imageScale(session, viewport.window_ref, full));
- if (!view.ocr) return result;
+ const result = await this.showViewport(viewport, clipped, png, captured, this.imageScale(session, viewport.window_ref, full), session, signal);
+ if (!view.ocr || result.details.status !== "observed") return result;
const image = result.content.find(block => block.type === "image")!;
let lines;
// The whole image as a region: Windows OCR then reads it enlarged, so small text of a small viewport reads too.
diff --git a/src/driver.ts b/src/driver.ts
index ae9b8e7..08bd346 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -237,8 +237,13 @@ export class McpDriver implements Driver {
throw new Error(`native_runtime_schema_missing: ${name}`);
}
if (this.nativeVisualsAvailable) {
- const configured = CallToolResultSchema.parse(await client.callTool({ name: "set_agent_visuals", arguments: { ...this.visualSettings.current() } }, undefined, { signal, timeout: 15_000 }));
- if (configured.isError) throw new Error("The driver refused its native visual configuration.");
+ try {
+ const configured = CallToolResultSchema.parse(await client.callTool({ name: "set_agent_visuals", arguments: { ...this.visualSettings.current() } }, undefined, { signal, timeout: 2_000 }));
+ this.nativeVisualsAvailable = !configured.isError;
+ } catch {
+ // Presentation is optional. A renderer/configuration failure must not prevent native input.
+ this.nativeVisualsAvailable = false;
+ }
} else await this.jumpCursor(client, signal);
return client;
} catch (error) {
@@ -320,7 +325,9 @@ export class McpDriver implements Driver {
* connection, so it has none.
*/
companion(): McpDriver | undefined {
- return this.config.endpoint ? undefined : new McpDriver(this.config, this.version);
+ if (this.config.endpoint) return undefined;
+ const visuals = new VisualSettings(); visuals.update({ preset: "off" });
+ return new McpDriver(this.config, this.version, visuals);
}
/** One cancellable native call. This method never retries input. */
async call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult> {
diff --git a/src/session.ts b/src/session.ts
index e3a04a1..f1fd73a 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -640,6 +640,22 @@ export class ComputerSession {
native: { capture_id: state.capture_id, agent_overlay_capture: state.agent_overlay_capture } };
}
+ /** A temporal cue represents one returned operation; background capture samples stay silent. */
+ private async frameActivity(source: ObservationSource | undefined, image?: Content, signal?: AbortSignal, action = "frames"): Promise<void> {
+ if (!source || signal?.aborted || !this.observer) return;
+ const bounds = screenRect(source.image.native_bounds);
+ if (source.kind === "window") {
+ const entry = this.entries.get(String(source.identity.window_ref));
+ if (entry && this.allowed(entry)) await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,
+ { kind: "inspect" }, image, action);
+ } else if (this.getMode() === "yolo") {
+ const display = String(source.identity.display_id ?? "primary");
+ const original = source.image.original_dimensions;
+ const physical = bounds ?? (display === "primary" && original ? { x: 0, y: 0, ...original } : undefined);
+ await notifyHighlight(this.observer, physical ? { display, bounds: physical, mark: { kind: "inspect" },
+ ...(image?.type === "image" ? { image } : {}), action } : undefined, signal);
+ }
+ }
/** Capture one authorized source into a private bounded resource; no frame grants input or detail authority. */
captureSequence(target: { window_ref?: string; display?: string }, options: SequenceOptions, signal?: AbortSignal): Promise<ComputerResult> {
return this.queue(async () => {
@@ -696,6 +712,8 @@ export class ComputerSession {
this.sequences.put(sequence);
const metadata = sequenceMetadata(sequence);
const complete = ["completed", "frame_limit"].includes(sequence.stop_reason);
+ if (complete) await this.frameActivity(sequence.frames.at(-1)?.source, undefined, signal, "sequence");
+ if (!permitted()) return denied();
return this.result(complete ? "observed" : "refused", `Sequence ${sequence.stop_reason}: ${sequence.frames.length} frames, ${sequence.skipped_slots.length} skipped slots. No images forwarded; select frames explicitly. Read only; observe again before input.`,
undefined, { sequence: metadata });
} catch (error) {
@@ -734,13 +752,17 @@ export class ComputerSession {
layout.tiles.map((tile, i) => `Tile ${i + 1} (row ${Math.floor(i / layout.columns!) + 1}, column ${i % layout.columns! + 1}): frame ${tile.index} at ${tile.timestamp_ms.toFixed(1)} ms`).join("\n")
: `Per-pixel ${rendering.as === "max" ? "maximum" : "minimum"} of ${frames.length} frames${crop} (frames ${frames.map(frame => frame.index).join(", ")}); image ${layout.width}x${layout.height}. ` +
`${rendering.as === "max" ? "Bright" : "Dark"} detail from any frame shows at once, for example text revealed by a moving light.`;
- return this.result("observed", `${summary}\nRead only; observe again before input.`, undefined, { selected_frames: metadata, rendered_frames: layout },
- [{ type: "image", mimeType: "image/png", data: png.toString("base64") }]);
+ const image: Content = { type: "image", mimeType: "image/png", data: png.toString("base64") };
+ await this.frameActivity(frames.at(-1)?.source, image, signal);
+ if (!permitted() || signal?.aborted) return denied();
+ return this.result("observed", `${summary}\nRead only; observe again before input.`, undefined, { selected_frames: metadata, rendered_frames: layout }, [image]);
}
// Interleave frame labels with their images so both direct calls and Codemode preserve correspondence.
const result = this.result("observed", `Selected ${frames.length} read-only sequence frames. Their indexes and measured timestamps label the image order.`, undefined,
{ selected_frames: metadata });
for (const frame of frames) result.content.push({ type: "text", text: `Sequence ${id} · frame ${frame.index} · ${frame.timestamp_ms.toFixed(1)} ms` }, frame.image);
+ await this.frameActivity(frames.at(-1)?.source, frames.at(-1)?.image, signal);
+ if (!permitted() || signal?.aborted) return denied();
return result;
} catch {
if (!permitted()) return denied();
@@ -841,6 +863,8 @@ export class ComputerSession {
return this.result("refused", `The watch did not start: ${signal?.aborted ? "cancelled" : ended.stop_reason === "stopped" ? "no frame came within 30 s" : ended.stop_reason}.`);
}
const size = sequence.frames[0].source.image.dimensions;
+ await this.frameActivity(sequence.frames[0].source, undefined, signal, "watch");
+ if (!this.allowed(entry) || signal?.aborted) { watch.stop.abort(); return this.result("refused", "Watch access revoked or cancelled."); }
return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +
`Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +
`It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,
@@ -912,6 +936,8 @@ export class ComputerSession {
{ sequence: sequenceMetadata(sequence), ...(rendered ? { rendered_frames: rendered } : {}),
watch: { sequence_id: id, watching: watch.watching, running: false, marks: watch.marks, changed_frames: distinct, ...(written ? { files: written } : {}) } });
result.content.push(...content);
+ await this.frameActivity(frames.at(-1)?.source, content.find(block => block.type === "image"), undefined, "watch");
+ if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") return this.result("refused", "Watch access revoked; pixels withheld.");
return result;
}
/** Enumerate native Windows monitors in YOLO. References grant read access, never input. */
@@ -1212,7 +1238,7 @@ export class ComputerSession {
output: { width: state.width, height: state.height, bytes: bytes.length, mimeType: image.mimeType, hash: createHash("sha256").update(bytes).digest("hex") }, input: "none" } }, images);
} catch { return this.result("refused", "Window detail failed; pixels and diagnostics withheld. Observe again."); }
}
- private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false): Promise<ComputerResult> {
+ private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false, deferPresentation = false): Promise<ComputerResult> {
validateUiCapture(options);
// A native capture keeps the page's DOM snapshot: reading changes nothing, so both stay current until input.
entry.windowState = undefined;
@@ -1298,7 +1324,7 @@ export class ComputerSession {
const extra = { ...this.addressed(entry, state.virtual_desktop), observation_id: id,
source, granularity, captured: { image: hasImage, ui: hasUi } };
if (!hasImage) {
- highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });
+ highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });
if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; data withheld.");
return this.result("observed", `Fresh UI-only window observation: ${ui!.elements.length} UI elements. No pixels or image detail; current hints/tokens only.`, state, { ...extra, ...(highlight ? { highlight } : {}), ui: structuredClone(ui!) });
}
@@ -1316,7 +1342,7 @@ export class ComputerSession {
const rendered = await annotateUi(raw, annotationUi, observed.width, observed.height, hasUi && (options.hints ?? settings.hints ?? true), undefined, annotationUi.elements,
{ options: settings.guides, action: observed.actionMark });
if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled during annotation; pixels withheld."); }
- highlight = quiet ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);
+ highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);
if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled; pixels withheld."); }
const metadata = this.imageMetadata([rendered.image], state, settings);
const message = hasUi
@@ -1337,9 +1363,10 @@ export class ComputerSession {
return entry;
}
- private webFailure(entry: WindowEntry, error: unknown, acting = false): ComputerResult {
+ private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal): Promise<ComputerResult> {
const reason = error instanceof Error ? error.message : String(error);
const uncertain = acting && error instanceof WebError && (error.timeout || error.transport);
+ if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused");
return this.result(uncertain ? "unknown" : "refused", uncertain
? `The action may have reached the page. ${reason} Read the window with dom before deciding; never replay it blindly.`
: `Playwright refused: ${reason}`, undefined, this.addressed(entry));
@@ -1429,9 +1456,11 @@ export class ComputerSession {
entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
try {
const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", action.action);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
return this.result("delivered", `Playwright performed ${action.action}${action.target && action.action !== "evaluate" ? " after its actionability checks" : ""}. Check the fresh snapshot below for the application's response.`,
undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
- } catch (error) { return this.webFailure(entry, error, true); }
+ } catch (error) { return this.webFailure(entry, error, true, signal); }
});
}
@@ -1448,9 +1477,11 @@ export class ComputerSession {
entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
try {
const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
+ await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", action.action);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
return this.result("delivered", `Playwright performed ${action.action}. Check the fresh snapshot below for the page's response.`,
undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
- } catch (error) { return this.webFailure(entry, error, true); }
+ } catch (error) { return this.webFailure(entry, error, true, signal); }
});
}
@@ -1536,6 +1567,9 @@ export class ComputerSession {
catch (error) { return this.result("refused", (error as Error).message, undefined, this.addressed(entry)); }
const text = observed.texts?.get(hint) ?? [element.name, element.value].filter(item => item).join("\n");
const cut = text.length >= ELEMENT_TEXT_LIMIT;
+ const region = observed.frames.get(element.element_token);
+ await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) });
+ if (!this.allowed(entry)) return this.denied(ref);
return this.result("observed", `Element ${hint} text from this observation: ${text.length} characters${cut ? ", cut at the 200,000-character limit" : ""}. Page it with offset/limit.`,
undefined, { ...this.addressed(entry), observation_id: id, element_text: { hint, role: element.role, name: element.name, text, cut } });
});
@@ -1589,6 +1623,36 @@ export class ComputerSession {
});
}
+ /** Present a post-processed observation without recapturing or changing its input authority. */
+ presentObservation(ref: string, result: ComputerResult, signal?: AbortSignal): Promise<ComputerResult> {
+ return this.queue(async () => {
+ const entry = await this.current(ref, signal);
+ if (!entry || !this.allowed(entry) || signal?.aborted || entry.observation?.id !== result.details.observation_id)
+ return this.result("refused", "Observation source revoked or cancelled; data withheld.");
+ const image = result.content.filter(block => block.type === "image").at(-1);
+ const highlight = await this.activity(entry, { kind: "inspect" }, signal, image);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation source revoked or cancelled; data withheld.");
+ return { ...result, details: { ...result.details, ...(highlight ? { highlight } : {}) } };
+ });
+ }
+ /** Present a final viewport image without changing observation/input authority. */
+ presentViewport(ref: string, image: Content, captured: ComputerResult, rect: PixelBox, signal?: AbortSignal): Promise<ComputerResult> {
+ return this.queue(async () => {
+ const entry = await this.current(ref, signal);
+ if (!entry || !this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");
+ const bounds = screenRect(captured.details.state?.window_bounds);
+ const observed = entry.observation;
+ const mapping = observed?.mapping;
+ const sameBounds = bounds && observed?.bounds && (["x", "y", "width", "height"] as const).every(key => bounds[key] === observed.bounds![key]);
+ // The earlier AX witnesses must prove the same full-resolution width and native source.
+ const region = sameBounds && mapping && observed?.width && observed.height && mapping.scale * observed.width === captured.details.state?.screenshot_width && mapping.scale * observed.height === captured.details.state?.screenshot_height
+ ? windowRegion({ image_id: "viewport", ...rect }, { ...mapping, scale: 1 }, bounds) : undefined;
+ const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,
+ region ? { kind: "detail", region } : { kind: "inspect" }, image);
+ if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");
+ return this.result("observed", "", undefined, highlight ? { highlight } : {});
+ });
+ }
/**
* A fresh observation without the human frame, for successors: UI-only by default, or with
* pixels at a given overview size so they compare with an earlier capture.
@@ -1821,7 +1885,7 @@ export class ComputerSession {
});
}
/** Read fresh window parts for an existing reference. Each mode retires this window's previous observation. */
- observe(ref: string, signal?: AbortSignal, query?: string, zoom?: DetailRequest, ui?: UiRequest, options?: UiCaptureOptions): Promise<ComputerResult> {
+ observe(ref: string, signal?: AbortSignal, query?: string, zoom?: DetailRequest, ui?: UiRequest, options?: UiCaptureOptions, deferPresentation = false): Promise<ComputerResult> {
return this.queue(async () => {
if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
try {
@@ -1835,7 +1899,7 @@ export class ComputerSession {
catch (error) { return this.result("refused", (error as Error).message); }
}
if (zoom && !known?.detail) return this.result("refused", "Detail needs a current overview. Observe again.");
- return entry ? ui ? await this.refineUi(entry, ui, signal) : zoom ? await this.windowDetail(entry, zoom, signal) : await this.capture(entry, signal, query, options) : this.result("refused", "Unknown or closed window_ref. Discover again.");
+ return entry ? ui ? await this.refineUi(entry, ui, signal) : zoom ? await this.windowDetail(entry, zoom, signal) : await this.capture(entry, signal, query, options, false, deferPresentation) : this.result("refused", "Unknown or closed window_ref. Discover again.");
} catch (error) { return this.failed(error, false); }
});
}
diff --git a/test/agent-cursor.test.ts b/test/agent-cursor.test.ts
index 5d5c6f4..d781c8e 100644
--- a/test/agent-cursor.test.ts
+++ b/test/agent-cursor.test.ts
@@ -7,7 +7,7 @@ import { McpDriver } from "../src/driver.ts";
const dirs: string[] = [];
afterEach(async () => { await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); });
-async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals") {
+async function start(mode: "applies" | "refuses" | "ignores" | "no-overlay" | "visuals" | "visuals-refused") {
const dir = await mkdtemp(path.join(tmpdir(), "agent-cursor-"));
dirs.push(dir);
const log = path.join(dir, "calls.jsonl");
@@ -39,6 +39,23 @@ test("a native visual driver receives presentation settings without forcing a on
expect((await calls()).at(-1)).toMatchObject({ name: "present_agent_activity", arguments: { kind: "overview", generation: 1 } });
} finally { await driver.close(); }
});
+test("presentation configuration failure leaves input usable without replay", async () => {
+ const { driver, calls } = await start("visuals-refused");
+ try {
+ expect((await driver.call("list_windows", {})).isError).toBeFalsy();
+ expect((await calls()).map(call => call.name)).toEqual(["set_agent_visuals", "list_windows"]);
+ } finally { await driver.close(); }
+});
+
+test("a capture companion disables its own native presentation", async () => {
+ const { driver, calls } = await start("visuals"); const companion = driver.companion()!;
+ try {
+ await companion.call("list_windows", {});
+ expect((await calls())[0]).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "off" } });
+ await driver.call("list_windows", {});
+ expect((await calls())[2]).toMatchObject({ name: "set_agent_visuals", arguments: { preset: "full" } });
+ } finally { await companion.close(); await driver.close(); }
+});
test("a refused cursor setting stops the driver before any input", async () => {
const { driver, calls } = await start("refuses");
await expect(driver.call("list_windows", {})).rejects.toThrow("refused the agent cursor glide setting");
diff --git a/test/attach-browser.test.ts b/test/attach-browser.test.ts
index ee36758..79d2837 100644
--- a/test/attach-browser.test.ts
+++ b/test/attach-browser.test.ts
@@ -1,12 +1,13 @@
import { expect, test, vi } from "vitest";
import { ComputerSession, type ComputerMode, type Driver, type DriverResult } from "../src/session.ts";
import type { WebBrowser, WebLauncher } from "../src/web-browser.ts";
+import type { WindowObserver } from "../src/native-visuals.ts";
// LPT-850: Playwright attaches to a running Chrome or Edge only in YOLO, binds it to the addressed window, and the
// session's close detaches through the worker instead of leaving the browser to anyone else.
const window = { pid: 4100, window_id: 77, title: "Attach target - Personal - Microsoft Edge", app_name: "msedge.exe", bounds: { x: 0, y: 0, width: 800, height: 600 } };
-function fixture() {
+function fixture(observer?: WindowObserver) {
let mode: ComputerMode = "yolo";
const call = vi.fn<Driver["call"]>(async (name): Promise<DriverResult> => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [window] } };
@@ -15,11 +16,26 @@ function fixture() {
const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({ url: "data:", title: "Attach target", snapshot: "- textbox \"Field\"", tabs: [], events: [] })) };
const launch = vi.fn<WebLauncher>(async () => ({ browser: browser as unknown as WebBrowser,
opened: { pid: window.pid, version: "155", url: "data:", title: "Attach target", via: "cdp" } }));
- const session = new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }, () => mode, undefined, undefined, launch);
+ const session = new ComputerSession({ call, identity: vi.fn(async () => `PiComputerUse.${"a".repeat(32)}:4100:1`), close: vi.fn(async () => {}) }, () => mode, undefined, observer, launch);
const ref = async () => ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
return { session, launch, browser, ref, setMode(value: ComputerMode) { mode = value; } };
}
+test("Playwright activity stays source-associated without exposing text or inventing a desktop point", async () => {
+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = fixture(observer);
+ const ref = await f.ref(); await f.session.attachBrowser(ref);
+ const read = await f.session.observeDom(ref);
+ expect(observer.mock.calls.at(-1)?.[0]).toMatchObject({ bounds: window.bounds, mark: { kind: "inspect" } });
+ observer.mockClear();
+ const result = await f.session.actDom(ref, read.details.observation_id!, { action: "fill", target: { ref: "e1" }, text: "synthetic-secret" });
+ expect(result.details.status).toBe("delivered");
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "action" }, action: "fill", outcome: "delivered" });
+ expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.point");
+ expect(observer.mock.calls[0][0]).not.toHaveProperty("mark.region");
+ expect(JSON.stringify(observer.mock.calls)).not.toContain("synthetic-secret");
+ await f.session.close();
+});
test("Safe mode refuses before any Playwright work", async () => {
const f = fixture();
const ref = await f.ref();
diff --git a/test/fixtures/agent-cursor-driver.mjs b/test/fixtures/agent-cursor-driver.mjs
index cc35c99..5ab810f 100644
--- a/test/fixtures/agent-cursor-driver.mjs
+++ b/test/fixtures/agent-cursor-driver.mjs
@@ -8,9 +8,10 @@ import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprot
const [log, mode] = process.argv.slice(2);
let glide = 0;
const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });
-server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode === "visuals" ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: mode.startsWith("visuals") ? ["present_agent_activity", "set_agent_visuals"].map(name => ({ name, inputSchema: { type: "object" } })) : [] }));
server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {
appendFileSync(log, JSON.stringify({ name, arguments: args }) + "\n");
+ if (name === "set_agent_visuals" && mode === "visuals-refused") return { content: [], isError: true };
if (name === "set_agent_cursor_motion") {
if (mode === "refuses") return { content: [{ type: "text", text: "refused" }], isError: true };
if (mode === "applies") glide = args.glide_duration_ms;
diff --git a/test/sequence-session.test.ts b/test/sequence-session.test.ts
index 79ea550..ddf2340 100644
--- a/test/sequence-session.test.ts
+++ b/test/sequence-session.test.ts
@@ -1,22 +1,39 @@
import sharp from "sharp";
import { expect, test, vi } from "vitest";
import { ComputerSession, type Driver, type ComputerMode } from "../src/session.ts";
+import type { WindowObserver } from "../src/native-visuals.ts";
-async function fixture() {
+async function fixture(observer?: WindowObserver) {
const bytes = await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer();
const image = { type: "image" as const, mimeType: "image/png", data: bytes.toString("base64") };
- let mode: ComputerMode = "yolo"; let lifetime = "initial";
+ let mode: ComputerMode = "yolo"; let lifetime = `PiComputerUse.${"a".repeat(32)}:42:1`;
const driver: Driver = {
identity: vi.fn(async () => lifetime), close: vi.fn(async () => {}),
call: vi.fn(async (name) => name === "list_windows" ? { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } } :
{ content: [image], structuredContent: { pid: 42, window_id: 91, window_bounds: { x: 10, y: 20, width: 40, height: 30 },
screenshot_width: 40, screenshot_height: 30, screenshot_original_width: 40, screenshot_original_height: 30, elements: [] } }),
};
- const session = new ComputerSession(driver, () => mode);
+ const session = new ComputerSession(driver, () => mode, undefined, observer);
const windows = await session.windows(); const ref = (windows.details.state!.windows as { window_ref: string }[])[0].window_ref;
return { session, driver, ref, setMode: (value: ComputerMode) => { mode = value; }, replace: () => { lifetime = "replacement"; } };
}
+test("temporal playback emits a bounded completion cue and previews only explicitly returned images", async () => {
+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);
+ const captured = await f.session.captureSequence({ window_ref: f.ref }, { duration_ms: 1 });
+ expect(captured.details.status).toBe("observed");
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image");
+ observer.mockClear();
+ const before = vi.mocked(f.driver.call).mock.calls.length;
+ const selected = await f.session.sequenceFrames(captured.details.sequence!.sequence_id, "all", undefined, { as: "sheet" });
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ image: selected.content.find(block => block.type === "image"), action: "frames" });
+ expect(vi.mocked(f.driver.call).mock.calls).toHaveLength(before);
+ f.setMode("safe"); observer.mockClear();
+ expect((await f.session.sequenceFrames(captured.details.sequence!.sequence_id, "all")).details.status).toBe("refused");
+ expect(observer).not.toHaveBeenCalled();
+});
test("sequence captures no context images, revokes old input, and explicit selection returns read-only frames", async () => {
const f = await fixture(); const overview = await f.session.observe(f.ref);
const capture = await f.session.captureSequence({ window_ref: f.ref }, { duration_ms: 1 });
diff --git a/test/window-observer.test.ts b/test/window-observer.test.ts
index 6540cbc..5b96329 100644
--- a/test/window-observer.test.ts
+++ b/test/window-observer.test.ts
@@ -2,6 +2,7 @@ import { expect, test, vi } from "vitest";
import sharp from "sharp";
import { ComputerSession, type ComputerMode, type Driver } from "../src/session.ts";
import type { WindowObserver } from "../src/native-visuals.ts";
+import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
async function fixture(observer: WindowObserver) {
let mode: ComputerMode = "yolo"; let marker = 1;
@@ -19,7 +20,7 @@ async function fixture(observer: WindowObserver) {
const created = { pid: 42, window_id: 101, app_name: "Calculator.exe", title: "Owned", bounds };
windows.push(created); return { content: [], structuredContent: { pid: 42, windows: [created] } };
}
- if (name === "get_window_state") return { content: [png], structuredContent: { ...args, elements: [{ element_token: "root", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: 500, screenshot_height: 350, window_bounds: bounds } };
+ if (name === "get_window_state") return { content: [args.max_image_dimension === 4096 ? raw : png], structuredContent: { ...args, elements: [{ element_token: "root", frame: { x: 30, y: 40, w: 1000, h: 700 }, screenshot_frame: { x: -1, y: -1, w: 500, h: 350 } }, { element_token: "button", frame: { x: 230, y: 240, w: 120, h: 60 }, screenshot_frame: { x: 100, y: 100, w: 60, h: 30 } }], screenshot_width: args.max_image_dimension === 4096 ? 1000 : 500, screenshot_height: args.max_image_dimension === 4096 ? 700 : 350, window_bounds: bounds } };
if (name === "zoom") return { content: [jpeg], structuredContent: { width: 200, height: 100 } };
if (name === "get_desktop_state") return { content: [args.max_image_dimension === 0 ? raw : png], structuredContent: { display: "primary", platform: "windows", screenshot_width: args.max_image_dimension === 0 ? 1000 : 500, screenshot_height: args.max_image_dimension === 0 ? 700 : 350, screenshot_original_width: 1000, screenshot_original_height: 700, screen_width: 1000, screen_height: 700, scale_factor: 1 } };
if (name === "click" && args.x !== undefined) return { content: [{ type: "text", text: "✅ Posted click to Chromium pid 42 at (231,241) (background, no foreground swap)." }], structuredContent: { path: "post_message", effect: "unverifiable" } };
@@ -35,6 +36,23 @@ async function outside(session: ComputerSession) {
return (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
}
+test("change reads present only images in the final returned result", async () => {
+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" })); const f = await fixture(observer);
+ const ref = await outside(f.session); const resources = new ComputerResources(async () => f.session, async () => f.session.close());
+ const address = requireComputerAddress(`computer:window/${ref}`);
+ await resources.read(address, process.cwd()); observer.mockClear();
+ const original = f.driver.call;
+ const changed = { type: "image" as const, mimeType: "image/png", data: (await sharp(Buffer.from(f.png.data, "base64"))
+ .composite([{ input: { create: { width: 40, height: 30, channels: 3, background: "red" } }, left: 60, top: 60 }]).png().toBuffer()).toString("base64") };
+ f.driver.call = vi.fn(async (name, args, signal) => { const reply = await original(name, args, signal);
+ return name === "get_window_state" ? { ...reply, content: [changed] } : reply; });
+ const result = await resources.read(address, process.cwd(), ["changes"]);
+ const images = result!.content.filter(block => block.type === "image");
+ expect(result!.details.status).toBe("observed"); expect(images.length).toBeGreaterThan(0);
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ image: images.at(-1) });
+ expect(observer.mock.calls[0][0]).not.toHaveProperty("image.data", changed.data);
+});
test("overview, requested window zoom and primary-display crop flash their own physical scopes", async () => {
const observer = vi.fn<WindowObserver>(async () => ({ status: "shown" })); const f = await fixture(observer);
f.setMode("safe"); const opened = await f.session.open("Calculator");
@@ -136,6 +154,16 @@ test("snapshot refinement previews the final annotated image and verification em
expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "verify" } });
expect(observer.mock.calls[0][0]).not.toHaveProperty("image");
});
+test("a viewport previews its returned crop and marks only the proven physical region", async () => {
+ const observer = vi.fn<WindowObserver>(async () => ({ status: "queued" }));
+ const f = await fixture(observer); const ref = await outside(f.session);
+ await f.session.observe(ref); observer.mockClear();
+ const resources = new ComputerResources(async () => f.session, async () => f.session.close());
+ const read = await resources.view(requireComputerAddress(`computer:window/${ref}/pano`), process.cwd(), { x: 10, y: 10, width: 100, height: 70 });
+ expect(read.details.status).toBe("observed");
+ expect(observer).toHaveBeenCalledTimes(1);
+ expect(observer.mock.calls[0][0]).toMatchObject({ mark: { kind: "detail", region: { x: 51, y: 61, width: 200, height: 140 } }, image: read.content.find(block => block.type === "image") });
+});
test("renderer failure does not corrupt or replay a valid desktop capture", async () => {
const observer = vi.fn<WindowObserver>(async () => { throw new Error("render failure"); }); const f = await fixture(observer);
const read = await f.session.observe(await outside(f.session), undefined, undefined, undefined, undefined, { hints: false });