Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

feat: named viewports of a window with computer_view (LPT-931)

alexshpunt-clanker[bot]758537c2e853
diff --git a/src/computer-address.ts b/src/computer-address.ts
index 76bec2b..13bf613 100644
--- a/src/computer-address.ts
+++ b/src/computer-address.ts
@@ -9,7 +9,12 @@ export type ComputerAddress =
/** A display of one machine: `primary`, or a reference listed by that machine's displays. */
| { kind: "display"; source: string; machine: string; ref: string }
/** `hint` names one element of the observation for an internal input step; parsed addresses never carry it. */
- | { kind: "observation"; source: string; ref: string; hint?: string };
+ | { kind: "observation"; source: string; ref: string; hint?: string }
+ /** A named rectangle of a window: `ref` is the window's reference, `name` the viewport's name in that window. */
+ | { kind: "viewport"; source: string; ref: string; name: string };
+
+/** A name the agent gives a window (computer:<name>) or a viewport of a window (computer:<window>/<name>). */
+export const NAME = /^[a-z][a-z0-9_-]{0,31}$/;
const references = new Set(["app", "window", "observation", "sequence", "sandbox", "machine"]);
/** Words a long address starts with; machine names and window names never take them. */
@@ -40,6 +45,10 @@ export function parseComputerAddress(source: string): ComputerAddress | undefine
if (query !== undefined) return invalid();
return { kind: "display", source, machine: reference(segments[1]), ref: reference(segments[3]) };
}
+ if (kind === "window" && segments.length === 3) {
+ if (query !== undefined || !NAME.test(segments[2])) return invalid();
+ return { kind: "viewport", source, ref: reference(segments[1]), name: segments[2] };
+ }
const machinePart = kind === "machine" && segments.length === 3 ? segments[2] : undefined;
if (!references.has(kind) || (segments.length !== 2 && !(machinePart && parts.has(machinePart)))) return invalid();
const ref = reference(segments[1]);
@@ -55,6 +64,13 @@ export function computerSource(kind: "app" | "window" | "observation" | "sequenc
return source;
}
+/** The long address of a window's viewport. */
+export function viewportSource(window: string, name: string): string {
+ const source = `${computerSource("window", window)}/${name}`;
+ parseComputerAddress(source);
+ return source;
+}
+
/** The address of one machine's display: `primary` or a reference its displays listing issued. */
export function displaySource(machine: string, ref: string): string {
const source = `computer:machine/${encodeURIComponent(machine)}/display/${encodeURIComponent(ref)}`;
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 9d728dc..70e3db7 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -1,5 +1,8 @@
import { randomUUID } from "node:crypto";
-import { parseComputerAddress, computerSource, displaySource, ADDRESS_KINDS, type ComputerAddress } from "./computer-address.ts";
+import { parseComputerAddress, computerSource, displaySource, viewportSource, ADDRESS_KINDS, NAME, type ComputerAddress } from "./computer-address.ts";
+import { clipViewport, pngSize, viewportImage, viewportImageSize, viewportPoint, viewportRect, windowPoint, VIEWPORT_CAP, type Viewport } from "./viewport.ts";
+import { FULL_RESOLUTION } from "./cua-image-settings.ts";
+import type { PixelBox } from "./pixel-diff.ts";
import { ShortAddresses, SHORT_REFERENCE } from "./short-addresses.ts";
import { parseComputerViews, type ComputerViews } from "./computer-views.ts";
import { checkedName, type ProcessHealth } from "./machine-host.ts";
@@ -122,7 +125,9 @@ interface FollowUp { successor: boolean; explicit: boolean; frame?: WindowFrame;
/** A Playwright screenshot of the page before a page action, in CSS pixels, for its pixel change list. */
page?: Buffer;
/** The window's session, resolved before the action: a failed action can make the session forget the window. */
- session?: ComputerSession }
+ session?: ComputerSession;
+ /** The viewport the input went to, in the window's full-resolution pixels: the pixel comparison covers only it. */
+ viewport?: PixelBox }
/** How long a native successor keeps looking for a change before reporting that nothing changed. */
const SETTLE_MS = 300;
/**
@@ -303,8 +308,6 @@ function object(value: unknown): Record<string, unknown> {
if (!value || typeof value !== "object" || Array.isArray(value)) throw new ComputerRequestError("Use a JSON command object.");
return value as Record<string, unknown>;
}
-/** A window name the agent gives: computer:<name>. */
-const NAME = /^[a-z][a-z0-9_-]{0,31}$/;
function only(value: Record<string, unknown>, fields: readonly string[]): void {
if (Object.keys(value).some(key => !fields.includes(key))) throw new ComputerRequestError("Command contains unsupported fields.");
}
@@ -329,6 +332,8 @@ export class ComputerResources {
private readonly names = new Map<string, { window_ref: string; title: string }>();
/** Window titles from the latest window listings, for naming by title and for listing names. */
private readonly titles = new Map<string, string>();
+ /** Viewports the agent made, by their long address (computer:window/<ref>/<name>). */
+ private readonly viewports = new Map<string, Viewport>();
/** This machine's session once started; reference checks need it synchronously. */
private local?: ComputerSession;
@@ -390,11 +395,21 @@ export class ComputerResources {
}
return requireComputerAddress(computerSource("window", entry.window_ref));
}
- if (segments.length === 2 && !ADDRESS_KINDS.has(segments[0]) && SHORT_REFERENCE.test(segments[1])) {
+ // A viewport of a named window: computer:game/pano. Window names never take a machine's name.
+ if (segments.length === 2 && this.names.has(segments[0])) {
+ if (query) throw new ComputerRequestError("A viewport address takes no options.");
+ // Resolving the name refuses a window that has closed.
+ this.address(`computer:${segments[0]}`);
+ if (!NAME.test(segments[1])) throw new ComputerRequestError("A viewport name is 1-32 lowercase letters, digits, - or _ and starts with a letter.");
+ return requireComputerAddress(viewportSource(this.names.get(segments[0])!.window_ref, segments[1]));
+ }
+ // A short window address, or a viewport of one: computer:zinpc/w3qk or computer:zinpc/w3qk/pano.
+ if ((segments.length === 2 || segments.length === 3) && !ADDRESS_KINDS.has(segments[0]) && SHORT_REFERENCE.test(segments[1])) {
let long: string;
try { long = this.shorts.resolve(segments[0], segments[1]); }
catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
- return requireComputerAddress(long + query);
+ if (segments.length === 3 && (query || !long.startsWith("computer:window/"))) throw new ComputerRequestError("Only a window address takes a viewport name after it.");
+ return requireComputerAddress(long + (segments.length === 3 ? `/${segments[2]}` : query));
}
return requireComputerAddress(source);
}
@@ -411,7 +426,7 @@ export class ComputerResources {
}
/** The owner of a window, display, observation or sequence address. */
private async routeAddress(cwd: string, address: ComputerAddress, signal?: AbortSignal): Promise<Owner> {
- if (address.kind === "window") return this.route(cwd, { window_ref: address.ref }, signal);
+ if (address.kind === "window" || address.kind === "viewport") return this.route(cwd, { window_ref: address.ref }, signal);
if (address.kind === "display") return this.route(cwd, { display: address.ref, machine: address.machine }, signal);
if (address.kind === "observation") return this.route(cwd, this.snapshot(address.ref).target, signal);
if (address.kind === "sequence")
@@ -432,7 +447,7 @@ export class ComputerResources {
case "app": { const app = this.apps.get(address.ref); return app && { machine: app.machine }; }
case "sandbox": { const sandbox = this.sandboxes.get(address.ref); return sandbox && { machine: sandbox.machine ?? LOCAL_MACHINE, sandbox: sandbox.ref }; }
case "sequence": return holder(session => session.hasSequence(address.ref));
- case "window": return holder(session => session.hasWindow(address.ref));
+ case "window": case "viewport": return holder(session => session.hasWindow(address.ref));
case "observation": {
const target = this.snapshots.get(address.ref)?.target;
return !target ? undefined : target.machine !== undefined ? { machine: target.machine } : holder(session => session.hasWindow(target.window_ref!));
@@ -592,12 +607,15 @@ export class ComputerResources {
// Names whose window closed disappear from the list and stop resolving.
for (const [name, entry] of this.names) if (!this.holdsWindow(entry.window_ref)) this.names.delete(name);
const names = [...this.names].map(([name, entry]) => `- computer:${name} → ${computerSource("window", entry.window_ref)}${entry.title ? ` | ${entry.title}` : ""}`);
+ for (const [key, viewport] of this.viewports) if (!this.holdsWindow(viewport.window_ref)) this.viewports.delete(key);
+ const viewports = [...this.viewports.values()].map(viewport => `- ${viewportSource(viewport.window_ref, viewport.name)}: ${viewport.rect.width}x${viewport.rect.height} at ${viewport.rect.x},${viewport.rect.y} in full-resolution window pixels`);
const windows = [...this.titles].filter(([ref]) => this.holdsWindow(ref) && this.shorts.issued(computerSource("window", ref)))
.map(([ref, title]) => `- ${computerSource("window", ref)} | ${title}`);
const sandboxes = [...this.sandboxes.values()].map(sandbox => `- ${computerSource("sandbox", sandbox.ref)} on ${computerSource("machine", sandbox.machine ?? LOCAL_MACHINE)}`);
return ["# Computer use", "Machines (computer:machine/local is this one; a machine configured under machines in .pi/computer-use.json starts on its first use). Every window, display, observation and sequence address belongs to the machine that issued it, and every receipt names that machine:",
...machines, ...(sandboxes.length ? ["Sandboxes and isolated sessions:", ...sandboxes] : []),
...(names.length ? ["Window names (computer_window name adds one):", ...names] : []),
+ ...(viewports.length ? ["Viewports (computer_view makes, moves and resizes them; Delete removes one):", ...viewports] : []),
...(windows.length ? ["Windows listed so far:", ...windows] : []),
"Delete this address to release every machine, browser, sandbox and session this computer use started; the result lists each part."].join("\n");
}
@@ -620,6 +638,7 @@ export class ComputerResources {
return result && this.stamp(place, result);
}
private async readAddress(address: ComputerAddress, cwd: string, views?: readonly string[], signal?: AbortSignal): Promise<ComputerResult | undefined> {
+ if (address.kind === "viewport") return this.readViewport(address, cwd, views, signal);
const view = parseComputerViews(views);
if (address.kind !== "window" && address.kind !== "observation") {
if (view.ocr) throw new ComputerRequestError("ocr reads a window, one of its observations or a region of one.");
@@ -846,7 +865,12 @@ export class ComputerResources {
if (address.kind === "sandbox") return this.closeSandbox(this.sandbox(address.ref));
if (address.kind === "machine" && !address.part) return this.closeMachine(address.ref);
if (address.kind === "session") return this.closeAll();
- throw new ComputerRequestError("Delete closes a window, a sandbox or isolated session, a machine or computer:session. Observations, sequences, apps and displays need no closing.");
+ if (address.kind === "viewport") {
+ const viewport = this.viewport(address);
+ this.viewports.delete(viewportSource(viewport.window_ref, viewport.name));
+ return { content: [{ type: "text", text: `Removed ${viewportSource(viewport.window_ref, viewport.name)}; its window stays open.` }], details: { status: "closed", mode: this.getMode(), window_ref: viewport.window_ref } };
+ }
+ throw new ComputerRequestError("Delete closes a window, a sandbox or isolated session, a machine or computer:session, or removes a viewport. Observations, sequences, apps and displays need no closing.");
}
/** Name a window by its address, so computer:<name> resolves to it until it closes. The receipt names its machine. */
async name(address: ComputerAddress, cwd: string, name: unknown, signal?: AbortSignal): Promise<ComputerResult> {
@@ -855,6 +879,86 @@ export class ComputerResources {
await this.route(cwd, { window_ref: address.ref }, signal);
return this.stamp(place, this.nameWindow(name, address.ref));
}
+ /** A viewport the agent made; refused when it was never made or its window has closed. */
+ private viewport(address: Extract<ComputerAddress, { kind: "viewport" }>): Viewport {
+ const source = viewportSource(address.ref, address.name);
+ if (!this.holdsWindow(address.ref)) {
+ for (const [key, viewport] of this.viewports) if (viewport.window_ref === address.ref) this.viewports.delete(key);
+ throw new ComputerRequestError(`The window of ${source} has closed or was released. Search its machine's windows again.`);
+ }
+ const viewport = this.viewports.get(source);
+ if (!viewport) throw new ComputerRequestError(`${source} is no viewport yet. Make it with computer_view and x, y, width and height in pixels of the window's latest image.`);
+ return viewport;
+ }
+ /** Full-resolution pixels per pixel of the window's latest image the agent saw, on each axis; 1 before any image. */
+ private imageScale(session: ComputerSession, window: string, full: { width: number; height: number }) {
+ const seen = session.inputView(window).seen;
+ return seen ? { x: full.width / seen.width, y: full.height / seen.height } : { x: 1, y: 1 };
+ }
+ /** The PNG of a full-resolution window capture. */
+ private static pixels(captured: ComputerResult): Buffer | undefined {
+ const image = captured.details.status === "observed" ? captured.content.find(block => block.type === "image") : undefined;
+ return image?.type === "image" ? Buffer.from(image.data, "base64") : undefined;
+ }
+ /** The details that name a viewport in a receipt: its address, its window, its rectangle and the size of its image. */
+ private static viewportDetails(viewport: Viewport, clipped: PixelBox): NonNullable<ComputerResult["details"]["viewport"]> {
+ return { source: viewportSource(viewport.window_ref, viewport.name), name: viewport.name, window_ref: viewport.window_ref, rect: clipped, image: viewportImageSize(clipped) };
+ }
+ /** A viewport's image cut from a full-resolution window capture, with where it lies in the window's latest image. */
+ private async showViewport(viewport: Viewport, clipped: PixelBox, png: Buffer, captured: ComputerResult, scale: { x: number; y: number }, note = ""): Promise<ComputerResult> {
+ const image = await viewportImage(png, clipped), details = ComputerResources.viewportDetails(viewport, clipped), size = details.image;
+ const shown = { x: Math.round(clipped.x / scale.x), y: Math.round(clipped.y / scale.y), width: Math.round(clipped.width / scale.x), height: Math.round(clipped.height / scale.y) };
+ const text = `${details.source} is ${shown.width}x${shown.height} at ${shown.x},${shown.y} of the window's latest image. Its image is ${size.width}x${size.height}` +
+ (size.width === clipped.width ? ", the window's own resolution." : `: ${clipped.width}x${clipped.height} window pixels scaled down to fit ${VIEWPORT_CAP} px.`) +
+ ` Points of the computer_* input tools on ${details.source} are pixels of this image.${note}`;
+ return { content: [{ type: "text", text }, { type: "image", data: image.toString("base64"), mimeType: "image/png" }],
+ details: { ...captured.details, status: "observed", viewport: details } };
+ }
+ /**
+ * Make, move or resize a viewport of a window and show it. The rectangle is in pixels of the window's latest image,
+ * like points of the input tools; fields left out keep the viewport's current ones. The receipt names its machine.
+ */
+ async view(address: ComputerAddress, cwd: string, given: Partial<PixelBox>, signal?: AbortSignal): Promise<ComputerResult> {
+ if (address.kind !== "viewport")
+ throw new ComputerRequestError("computer_view needs a viewport address: computer:<window name>/<viewport name> such as computer:game/pano, or a window address with the viewport name after it such as computer:zinpc/w3qk/pano.");
+ const existing = this.viewports.get(viewportSource(address.ref, address.name));
+ if (existing) this.viewport(address);
+ const place = this.place(address);
+ const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);
+ const captured = await session.windowPixels(address.ref, signal);
+ const png = ComputerResources.pixels(captured);
+ if (!png) return this.stamp(place, captured);
+ const full = await pngSize(png), scale = this.imageScale(session, address.ref, full);
+ let rect: PixelBox;
+ try { rect = viewportRect(given, scale, existing?.rect); }
+ catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
+ const clipped = clipViewport(rect, full);
+ if (!clipped) throw new ComputerRequestError(`That rectangle lies outside the window: its latest image is ${Math.round(full.width / scale.x)}x${Math.round(full.height / scale.y)}.`);
+ const viewport: Viewport = { window_ref: address.ref, name: address.name, rect: clipped };
+ this.viewports.set(viewportSource(address.ref, address.name), viewport);
+ const unseen = session.inputView(address.ref).seen ? "" : " The window had no Read with an image yet, so the rectangle was taken in its full-resolution pixels.";
+ return this.stamp(place, await this.showViewport(viewport, clipped, png, captured, scale, unseen));
+ }
+ /** Read a viewport: its image at full resolution up to VIEWPORT_CAP, and with ocr the text in it. Only looks. */
+ private async readViewport(address: Extract<ComputerAddress, { kind: "viewport" }>, cwd: string, views: readonly string[] | undefined, signal?: AbortSignal): Promise<ComputerResult> {
+ const view = parseComputerViews(views);
+ if (view.kind !== "overview" || view.changes || view.regions || view.aria || view.capture.granularity === "ui" || Object.keys(view.capture).some(key => key !== "granularity"))
+ throw new ComputerRequestError("A viewport Read shows its image; add ocr for the text in it. Read its window's address for other views.");
+ const viewport = this.viewport(address);
+ const session = (await this.routeAddress(cwd, address, signal)).session;
+ const captured = await session.windowPixels(viewport.window_ref, signal);
+ const png = ComputerResources.pixels(captured);
+ if (!png) return captured;
+ const full = await pngSize(png), clipped = clipViewport(viewport.rect, full);
+ if (!clipped) throw new ComputerRequestError(`${viewportSource(viewport.window_ref, viewport.name)} lies outside its window now. Move it with computer_view.`);
+ const result = await this.showViewport(viewport, clipped, png, captured, this.imageScale(session, viewport.window_ref, full));
+ if (!view.ocr) return result;
+ const image = result.content.find(block => block.type === "image")!;
+ let lines;
+ try { lines = await session.recognizeText(image, undefined, signal); }
+ catch (error) { throw new ComputerRequestError(`Text recognition failed: ${error instanceof Error ? error.message : error}`); }
+ return { ...result, details: { ...result.details, text_lines: lines } };
+ }
/**
* Move, resize, minimize, maximize, focus or restore a window by its address: a fresh native state sample bounds the
* one command, so no window-state address is handed out.
@@ -899,24 +1003,29 @@ export class ComputerResources {
* window's latest image; without a current one the window is captured first. The receipt names its machine.
*/
async input(address: ComputerAddress, cwd: string, plan: InputPlan, signal?: AbortSignal): Promise<ComputerResult> {
- if (address.kind !== "window") throw new ComputerRequestError("Give the window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game.");
+ if (address.kind !== "window" && address.kind !== "viewport")
+ throw new ComputerRequestError("Give the window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game, or a viewport address such as computer:game/pano.");
+ // In a viewport, points are pixels of the viewport's image and its pixels show what the action changed.
+ const viewport = address.kind === "viewport" ? this.viewport(address) : undefined;
+ if (viewport && plan.space !== undefined) throw new ComputerRequestError("space does not apply in a viewport: its points are pixels of the viewport's image.");
const place = this.place(address);
const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);
- let result = await this.inputStep(address.ref, session, plan, cwd, signal);
+ let result = await this.inputStep(address.ref, session, plan, cwd, signal, viewport);
// A follow-up such as Enter after typing runs only after the first part was delivered.
if (plan.then && result.details.status === "delivered") {
- const next = await this.inputStep(address.ref, session, plan.then, cwd, signal);
+ const next = await this.inputStep(address.ref, session, plan.then, cwd, signal, viewport);
result = { ...next, content: [...result.content, ...next.content] };
}
return this.stamp(place, result);
}
- private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal, viewport?: Viewport): Promise<ComputerResult> {
const ref = refKind(plan.target);
const browser = session.isBrowserWindow(window);
- if (browser && plan.space !== "window" && ref !== "hint") return this.pageInput(window, session, plan, cwd, signal);
+ // A viewport of a browser window is pixels of the window, like space "window".
+ if (browser && !viewport && plan.space !== "window" && ref !== "hint") return this.pageInput(window, session, plan, cwd, signal);
if (plan.space !== undefined && !browser) throw new ComputerRequestError("space applies to browser windows only; points on other windows are pixels of the window's latest image.");
if (ref === "page") throw new ComputerRequestError(`${plan.target} is a page ref from a DOM Read; ${browser ? "omit space: \"window\" to act on the page" : "this window is not a browser page, so use a ref from its latest Read or a Playwright locator"}.`);
- return this.nativeStep(window, session, plan, cwd, signal);
+ return this.nativeStep(window, session, plan, cwd, signal, viewport);
}
/** Typed input on the active page of a browser window through Playwright; a current DOM observation gives the change list. */
@@ -940,7 +1049,7 @@ export class ComputerResources {
* window's current observation, captured first when there is none with the pixels a point needs. A click or scroll
* on a target goes through UI Automation; when no pattern carries it, which sends nothing, it goes again as real input.
*/
- private async nativeStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal): Promise<ComputerResult> {
+ private async nativeStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal, viewport?: Viewport): Promise<ComputerResult> {
if (typeof plan.native === "string") throw new ComputerRequestError(plan.native);
const command: Record<string, unknown> = structuredClone(plan.native);
const action = String(command.action);
@@ -963,6 +1072,8 @@ export class ComputerResources {
delete command.option;
}
let result: ComputerResult;
+ // The viewport's part of the window's full-resolution capture, when the action is in a viewport.
+ let box: PixelBox | undefined;
// What the real-input retry sends: the input observation is spent by then, so its refs become locators or points.
let retry = (): InputPlan => plan;
if (refKind(target) === "locator") {
@@ -974,8 +1085,11 @@ export class ComputerResources {
delete command.timeout_ms;
const pixels = plan.point !== undefined || plan.end !== undefined || command.unit === "wheel_detents" || action === "timeline";
if (hint) this.hinted(view, hint);
- else if (!view.native || pixels && view.native.width === undefined) {
- const captured = await this.read({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, ["image"], signal);
+ else if (!view.native || pixels && view.native.width === undefined || viewport && !view.native.full) {
+ // A viewport maps its points into a full-resolution capture of its window; the image the agent saw stays its latest.
+ if (viewport) this.retireWindow(window, "native");
+ const captured = viewport ? this.remember({ window_ref: window }, await session.observeFull(window, signal))
+ : await this.read({ kind: "window", ref: window, source: computerSource("window", window) }, cwd, ["image"], signal);
if (!captured || captured.details.status !== "observed") return captured ?? { content: [{ type: "text", text: "The window could not be captured; nothing was sent." }], details: { status: "refused", mode: this.getMode(), window_ref: window }, isError: true };
view = session.inputView(window);
}
@@ -983,8 +1097,16 @@ export class ComputerResources {
if (pixels && native.width === undefined) throw new ComputerRequestError(`${plan.label} at a point needs the window's pixels: Read the window with an image, or give a locator.`);
// Points are in the latest image the agent saw; the current observation can be a capture of another size.
const sx = native.width && view.seen ? native.width / view.seen.width : 1, sy = native.height && view.seen ? native.height / view.seen.height : 1;
- const at = (point: InputPoint) => ({ x: Math.min(native.width! - 1, Math.round(point.x * sx)), y: Math.min(native.height! - 1, Math.round(point.y * sy)) });
- const seen = (point: InputPoint) => ({ x: point.x / sx, y: point.y / sy });
+ // In a viewport they are pixels of its image instead, mapped into the full-resolution capture.
+ box = viewport && native.full ? clipViewport(viewport.rect, { width: native.width!, height: native.height! }) : undefined;
+ if (viewport && !hint && !box) throw new ComputerRequestError(`${viewportSource(viewport.window_ref, viewport.name)} lies outside its window now. Move it with computer_view.`);
+ const at = (point: InputPoint) => {
+ if (!box) return { x: Math.min(native.width! - 1, Math.round(point.x * sx)), y: Math.min(native.height! - 1, Math.round(point.y * sy)) };
+ const inside = windowPoint(box, point), image = viewportImageSize(box);
+ if (!inside) throw new ComputerRequestError(`${point.x},${point.y} lies outside ${viewportSource(viewport!.window_ref, viewport!.name)}: its image is ${image.width}x${image.height}.`);
+ return inside;
+ };
+ const seen = (point: InputPoint) => box ? viewportPoint(box, point) : { x: point.x / sx, y: point.y / sy };
const centre = (ref: string) => {
const box = this.hinted(view, ref).bounds.window_image;
if (!box || native.width === undefined) throw new ComputerRequestError(`${ref} has no place in the window's image; give a point or a locator.`);
@@ -1000,16 +1122,19 @@ export class ComputerResources {
} else if (action === "timeline") {
command.events = (command.events as Record<string, unknown>[]).map(event => typeof event.x === "number" && typeof event.y === "number" ? { ...event, ...at(event as unknown as InputPoint) } : event);
} else if (plan.point) Object.assign(command, at(plan.point));
- else if (command.unit === "wheel_detents") Object.assign(command, { x: Math.floor(native.width! / 2), y: Math.floor(native.height! / 2) });
+ else if (command.unit === "wheel_detents") Object.assign(command, box ? { x: Math.floor(box.x + box.width / 2), y: Math.floor(box.y + box.height / 2) } : { x: Math.floor(native.width! / 2), y: Math.floor(native.height! / 2) });
if (hint && action !== "drag") retry = () => NATIVE_LOCATOR_ACTIONS.includes(LOCATOR_VERBS[action] ?? action) ? { ...plan, target: suggestLocator(this.hinted(view, hint)) }
: { ...plan, target: undefined, point: seen(centre(hint)) };
const observation = { kind: "observation" as const, ref: native.id, source: computerSource("observation", native.id), ...(hint && action !== "drag" ? { hint } : {}) };
- result = await this.performAt(observation, cwd, JSON.stringify(command), signal);
+ result = await this.performAt(observation, cwd, JSON.stringify(command), signal, box);
}
+ if (viewport) result = { ...result, details: { ...result.details, viewport: ComputerResources.viewportDetails(viewport, box ?? viewport.rect) } };
const capability = result.details.action_capability;
if (!structural || result.details.status !== "refused" || capability?.background !== "unavailable") return result;
- const again = await this.nativeStep(window, session, { ...retry(), real: true }, cwd, signal);
- return { ...again, content: [{ type: "text", text: `No UI Automation pattern of the target carried this ${plan.label}, so this call used the real pointer.` }, ...again.content] };
+ // A ref acted on a capture without the viewport's pixels; its retry point is in the window's latest image.
+ const again = await this.nativeStep(window, session, { ...retry(), real: true }, cwd, signal, box ? viewport : undefined);
+ return { ...again, content: [{ type: "text", text: `No UI Automation pattern of the target carried this ${plan.label}, so this call used the real pointer.` }, ...again.content],
+ ...(viewport ? { details: { ...again.details, viewport: again.details.viewport ?? result.details.viewport } } : {}) };
}
/** The element of a ref in the window's current observation. */
@@ -1023,7 +1148,7 @@ export class ComputerResources {
const place = this.place(address);
return this.stamp(place, await this.performAt(address, cwd, value, signal));
}
- private async performAt(address: ComputerAddress, cwd: string, value: string, signal?: AbortSignal): Promise<ComputerResult> {
+ private async performAt(address: ComputerAddress, cwd: string, value: string, signal?: AbortSignal, viewport?: PixelBox): Promise<ComputerResult> {
if (address.kind === "sandbox") {
const sandbox = this.sandbox(address.ref);
const command = commandObject(value);
@@ -1111,7 +1236,7 @@ export class ComputerResources {
return this.locatorAction(window, await this.sessionFor(cwd, { window_ref: window }), commandObject(value), cwd, signal, { successor: true, explicit: false });
}
const focus = inputFocus(commandObject(value), snapshot.result.details.ui, address.hint);
- let follow = window ? await this.withFrame({ successor: true, explicit: false, ...(focus ? { focus } : {}) }, window, address.ref, cwd) : undefined;
+ let follow = window ? await this.withFrame({ successor: true, explicit: false, ...(focus ? { focus } : {}), ...(viewport ? { viewport } : {}) }, window, address.ref, cwd) : undefined;
// A page action compares Playwright screenshots of the page, taken before and after it.
if (follow && window && snapshot.result.details.dom) {
const page = await this.pageShot(follow.session!, window, address.ref, signal);
@@ -1184,7 +1309,7 @@ export class ComputerResources {
let successor: ComputerResult | undefined, settled: StateDiff | undefined, crops: ComputerResult["content"] = [];
if (status !== "refused") {
if (result.details.dom) successor = result;
- else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal, follow.focus));
+ else ({ successor, diff: settled, crops } = await this.settle(window, session, before, follow.frame, signal, follow.focus, follow.viewport));
}
const after = successor?.details;
const diff = before.result.details.dom && after?.dom ? diffDom(before.result.details.dom, after.dom)
@@ -1210,16 +1335,22 @@ export class ComputerResources {
* to STEADY_MS, so the change list shows the settled window rather than a frame in the middle of a repaint.
* Captures keep the input observation's overview size so their pixels compare with it.
*/
- private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal, focus?: InputPoint):
+ private async settle(window: string, session: ComputerSession, before: Snapshot, frame: WindowFrame | undefined, signal?: AbortSignal, focus?: InputPoint, viewport?: PixelBox):
Promise<{ successor: ComputerResult; diff?: StateDiff; crops: ComputerResult["content"] }> {
const started = performance.now();
+ // Full-resolution captures continue at full resolution without changing the size of the image the agent saw.
+ const full = !!frame && frame.size.maxLongEdge >= FULL_RESOLUTION;
+ // A viewport compares only its own pixels, cut from full-resolution captures of the window.
+ const scope = full ? viewport : undefined;
+ const cut = async (png: Buffer) => scope ? viewportImage(png, scope).catch(() => undefined) : png;
let options: UiCaptureOptions = { granularity: "full", ...(frame ? frame.size : {}) };
- const baseline: Capture = { ui: before.result.details.ui, frame: frame ? await decodeFrame(frame.png) : undefined };
- const context = diffContext(before.result.details.ui, baseline.frame, focus);
- let successor: ComputerResult, diff: StateDiff | undefined, previous: Capture | undefined, latest: WindowFrame | undefined;
+ const first = frame ? await cut(frame.png) : undefined;
+ const baseline: Capture = { ui: before.result.details.ui, frame: first ? await decodeFrame(first) : undefined };
+ const context = scope ? focus ? { focus: viewportPoint(scope, focus) } : {} : diffContext(before.result.details.ui, baseline.frame, focus);
+ let successor: ComputerResult, diff: StateDiff | undefined, previous: Capture | undefined, latest: Buffer | undefined;
for (let attempt = 0; ; attempt++) {
this.retireWindow(window);
- successor = this.remember({ window_ref: window }, await session.observeQuiet(window, signal, options));
+ successor = this.remember({ window_ref: window }, await (full && options.granularity !== "ui" ? session.observeFull(window, signal) : session.observeQuiet(window, signal, options)));
const waited = Math.round(performance.now() - started);
// A window whose pixels cannot be captured (minimized, or a failed screenshot) still gets its UI compared.
if (successor.details.status !== "observed" && options.granularity === "full" && !signal?.aborted) {
@@ -1227,8 +1358,8 @@ export class ComputerResources {
}
if (successor.details.status !== "observed" || signal?.aborted) break;
const captured = session.windowFrame(window);
- latest = captured?.observation_id === successor.details.observation_id ? captured : undefined;
- const current: Capture = { ui: successor.details.ui, frame: latest ? await decodeFrame(latest.png) : undefined };
+ latest = captured && captured.observation_id === successor.details.observation_id ? await cut(captured.png) : undefined;
+ const current: Capture = { ui: successor.details.ui, frame: latest ? await decodeFrame(latest) : undefined };
diff = compareCaptures(baseline, current, context);
if (!diff) break;
diff.waited_ms = waited;
@@ -1243,7 +1374,8 @@ export class ComputerResources {
if (waited >= STEADY_MS) { diff.unsettled = true; break; }
previous = current;
}
- return { successor, diff, crops: diff && frame && latest ? await cropImages(frame.png, latest.png, diff) : [] };
+ if (scope && diff?.pixels) diff.pixels.space = "viewport";
+ return { successor, diff, crops: diff && first && latest ? await cropImages(first, latest, diff) : [] };
}
/**
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index 7cbf9d5..879601c 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -97,6 +97,14 @@ export function windowCommand(params: Static<typeof windowParameters>): Record<s
return { action, ...fields, ...(wanted.length ? { unit: "windows_desktop_pixels" } : {}) };
}
+const viewParameters = Type.Object({
+ viewport: Type.String({ description: "Viewport address: a window's name or address with the viewport's name after it, such as computer:game/pano or computer:zinpc/w3qk/pano. The name is 1-32 lowercase letters, digits, - or _, starting with a letter. A new name makes the viewport; the name of an existing one moves or resizes it." }),
+ x: Type.Optional(Type.Integer({ minimum: 0, description: "Left edge in pixels of the window's latest image. Required for a new viewport; omit to keep the current one." })),
+ y: Type.Optional(Type.Integer({ minimum: 0, description: "Top edge in pixels of the window's latest image. Required for a new viewport; omit to keep the current one." })),
+ width: Type.Optional(Type.Integer({ minimum: 1, description: "Width in pixels of the window's latest image. Required for a new viewport; omit to keep the current one." })),
+ height: Type.Optional(Type.Integer({ minimum: 1, description: "Height in pixels of the window's latest image. Required for a new viewport; omit to keep the current one." })),
+}, { additionalProperties: false });
+
const displayParameters = Type.Object({
display: Type.String({ description: "computer:machine/<name>/display/primary, or a display address listed by Read computer:machine/<name>/displays." }),
width: Type.Integer({ minimum: 640, maximum: 16384, description: "New width in physical pixels." }),
@@ -254,9 +262,9 @@ export function recordCommand(params: Static<typeof recordParameters>): { comman
return { command: { action: "audio_record", ...fields } };
}
-const inputWindow = Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game." });
+const inputWindow = Type.String({ description: "Window address from Search, Read or a name, such as computer:zinpc/w3qk or computer:game, or a viewport address from computer_view, such as computer:game/pano." });
const inputTarget = (what: string) => Type.Optional(Type.String({ description: `${what}: a ref from the window's latest Read (1:a in a window Read, e12 in a page snapshot) or a Playwright locator such as getByRole('button', { name: 'Save' }). On native windows a target is pressed through UI Automation in the background when it supports that, otherwise with the real pointer.` }));
-const coordinate = (axis: string) => Type.Optional(Type.Number({ minimum: 0, description: `${axis} of a point instead of target: pixels of the window's latest image, or CSS pixels of a browser page. A point on a native window gets the real pointer: the window comes to the front.` }));
+const coordinate = (axis: string) => Type.Optional(Type.Number({ minimum: 0, description: `${axis} of a point instead of target: pixels of the window's latest image, pixels of the viewport's image for a viewport address, or CSS pixels of a browser page. A point on a native window gets the real pointer: the window comes to the front.` }));
const pointSpace = Type.Optional(Type.Union([Type.Literal("page"), Type.Literal("window")], { description: "Browser windows: page (default) takes refs, locators and CSS points of the page; window takes refs and points of a window Read, such as the browser's own toolbar." }));
const modifiersField = Type.Optional(Type.Array(Type.Union([Type.Literal("ctrl"), Type.Literal("shift"), Type.Literal("alt"), Type.Literal("win")]), { maxItems: 4, description: "Keys held during the action." }));
const locatorTimeout = Type.Optional(Type.Integer({ minimum: 0, maximum: 60000, description: "How long a locator target may take to match one ready element, default 5000." }));
@@ -508,6 +516,18 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
}, address => command.action === "name" ? resources.name(address, ctx.cwd, command.as, signal) : resources.manageWindow(address, ctx.cwd, command, signal)));
},
});
+ pi.registerTool({
+ ...common, name: "computer_view", label: "Computer view",
+ description: "Use computer_view to make, move or resize a viewport: a named rectangle of a window that follows the window. The result shows the viewport at the window's own resolution. Give its address to the computer_* input tools to act at points of that image and see what changed only there; Read it to look again.",
+ parameters: viewParameters,
+ annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+ ...rendered("computer_view", args => `${args.viewport}${args.x !== undefined || args.y !== undefined ? ` at ${args.x ?? "_"},${args.y ?? "_"}` : ""}${args.width !== undefined || args.height !== undefined ? ` ${args.width ?? "_"}x${args.height ?? "_"}` : ""}`),
+ async execute(_id, params, signal, _update, ctx) {
+ const { viewport, ...rect } = params;
+ return actionResult(await computerAction(resources, viewport, "view", () => resources.address(viewport),
+ address => resources.view(address, ctx.cwd, rect, signal)));
+ },
+ });
pi.registerTool({
...common, name: "computer_wait", label: "Computer wait",
description: "Use computer_wait to wait until a window's UI or a browser page meets conditions, or to check them once, without sending input. The last sample becomes the window's current observation.",
diff --git a/src/cua-image-settings.ts b/src/cua-image-settings.ts
index 981fbec..cbba142 100644
--- a/src/cua-image-settings.ts
+++ b/src/cua-image-settings.ts
@@ -5,6 +5,8 @@ export interface CuaVisionSettings {
}
/** Default overview long edge (HD): large enough to read UI text, small enough for model image budgets. */
export const DEFAULT_MAX_LONG_EDGE = 1280;
+/** The largest long edge CUA captures for this extension: windows up to it come at their own full resolution. */
+export const FULL_RESOLUTION = 4096;
const defaults = (): CuaVisionSettings => ({ kernel: "lanczos3", maxLongEdge: DEFAULT_MAX_LONG_EDGE, scale: 1, version: 0, route: "cua_native" });
/** Validate and freeze the native cap; returned raster sizes are never adjusted by the extension. */
diff --git a/src/provider.ts b/src/provider.ts
index c7a970f..0cdaa27 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -19,7 +19,7 @@ const record = (value: unknown): Record<string, unknown> => value !== null && ty
const label = (value: unknown) => typeof value === "string" ? value.replace(/[\r\n\t]/g, " ").slice(0, 1024) : "";
/** Agent guidance shared by the IDE plugin and the standalone tools. */
-export const COMPUTER_PATH_HELP = "Read docs:computer for native resources. computer:session, computer:machine/<name> (local, or a configured machine) with its /windows, /apps and /displays, its /display/primary or listed /display/ref, and returned computer:window/ref, observation/id or sequence/id, each running on the machine that issued it; results show short forms such as computer:<machine>/w3qk (o observation, s sequence), and computer:<name> after computer_window name. Read only looks; the computer_* tools act.";
+export const COMPUTER_PATH_HELP = "Read docs:computer for native resources. computer:session, computer:machine/<name> (local, or a configured machine) with its /windows, /apps and /displays, its /display/primary or listed /display/ref, and returned computer:window/ref, observation/id or sequence/id, each running on the machine that issued it; results show short forms such as computer:<machine>/w3qk (o observation, s sequence), and computer:<name> after computer_window name. A viewport that computer_view made is its window's address with its name after it, such as computer:game/pano or computer:<machine>/w3qk/pano; Read shows its image at full resolution, ocr adds its text. Read only looks; the computer_* tools act.";
export const COMPUTER_VIEWS_HELP = "Computer: image/ui/full; dom alone or dom:{target,boxes,depth,log} on a browser window opened or attached with computer_open returns a fresh Playwright aria snapshot with [ref=...] values of the active tab; log:true adds the retained console/page-error/network history; target is {ref} or locator fields role,name,label,text,placeholder,test_id,css,exact,frames. On a DOM observation: image is a page screenshot, element:crop|highlight,<ref> an element image; Search filters its lines. state alone on a window returns its minimized, maximized, visible and foreground state and bounds without pixels/UI, plus whether the window is hung and its process memory/CPU; computer_window changes them. ocr (alone or with capture options on a window, alone on an observation, or with region:x,y,width,height on an observation) adds Windows OCR lines with boxes in overview pixels; use maxLongEdge 3840 for small text. aria (alone or with ui capture options, on a window) returns its UI as a Playwright aria snapshot with hints as [ref=...]. changes (alone, or with ui, image or full and capture options, on a window) takes a fresh capture and lists only rows changed, added or removed since that window's previous observation, with the new observation's hints, plus changed pixel areas with before/after crops instead of the full image. regions (alone or with image/full options on a window, alone on its observation) lists icons and text areas found by Cua perception with boxes in overview pixels; click a box centre with computer_click at its x,y. capture:{maxLongEdge,scale,max_elements,max_depth,hints,guides}. region:x,y,width,height or element:inspect|scope|highlight|crop|text,hint on an observation; element:text returns one element's full text. frames:first|last|all|endpoints|changed or frames:{indexes:[0]} / {from:0,to:2} / {select:'changed'} on a sequence address from computer_record; add as:'sheet' (one tiled image with index/time labels, optional columns) or as:'max'|'min' (one per-pixel composite), with optional region:{x,y,width,height} in frame pixels and maxLongEdge. Detail/frames are readonly; text offset/limit select UI lines, never image cells.";
export const COMPUTER_SEARCH_HELP = "Search computer:machine/<name>/apps or computer:machine/<name>/windows (local is this machine) for installed-app/permitted-window metadata, or an observation for its collected UI/readonly DOM. Read computer:machine/<name>/displays for monitor metadata. Discovery launches nothing; observation Search performs no new native walk or DOM query. Treat missing matches in a partial projection as unknown.";
export const COMPUTER_DELETE_HELP = "A computer window address closes that window like its close button (add ?force=true to kill the hung app of a window this session opened); a sandbox, isolated session or machine address releases it; computer:session releases everything computer use started.";
@@ -48,6 +48,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str
// Every routed receipt names the machine (and sandbox) it ran on.
const lines = [`Computer ${d.status} (${d.mode})${d.machine ? ` on ${d.machine}${d.sandbox ? `, sandbox ${d.sandbox}` : ""}` : ""}`];
if (d.window_ref) lines.push(`Window: ${computerSource("window", d.window_ref)}`);
+ if (d.viewport) lines.push(`Viewport: ${d.viewport.source}; points on it are pixels of its ${d.viewport.image.width}x${d.viewport.image.height} image`);
if (d.window_state_id && d.status === "observed") {
const sample = d.window_state_kind === "management" ? record(state.state) : state;
lines.push(`Window state: minimized=${sample.minimized}, visible=${sample.visible}; foreground=${sample.foreground_at_target}, keyboard_focus=${sample.keyboard_focus_at_target}${d.window_state_kind === "management" ? `; maximized=${sample.maximized}` : ""}`);
@@ -186,6 +187,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str
: d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; refs of this sample are current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly.")
: d.dom ? "Act on the window with the computer_* input tools, a ref such as e2 as target. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot."
: d.window_state_id ? "This Read only looks; computer_window changes the window's state."
+ : d.viewport && d.status === "observed" ? "Act in this viewport with the computer_* input tools at its address; a viewport Read gives no refs."
: d.status === "observed" ? "Refs of this Read are current for the next input to its window; detail, display and frame Reads give none."
: "Native delivery is not verified application state. Never replay unknown input; Read a fresh window overview/state before another action.");
const text = lines.join("\n");
@@ -225,7 +227,8 @@ function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {
const share = pixels.share * 100, percent = share < 0.1 ? share.toFixed(2) : share.toFixed(1), crops = pixels.crops ?? [];
const rect = (b: PixelBox) => `x=${b.x} y=${b.y} w=${b.width} h=${b.height}`;
const chrome = pixels.chrome ? [`Browser chrome outside the page also changed (${rect(pixels.chrome.box)}, ${pixels.chrome.regions} ${pixels.chrome.regions === 1 ? "area" : "areas"}): tab strip, address bar or loading state; not listed.`] : [];
- const page = pixels.space === "page", whole = page ? "page" : "window", unit = page ? "CSS pixels of the page" : "overview pixels";
+ const page = pixels.space === "page", viewport = pixels.space === "viewport";
+ const whole = page ? "page" : viewport ? "viewport" : "window", unit = page ? "CSS pixels of the page" : viewport ? "pixels of the viewport's image" : "overview pixels";
if (pixels.full) return [pixels.navigated ? `The page navigated (${percent}% of its pixels changed): image 1 shows the whole page after the action.`
: `Most of the ${whole} changed${since} (${percent}% of its pixels): image 1 shows the whole ${whole} after the action.`, ...chrome];
if (!pixels.areas.length) return chrome;
@@ -274,6 +277,8 @@ export interface ComputerReceipt {
action: string | null;
source: string;
window: string | null;
+ /** The viewport a Read showed or an input went to. */
+ viewport?: string;
/** The sandbox address computer_open started. */
sandbox?: string;
/** The new current observation issued by this call, such as a DOM action's next snapshot. */
@@ -316,6 +321,7 @@ export function computerReceipt(source: string, action: string | null, outcome:
return {
outcome: d.status === "listed" ? "observed" : d.status, effect: computerEffect(outcome.result), stage: "target", action, source,
window: typeof d.window_ref === "string" ? computerSource("window", d.window_ref) : null,
+ ...(d.viewport ? { viewport: d.viewport.source } : {}),
...(typeof d.sandbox_ref === "string" ? { sandbox: computerSource("sandbox", d.sandbox_ref) } : {}),
observation: next ? computerSource("observation", next) : null,
input_consumed: outcome.inputConsumed, application_effect: "unverified",
@@ -400,7 +406,7 @@ export function deleteComputer(resources: ComputerResources, source: string, inp
function shortReceipt(resources: ComputerResources, receipt: ComputerReceipt): ComputerReceipt {
const short = (value: string | null | undefined) => value ? resources.shorten(value) : value;
return { ...receipt, window: short(receipt.window) ?? null, observation: short(receipt.observation) ?? null,
- ...(receipt.sandbox ? { sandbox: short(receipt.sandbox)! } : {}), ...(receipt.reason ? { reason: resources.shorten(receipt.reason) } : {}), text: resources.shorten(receipt.text) };
+ ...(receipt.sandbox ? { sandbox: short(receipt.sandbox)! } : {}), ...(receipt.viewport ? { viewport: short(receipt.viewport)! } : {}), ...(receipt.reason ? { reason: resources.shorten(receipt.reason) } : {}), text: resources.shorten(receipt.text) };
}
function nativeBlocks(body: string, result: ComputerResult | undefined, uiOnly: boolean): Content[] {
return [{ type: "text", text: body }, ...(uiOnly || !result ? [] : result.details.selected_frames
diff --git a/src/session.ts b/src/session.ts
index e3fda98..f4aa82f 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -14,7 +14,7 @@ import { notifyWindowObserver, notifyHighlight, type WindowObserver, type Highli
import { screenRect, elementRegion, windowImageMapping, windowRegion, nativeClickPoint, type ScreenRect } from "./highlight-geometry.ts";
import { observationSource, sourceIdentity, type ObservationSource, type SourceIdentity, type NativePlatform } from "./observation-source.ts";
import { actionCapability, type ActionCapability } from "./action-capability.ts";
-import { DEFAULT_MAX_LONG_EDGE, nativeImageCap, type CuaVisionSettings } from "./cua-image-settings.ts";
+import { DEFAULT_MAX_LONG_EDGE, FULL_RESOLUTION, nativeImageCap, type CuaVisionSettings } from "./cua-image-settings.ts";
import type { DisplayMode } from "./display-mode.ts";
import { WEBVIEW2_DEBUGGING, type MachineHost, type ProcessHealth, type TextLine } from "./machine-host.ts";
import { setTimeout as delay } from "node:timers/promises";
@@ -91,6 +91,8 @@ export interface ComputerResult {
mode: ComputerMode;
target?: WindowTarget;
window_ref?: string;
+ /** The viewport a Read showed or an input acted in: rect in the window's full-resolution pixels, image the size of its image. */
+ viewport?: { source: string; name: string; window_ref: string; rect: { x: number; y: number; width: number; height: number }; image: { width: number; height: number } };
observation_id?: string;
/** One-use window-state authority for native restore or management, never ordinary input. */
window_state_id?: string;
@@ -1284,6 +1286,54 @@ export class ComputerSession {
/** The window's latest unannotated capture with pixels, the baseline for comparing its pixels; undefined when none is kept. */
windowFrame(ref: string): WindowFrame | undefined { return this.entries.get(ref)?.frame; }
+ /**
+ * A full-resolution capture of a window for input in one of its viewports: it becomes the window's current
+ * observation, but the size of the image the agent last saw stays, so points of the window's latest image keep
+ * their meaning. Quiet, like a successor.
+ */
+ observeFull(ref: string, signal?: AbortSignal, granularity: "image" | "full" = "full"): Promise<ComputerResult> {
+ return this.queue(async () => {
+ if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
+ try {
+ const entry = await this.current(ref, signal);
+ if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
+ if (!this.allowed(entry)) return this.denied(entry.ref);
+ const seen = entry.seen;
+ try { return await this.capture(entry, signal, undefined, { granularity, maxLongEdge: FULL_RESOLUTION, scale: 1, hints: false }, true); }
+ finally { entry.seen = seen; }
+ } catch (error) { return this.failed(error, false); }
+ });
+ }
+
+ /**
+ * A window's pixels at full resolution for a viewport, without UI. Only looks: the window's observations, its pixel
+ * baseline and the size of the image the agent last saw stay as they are.
+ */
+ windowPixels(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
+ return this.queue(async () => {
+ if (signal?.aborted) return this.result("refused", "Cancelled before capture.");
+ try {
+ const entry = await this.current(ref, signal);
+ if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
+ if (!this.allowed(entry)) return this.denied(entry.ref);
+ const native = await this.driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: FULL_RESOLUTION }, signal);
+ if (!this.allowed(entry)) return this.denied(entry.ref);
+ if (signal?.aborted) return this.result("refused", "Cancelled during capture; pixels withheld.");
+ const state = native.structuredContent ?? {};
+ if (native.isError || typeof state.screenshot_error === "string" && state.screenshot_error.trim())
+ return this.result("refused", `Window screenshot unavailable${typeof state.screenshot_error === "string" ? `. Native reason: ${state.screenshot_error.trim().slice(0, 2000)}` : "."}`, undefined, this.addressed(entry, state.virtual_desktop));
+ const images = native.content.filter(block => block.type === "image");
+ const width = state.screenshot_width, height = state.screenshot_height;
+ if (images.length !== 1 || !Number.isSafeInteger(width) || !Number.isSafeInteger(height) || (width as number) <= 0 || (height as number) <= 0)
+ throw new Error("Driver returned no window image.");
+ if (!await this.current(entry.ref, signal) || !this.allowed(entry)) return this.result("refused", "Window changed during capture. Discover again.");
+ const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal);
+ return this.result("observed", `Full-resolution window pixels: ${width}x${height}.`, undefined,
+ { ...this.addressed(entry, state.virtual_desktop), ...(highlight ? { highlight } : {}) }, images);
+ } catch (error) { return this.failed(error, false); }
+ });
+ }
+
/**
* A fresh observation without the human frame, for successors: UI-only by default, or with
* pixels at a given overview size so they compare with an earlier capture.
@@ -1790,10 +1840,12 @@ export class ComputerSession {
* capture with its size and, until the first input spends them, its UI hints) and the size of its latest image,
* which is the space of the agent's points.
*/
- inputView(ref: string): { dom?: string; native?: { id: string; width?: number; height?: number; ui?: UiResource }; seen?: { width: number; height: number } } {
+ inputView(ref: string): { dom?: string; native?: { id: string; width?: number; height?: number; ui?: UiResource; full: boolean }; seen?: { width: number; height: number } } {
const entry = this.entries.get(ref), observed = entry?.observation;
+ // A full-resolution observation's pixels are the window's own: viewport rectangles apply to them as they are.
+ const full = !!observed && observed.width !== undefined && nativeImageCap(observed.settings) >= FULL_RESOLUTION;
return { ...(entry?.dom ? { dom: entry.dom.id } : {}),
- ...(observed ? { native: { id: observed.id, width: observed.width, height: observed.height, ...(observed.ui && !observed.pixelsOnly ? { ui: observed.ui } : {}) } } : {}),
+ ...(observed ? { native: { id: observed.id, width: observed.width, height: observed.height, full, ...(observed.ui && !observed.pixelsOnly ? { ui: observed.ui } : {}) } } : {}),
...(entry?.seen ? { seen: { ...entry.seen } } : {}) };
}
/** Whether this session issued a still-registered window reference. */
diff --git a/src/successor.ts b/src/successor.ts
index 2b5266c..28205cb 100644
--- a/src/successor.ts
+++ b/src/successor.ts
@@ -8,8 +8,11 @@ import { diffFrames, pixelsChanged, type DiffContext, type PixelBox, type PixelD
* scale, and `full` when most of the window changed and the only image is the whole window after the action.
*/
export type PixelReport = PixelDiff & { crops?: (PixelBox & { scale: number })[]; overview?: true; full?: true; navigated?: true; since?: string;
- /** `page` when the pixels are Playwright screenshots of a browser page in CSS pixels, not window overviews. */
- space?: "page" };
+ /**
+ * `page` when the pixels are Playwright screenshots of a browser page in CSS pixels, `viewport` when they are images of
+ * the viewport the action went to; otherwise window overviews.
+ */
+ space?: "page" | "viewport" };
/** Compact change list between an action's input observation and its successor. Bounded; full successors stay readable. */
export interface StateDiff {
diff --git a/src/viewport.ts b/src/viewport.ts
new file mode 100644
index 0000000..eb2a6ad
--- /dev/null
+++ b/src/viewport.ts
@@ -0,0 +1,75 @@
+import sharp from "sharp";
+import type { PixelBox } from "./pixel-diff.ts";
+
+/**
+ * The longest edge of a viewport's image. A larger viewport is scaled down to it; a smaller one keeps the window's own
+ * resolution. Models shrink larger images anyway (Claude above 1568 px).
+ */
+export const VIEWPORT_CAP = 1568;
+/** The smallest viewport side, in full-resolution window pixels. */
+const MIN_SIDE = 8;
+
+/** A point in pixels of an image. */
+export interface Point { x: number; y: number }
+/** A named rectangle of a window that the agent reads and acts in. */
+export interface Viewport {
+ window_ref: string;
+ name: string;
+ /** In pixels of the window's full-resolution capture, from the window's top-left corner, so it follows the window. */
+ rect: PixelBox;
+}
+
+/** The size of a viewport's image: its full-resolution size, scaled down to fit VIEWPORT_CAP. Input points on the viewport are its pixels. */
+export function viewportImageSize(rect: PixelBox): { width: number; height: number } {
+ const scale = Math.min(1, VIEWPORT_CAP / Math.max(rect.width, rect.height));
+ return { width: Math.max(1, Math.round(rect.width * scale)), height: Math.max(1, Math.round(rect.height * scale)) };
+}
+
+/** Where a point of the viewport's image lies in the window's full-resolution pixels; undefined outside the image. */
+export function windowPoint(rect: PixelBox, point: Point): Point | undefined {
+ const image = viewportImageSize(rect);
+ if (point.x < 0 || point.y < 0 || point.x >= image.width || point.y >= image.height) return undefined;
+ return { x: Math.min(rect.x + rect.width - 1, Math.round(rect.x + point.x * rect.width / image.width)),
+ y: Math.min(rect.y + rect.height - 1, Math.round(rect.y + point.y * rect.height / image.height)) };
+}
+
+/** Where a point in the window's full-resolution pixels lies in the viewport's image. */
+export function viewportPoint(rect: PixelBox, point: Point): Point {
+ const image = viewportImageSize(rect);
+ return { x: (point.x - rect.x) * image.width / rect.width, y: (point.y - rect.y) * image.height / rect.height };
+}
+
+/** The part of a viewport inside a window capture of this size; undefined when too little of it is inside. */
+export function clipViewport(rect: PixelBox, size: { width: number; height: number }): PixelBox | undefined {
+ const x = Math.max(0, rect.x), y = Math.max(0, rect.y);
+ const width = Math.min(size.width, rect.x + rect.width) - x, height = Math.min(size.height, rect.y + rect.height) - y;
+ return width >= MIN_SIDE && height >= MIN_SIDE ? { x, y, width, height } : undefined;
+}
+
+/**
+ * A viewport rectangle given in pixels of the window's latest image, which can be scaled down from the window's full
+ * resolution by `scale` (full-resolution pixels per image pixel on each axis). Missing fields keep `current`.
+ */
+export function viewportRect(given: Partial<PixelBox>, scale: Point, current?: PixelBox): PixelBox {
+ const pick = (key: keyof PixelBox, factor: number) => given[key] !== undefined ? Math.round(given[key]! * factor) : current?.[key];
+ const rect = { x: pick("x", scale.x), y: pick("y", scale.y), width: pick("width", scale.x), height: pick("height", scale.y) };
+ if (rect.x === undefined || rect.y === undefined || rect.width === undefined || rect.height === undefined)
+ throw Error("A new viewport needs x, y, width and height.");
+ if (rect.x < 0 || rect.y < 0 || rect.width < MIN_SIDE || rect.height < MIN_SIDE)
+ throw Error(`A viewport starts at x, y of 0 or more and is at least ${MIN_SIDE} full-resolution pixels wide and high.`);
+ return rect as PixelBox;
+}
+
+/** The size of a PNG image. */
+export async function pngSize(png: Buffer): Promise<{ width: number; height: number }> {
+ const { width, height } = await sharp(png).metadata();
+ if (!width || !height) throw Error("The window image has no size.");
+ return { width, height };
+}
+
+/** Cut a viewport's image out of a full-resolution window capture. */
+export async function viewportImage(png: Buffer, rect: PixelBox): Promise<Buffer> {
+ const { width, height } = viewportImageSize(rect);
+ const crop = sharp(png, { limitInputPixels: 32 * 1024 * 1024 }).extract({ left: rect.x, top: rect.y, width: rect.width, height: rect.height });
+ return (width === rect.width && height === rect.height ? crop : crop.resize(width, height, { kernel: "lanczos3" })).png().toBuffer();
+}
diff --git a/test/computer-address.test.ts b/test/computer-address.test.ts
index 2b753cf..d7ed44f 100644
--- a/test/computer-address.test.ts
+++ b/test/computer-address.test.ts
@@ -24,7 +24,7 @@ describe("computer resource addresses", () => {
"computer:window/window-1?delivery=foreground", "computer:observation/snapshot?delivery=automatic",
"computer:observation/snapshot?delay_ms=1.5", "computer:observation/snapshot?delay_ms=201",
"computer:observation/snapshot#e4", "computer:observation/%2Fother", "computer:observation/%00",
- "computer:windows", "computer:machine/local/files", "computer:window/w-1/apps", "computer:machine/local/windows/x",
+ "computer:windows", "computer:machine/local/files", "computer:window/w-1/pano/x", "computer:window/w-1/Pano", "computer:window/w-1/pano?x=1", "computer:machine/local/windows/x",
]) expect(() => parseComputerAddress(source), source).toThrow();
});
});
diff --git a/test/computer-tools.test.ts b/test/computer-tools.test.ts
index 9365f38..73061fa 100644
--- a/test/computer-tools.test.ts
+++ b/test/computer-tools.test.ts
@@ -280,7 +280,7 @@ test("Delete reports a window that stays open and refuses what it cannot close",
expect((await kept.remove("computer:machine/local?force=true")).summary).toContain("?force=true applies to window addresses only.");
// A malformed address is refused before anything is sent, like an invalid field.
expect((await kept.remove("computer:display/primary")).script).toMatchObject({ outcome: "refused", effect: "not-applied" });
- expect((await kept.remove("computer:machine/local/windows")).summary).toContain("Delete closes a window, a sandbox or isolated session, a machine or computer:session.");
+ expect((await kept.remove("computer:machine/local/windows")).summary).toContain("Delete closes a window, a sandbox or isolated session, a machine or computer:session, or removes a viewport.");
// Safe mode closes only windows this session opened, like any other input.
const safe = fixture("closed");
diff --git a/test/viewport.test.ts b/test/viewport.test.ts
new file mode 100644
index 0000000..d50d735
--- /dev/null
+++ b/test/viewport.test.ts
@@ -0,0 +1,177 @@
+import sharp from "sharp";
+import { describe, expect, test, vi } from "vitest";
+import { computerSource, parseComputerAddress } from "../src/computer-address.ts";
+import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
+import { presentComputerResult } from "../src/provider.ts";
+import { ComputerSession, type ComputerResult, type Content, type Driver } from "../src/session.ts";
+import type { ShortAddresses } from "../src/short-addresses.ts";
+import { clipViewport, VIEWPORT_CAP, viewportImageSize, viewportPoint, viewportRect, windowPoint } from "../src/viewport.ts";
+
+const cwd = "/synthetic-viewports";
+
+// LPT-931: the agent names a rectangle of a window, reads it at the window's own resolution and acts in its pixels.
+describe("viewport geometry", () => {
+ test("a viewport keeps the window's resolution up to the cap, and points map both ways", () => {
+ expect(viewportImageSize({ x: 0, y: 0, width: 300, height: 200 })).toEqual({ width: 300, height: 200 });
+ expect(viewportImageSize({ x: 0, y: 0, width: 3136, height: 1000 })).toEqual({ width: VIEWPORT_CAP, height: 500 });
+ const rect = { x: 100, y: 40, width: 3136, height: 1000 };
+ expect(windowPoint(rect, { x: 10, y: 20 })).toEqual({ x: 120, y: 80 });
+ expect(viewportPoint(rect, { x: 120, y: 80 })).toEqual({ x: 10, y: 20 });
+ expect(windowPoint(rect, { x: VIEWPORT_CAP, y: 0 })).toBeUndefined();
+ expect(windowPoint({ x: 5, y: 5, width: 50, height: 50 }, { x: 49, y: 49 })).toEqual({ x: 54, y: 54 });
+ });
+
+ test("a rectangle in the agent's image becomes full-resolution pixels, and missing fields keep the current ones", () => {
+ expect(viewportRect({ x: 50, y: 50, width: 100, height: 60 }, { x: 2, y: 2 })).toEqual({ x: 100, y: 100, width: 200, height: 120 });
+ expect(viewportRect({ x: 10 }, { x: 2, y: 2 }, { x: 100, y: 100, width: 200, height: 120 })).toEqual({ x: 20, y: 100, width: 200, height: 120 });
+ expect(() => viewportRect({ x: 1, y: 1, width: 10 }, { x: 1, y: 1 })).toThrow("A new viewport needs x, y, width and height.");
+ expect(() => viewportRect({ x: 1, y: 1, width: 2, height: 50 }, { x: 1, y: 1 })).toThrow("at least 8");
+ expect(clipViewport({ x: 900, y: 600, width: 300, height: 100 }, { width: 1000, height: 640 })).toEqual({ x: 900, y: 600, width: 100, height: 40 });
+ expect(clipViewport({ x: 995, y: 0, width: 300, height: 100 }, { width: 1000, height: 640 })).toBeUndefined();
+ });
+
+ test("a viewport address is its window's address with the viewport's name after it", () => {
+ expect(parseComputerAddress("computer:window/w-1/pano")).toEqual({ kind: "viewport", source: "computer:window/w-1/pano", ref: "w-1", name: "pano" });
+ });
+});
+
+/** A 1000x640 window whose captures honour max_image_dimension, with a blue mark at 150,150 and a dot wherever a click lands. */
+async function fixture() {
+ const FULL = { width: 1000, height: 640 };
+ const dots: { x: number; y: number }[] = [];
+ const scales = new Map<string, number>();
+ let serial = 0, last = 1;
+ const frame = async (scale: number) => {
+ const shapes = [`<rect x="150" y="150" width="10" height="10" fill="#00f"/>`, ...dots.map(dot => `<rect x="${dot.x - 3}" y="${dot.y - 3}" width="6" height="6" fill="#c00"/>`)];
+ const full = await sharp({ create: { ...FULL, channels: 3, background: "#fff" } })
+ .composite([{ input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${FULL.width}" height="${FULL.height}">${shapes.join("")}</svg>`) }]).png().toBuffer();
+ const png = scale === 1 ? full : await sharp(full).resize(Math.round(FULL.width * scale), Math.round(FULL.height * scale), { kernel: "nearest" }).png().toBuffer();
+ return { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
+ };
+ const driver: Driver = {
+ identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),
+ call: vi.fn<Driver["call"]>(async (name, args) => {
+ const input = args as { x?: number; y?: number; capture_id?: string; include_screenshot?: boolean; include_accessibility_tree?: boolean; max_image_dimension?: number; element_token?: string };
+ if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };
+ if (name === "get_window_state") {
+ serial++;
+ const pixels = input.include_screenshot !== false, ui = input.include_accessibility_tree !== false;
+ const scale = Math.min(1, (input.max_image_dimension ?? 1280) / FULL.width);
+ if (pixels) { scales.set(`c${serial}`, scale); last = scale; }
+ const width = Math.round(FULL.width * scale), height = Math.round(FULL.height * scale);
+ return { content: pixels ? [await frame(scale)] : [], structuredContent: { pid: 42, window_id: 91, window_bounds: { x: 80, y: 90, ...FULL },
+ ...(pixels ? { capture_id: `c${serial}`, screenshot_width: width, screenshot_height: height } : {}),
+ ...(ui ? { snapshot_id: `s${serial}`, elements_complete: true, elements: [{ element_token: `s${serial}:0`, role: "Button", label: "Paint", enabled: true, actions: ["invoke"],
+ frame: { x: 0, y: 0, w: 40, h: 20 }, ...(pixels ? { screenshot_frame: { x: 0, y: 0, w: Math.round(40 * scale), h: Math.round(20 * scale) } } : {}) }] } : {}) } };
+ }
+ // A pixel click paints a dot where it lands, in the window's full-resolution pixels; the button paints one at 900,600.
+ if (name === "click") {
+ if (input.element_token) dots.push({ x: 900, y: 600 });
+ else {
+ const scale = (input.capture_id ? scales.get(input.capture_id) : undefined) ?? last;
+ dots.push({ x: Math.round(input.x! / scale), y: Math.round(input.y! / scale) });
+ }
+ return { content: [], structuredContent: { effect: "confirmed" } };
+ }
+ throw Error(`Unexpected native call ${name}`);
+ }),
+ };
+ const session = new ComputerSession(driver, () => "yolo");
+ const resources = new ComputerResources(async () => session, async () => session.close());
+ const ref = ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const window = computerSource("window", ref);
+ // The agent's view of the window: an overview half the window's size.
+ const overview = async () => (await resources.read(requireComputerAddress(window), cwd, ["full", 'capture:{"maxLongEdge":500}']))!;
+ return { resources, session, ref, window, dots, overview, pano: requireComputerAddress(`${window}/pano`) };
+}
+const images = (result: ComputerResult) => result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
+const pixel = async (image: Extract<Content, { type: "image" }>, x: number, y: number) => {
+ const { data, info } = await sharp(Buffer.from(image.data, "base64")).raw().toBuffer({ resolveWithObject: true });
+ return [...data.subarray((y * info.width + x) * info.channels, (y * info.width + x) * info.channels + 3)];
+};
+const size = async (image: Extract<Content, { type: "image" }>) => {
+ const { width, height } = await sharp(Buffer.from(image.data, "base64")).metadata();
+ return { width, height };
+};
+
+describe("viewports of a window", () => {
+ test("computer_view takes a rectangle of the agent's image and shows it at the window's own resolution", async () => {
+ const f = await fixture();
+ await f.overview();
+ const made = await f.resources.view(f.pano, cwd, { x: 50, y: 50, width: 100, height: 60 });
+ expect(made.details.viewport).toMatchObject({ name: "pano", window_ref: f.ref, rect: { x: 100, y: 100, width: 200, height: 120 }, image: { width: 200, height: 120 } });
+ const [image] = images(made);
+ expect(await size(image)).toEqual({ width: 200, height: 120 });
+ // The mark at 150,150 of the window is at 50,50 of the viewport, one window pixel per image pixel.
+ expect(await pixel(image, 52, 52)).toEqual([0, 0, 255]);
+ const text = presentComputerResult(made);
+ expect(text).toContain(`${f.window}/pano is 100x60 at 50,50 of the window's latest image. Its image is 200x120, the window's own resolution.`);
+ expect(text).toContain(`Viewport: ${f.window}/pano; points on it are pixels of its 200x120 image`);
+ // Moving keeps the size; a Read shows the moved rectangle.
+ await f.resources.view(f.pano, cwd, { x: 60 });
+ const read = (await f.resources.read(f.pano, cwd, []))!;
+ expect(read.details.viewport?.rect).toEqual({ x: 120, y: 100, width: 200, height: 120 });
+ expect(await pixel(images(read)[0], 32, 52)).toEqual([0, 0, 255]);
+ });
+
+ test("a viewport Read only looks: the window's refs stay current for input", async () => {
+ const f = await fixture();
+ const view = await f.overview();
+ await f.resources.view(f.pano, cwd, { x: 50, y: 50, width: 100, height: 60 });
+ await f.resources.read(f.pano, cwd, []);
+ const hint = view.details.ui!.elements[0].hint;
+ const clicked = await f.resources.input(requireComputerAddress(f.window), cwd, { label: "click", target: hint, native: { action: "click" }, web: "" });
+ expect(clicked.details.status).toBe("delivered");
+ expect(f.dots).toEqual([{ x: 900, y: 600 }]);
+ });
+
+ test("a point in a viewport lands at that pixel of the viewport, and the change list covers only the viewport", async () => {
+ const f = await fixture();
+ await f.overview();
+ await f.resources.view(f.pano, cwd, { x: 50, y: 50, width: 100, height: 60 });
+ const clicked = await f.resources.input(f.pano, cwd, { label: "click", point: { x: 10, y: 20 }, native: { action: "click" }, web: "" });
+ expect(f.dots).toEqual([{ x: 110, y: 120 }]);
+ expect(clicked.details.viewport).toMatchObject({ source: `${f.window}/pano`, window_ref: f.ref });
+ expect(clicked.details.successor?.diff?.pixels).toMatchObject({ space: "viewport", width: 200, height: 120, box: { x: 7, y: 17, width: 6, height: 6 } });
+ const text = presentComputerResult(clicked);
+ expect(text).toContain(`Viewport: ${f.window}/pano`);
+ expect(text).toMatch(/Pixels changed in 1 area \(pixels of the viewport's image; [\d.]+% of the viewport\)/);
+ // A point of the window's latest image keeps its meaning after input in a viewport.
+ await f.resources.input(requireComputerAddress(f.window), cwd, { label: "click", point: { x: 250, y: 160 }, native: { action: "click" }, web: "" });
+ expect(f.dots.at(-1)).toEqual({ x: 500, y: 320 });
+ });
+
+ test("a point outside the viewport's image and space refuse before anything is sent", async () => {
+ const f = await fixture();
+ await f.overview();
+ await f.resources.view(f.pano, cwd, { x: 50, y: 50, width: 100, height: 60 });
+ await expect(f.resources.input(f.pano, cwd, { label: "click", point: { x: 200, y: 5 }, native: { action: "click" }, web: "" }))
+ .rejects.toThrow(`200,5 lies outside ${f.window}/pano: its image is 200x120.`);
+ await expect(f.resources.input(f.pano, cwd, { label: "click", point: { x: 5, y: 5 }, space: "window", native: { action: "click" }, web: "" }))
+ .rejects.toThrow("space does not apply in a viewport");
+ expect(f.dots).toEqual([]);
+ });
+
+ test("viewports resolve by window name and short address, list in the session and go away with Delete", async () => {
+ const f = await fixture();
+ await f.overview();
+ await f.resources.name(requireComputerAddress(f.window), cwd, "game");
+ expect(f.resources.address("computer:game/pano")).toEqual(f.pano);
+ await expect(f.resources.read(f.resources.address("computer:game/pano"), cwd, [])).rejects.toThrow("is no viewport yet");
+ await f.resources.view(f.resources.address("computer:game/pano"), cwd, { x: 50, y: 50, width: 100, height: 60 });
+ const tag = (f.resources as unknown as { shorts: ShortAddresses }).shorts.tag;
+ expect(f.resources.shorten(`${f.window}/pano`)).toBe(`computer:local/w1${tag}/pano`);
+ expect(f.resources.address(`computer:local/w1${tag}/pano`)).toEqual(f.pano);
+ expect(f.resources.descriptor(requireComputerAddress("computer:session"))).toContain(`- ${f.window}/pano: 200x120 at 100,100 in full-resolution window pixels`);
+ const removed = await f.resources.delete("computer:game/pano", cwd);
+ expect(removed.details.status).toBe("closed");
+ await expect(f.resources.read(f.pano, cwd, [])).rejects.toThrow("is no viewport yet");
+ });
+
+ test("a new viewport needs its whole rectangle inside the window", async () => {
+ const f = await fixture();
+ await f.overview();
+ await expect(f.resources.view(f.pano, cwd, { x: 10, y: 10 })).rejects.toThrow("A new viewport needs x, y, width and height.");
+ await expect(f.resources.view(f.pano, cwd, { x: 499, y: 10, width: 50, height: 50 })).rejects.toThrow("lies outside the window: its latest image is 500x320.");
+ });
+});