Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Watch a window or viewport in the background while acting (LPT-930)

alexshpunt-clanker[bot]f74fd94e75bd
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index c320a31..8c6c9d8 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
import { parseComputerAddress, computerSource, displaySource, viewportSource, ADDRESS_KINDS, NAME, type ComputerAddress } from "./computer-address.ts";
import { clipViewport, pngSize, viewportImage, viewportImageSize, viewportPoint, viewportRect, windowPoint, VIEWPORT_CAP, type Viewport } from "./viewport.ts";
import { FULL_RESOLUTION } from "./cua-image-settings.ts";
+import { WATCH_DEFAULTS } from "./watch.ts";
import type { PixelBox } from "./pixel-diff.ts";
import { ShortAddresses, SHORT_REFERENCE } from "./short-addresses.ts";
import { parseComputerViews, type ComputerViews } from "./computer-views.ts";
@@ -979,7 +980,9 @@ export class ComputerResources {
if (management && state.details.window_state_kind !== "management")
return this.stamp(place, { ...state, content: [{ type: "text", text: "This native runtime can only restore and activate windows, not move, resize, minimize, maximize or focus them." }],
details: { ...state.details, status: "refused" } });
- return this.stamp(place, management ? await session.manageWindow(address.ref, id, management, signal) : await session.restoreWindow(address.ref, id, signal));
+ const started = performance.now();
+ const result = management ? await session.manageWindow(address.ref, id, management, signal) : await session.restoreWindow(address.ref, id, signal);
+ return this.stamp(place, ComputerResources.watched(result, session.markWatches(address.ref, String(command.action), started, performance.now())));
}
/**
* Run one page operation (navigation, tabs, dialogs, files, page code) in a browser window by its address. The
@@ -1010,13 +1013,39 @@ export class ComputerResources {
if (viewport && plan.space !== undefined) throw new ComputerRequestError("space does not apply in a viewport: its points are pixels of the viewport's image.");
const place = this.place(address);
const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);
+ const started = performance.now();
let result = await this.inputStep(address.ref, session, plan, cwd, signal, viewport);
// A follow-up such as Enter after typing runs only after the first part was delivered.
if (plan.then && result.details.status === "delivered") {
const next = await this.inputStep(address.ref, session, plan.then, cwd, signal, viewport);
result = { ...next, content: [...result.content, ...next.content] };
}
- return this.stamp(place, result);
+ return this.stamp(place, ComputerResources.watched(result, session.markWatches(address.ref, plan.label, started, performance.now())));
+ }
+ /** An action's receipt with its spans in the running watches of its window. */
+ private static watched(result: ComputerResult, spans: NonNullable<ComputerResult["details"]["watch_spans"]>): ComputerResult {
+ return spans.length ? { ...result, details: { ...result.details, watch_spans: spans } } : result;
+ }
+ /**
+ * Start a watch for computer_record: frames of a window, or of a viewport at its own pixels, captured in the
+ * background while the agent acts. The receipt names the sequence address that grows until watch_stop.
+ */
+ async watch(address: ComputerAddress, cwd: string, options: { fps?: number; timeout_ms?: number; maxLongEdge?: number }, signal?: AbortSignal): Promise<ComputerResult> {
+ if (address.kind !== "window" && address.kind !== "viewport")
+ throw new ComputerRequestError("watch takes a window address, or a viewport address from computer_view such as computer:game/pano to watch part of a window.");
+ const viewport = address.kind === "viewport" ? this.viewport(address) : undefined;
+ const place = this.place(address);
+ const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);
+ const watching = viewport ? viewportSource(viewport.window_ref, viewport.name) : computerSource("window", address.ref);
+ return this.stamp(place, await session.startWatch(address.ref, watching, { fps: options.fps ?? WATCH_DEFAULTS.fps,
+ timeout_ms: options.timeout_ms ?? WATCH_DEFAULTS.timeout_ms, maxLongEdge: options.maxLongEdge ?? WATCH_DEFAULTS.maxLongEdge, ...(viewport ? { crop: viewport.rect } : {}) }, signal));
+ }
+ /** Stop a watch: its contact sheet with actions marked, change statistics and files, in `to` (a workspace folder) when given. */
+ async stopWatch(address: ComputerAddress, cwd: string, options: { to?: string; video?: boolean }, signal?: AbortSignal): Promise<ComputerResult> {
+ if (address.kind !== "sequence") throw new ComputerRequestError("watch_stop takes the sequence address its watch returned.");
+ const place = this.place(address);
+ const { session } = await this.routeAddress(cwd, address, signal);
+ return this.stamp(place, await session.stopWatch(address.ref, { ...(options.to !== undefined ? { dir: resolve(cwd, options.to) } : {}), animation: options.video === true }));
}
private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal, viewport?: Viewport): Promise<ComputerResult> {
const ref = refKind(plan.target);
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index 879601c..17d8e6b 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -220,16 +220,17 @@ export function clipboardCommand(params: Static<typeof clipboardParameters>): Re
}
const recordParameters = Type.Object({
- at: Type.String({ description: "video_start, video_stop and audio: computer:machine/<name>; sequence: a window or display address." }),
- action: Type.Union([Type.Literal("video_start"), Type.Literal("video_stop"), Type.Literal("audio"), Type.Literal("sequence")], { description: "video_start records the machine's main display to video (YOLO) until video_stop or seconds pass; video_stop ends it and can fetch the file; audio records the machine's playback for seconds (YOLO); sequence captures frames of a window or display over duration_ms and returns a sequence address, whose frames Read shows with a frames: view." }),
+ at: Type.String({ description: "video_start, video_stop and audio: computer:machine/<name>; sequence: a window or display address; watch: a window address, or a viewport address from computer_view such as computer:game/pano to watch part of a window; watch_stop: the sequence address watch returned." }),
+ action: Type.Union([Type.Literal("video_start"), Type.Literal("video_stop"), Type.Literal("audio"), Type.Literal("sequence"), Type.Literal("watch"), Type.Literal("watch_stop")], { description: "video_start records the machine's main display to video (YOLO) until video_stop or seconds pass; video_stop ends it and can fetch the file; audio records the machine's playback for seconds (YOLO); sequence captures frames of a window or display over duration_ms and returns a sequence address, whose frames Read shows with a frames: view; watch starts capturing a window or viewport in the background and returns a sequence address at once, while the agent keeps acting on the window and each action's receipt names its span and frames in the watch; watch_stop ends a watch and returns a contact sheet with the actions marked, per-frame change statistics and the frames as files." }),
seconds: Type.Optional(Type.Integer({ minimum: 1, maximum: 600, description: "video_start: recording limit, 1..600; audio: recording length, 1..300." })),
- to: Type.Optional(Type.String({ description: "video_stop and audio: workspace path to fetch the recorded file into." })),
+ to: Type.Optional(Type.String({ description: "video_stop and audio: workspace path to fetch the recorded file into; watch_stop: workspace folder for the frames, default a temporary folder removed when the computer session closes." })),
transcribe: Type.Optional(Type.Boolean({ description: "audio: add a speech transcript made by faster-whisper on this machine; it can be wrong." })),
+ video: Type.Optional(Type.Boolean({ description: "watch_stop: also write watch.webp, an animation of the frames in real time." })),
duration_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 60000, description: "sequence: capture length." })),
- fps: Type.Optional(Type.Integer({ minimum: 1, maximum: 30, description: "sequence: frames per second to aim for, default 4; the result reports the rate reached." })),
+ fps: Type.Optional(Type.Integer({ minimum: 1, maximum: 30, description: "sequence and watch: frames per second to aim for, default 4 for sequence and 10 for watch; the result reports the rate reached." })),
max_frames: Type.Optional(Type.Integer({ minimum: 1, maximum: 300, description: "sequence: frame limit, default 40." })),
- timeout_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 90000, description: "sequence: total time limit, at least duration_ms, default 15000." })),
- maxLongEdge: Type.Optional(Type.Integer({ minimum: 64, maximum: 4096, description: "sequence: longest frame edge in pixels." })),
+ timeout_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 300000, description: "sequence: total time limit, at least duration_ms, at most 90000, default 15000; watch: when the watch stops by itself, default 60000." })),
+ maxLongEdge: Type.Optional(Type.Integer({ minimum: 64, maximum: 4096, description: "sequence: longest frame edge in pixels; watch: longest frame edge, default 800, at most the viewport's own pixels." })),
scale: Type.Optional(Type.Number({ minimum: 0.05, maximum: 1, description: "sequence: frame scale instead of maxLongEdge." })),
region: Type.Optional(Type.Object({ x: Type.Integer({ minimum: 0 }), y: Type.Integer({ minimum: 0 }), width: Type.Integer({ minimum: 1 }), height: Type.Integer({ minimum: 1 }) },
{ additionalProperties: false, description: "sequence: part of the window or display to capture, in its local pixels." })),
@@ -237,11 +238,19 @@ const recordParameters = Type.Object({
const SEQUENCE_FIELDS = ["duration_ms", "fps", "max_frames", "timeout_ms", "maxLongEdge", "scale", "region"];
-/**
- * What computer_record runs: a machine command for video and audio, or the Read sequence view of a window or display.
- */
-export function recordCommand(params: Static<typeof recordParameters>): { command: Record<string, unknown> } | { views: string[] } {
+/** What computer_record runs: a machine command for video and audio, the Read sequence view of a window or display, or a watch start or stop. */
+export type RecordRun = { command: Record<string, unknown> } | { views: string[] }
+ | { watch: { fps?: number; timeout_ms?: number; maxLongEdge?: number } } | { stop: { to?: string; video?: boolean } };
+export function recordCommand(params: Static<typeof recordParameters>): RecordRun {
const { at: _at, action, ...fields } = params;
+ if (action === "watch") {
+ fieldsFor(action, fields, ["fps", "timeout_ms", "maxLongEdge"]);
+ return { watch: fields };
+ }
+ if (action === "watch_stop") {
+ fieldsFor(action, fields, ["to", "video"]);
+ return { stop: fields };
+ }
if (action === "sequence") {
fieldsFor(action, fields, SEQUENCE_FIELDS, ["duration_ms"]);
const views = [`sequence:${JSON.stringify(fields)}`];
@@ -586,22 +595,24 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
});
pi.registerTool({
...common, name: "computer_record", label: "Computer record",
- description: "Use computer_record to record a machine's display to video or its playback audio, or to capture frames of a window or display over time for motion and animation.",
+ description: "Use computer_record to record a machine's display to video or its playback audio, to capture frames of a window or display over time for motion and animation, or to watch a window or viewport in the background while acting on it, such as panning a panorama or dragging a 3D camera.",
parameters: recordParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
...rendered("computer_record", args => `${args.at} ${args.action}${args.seconds ? ` ${args.seconds} s` : args.duration_ms ? ` ${args.duration_ms} ms` : ""}`),
async execute(_id, params, signal, _update, ctx) {
- let run: ReturnType<typeof recordCommand> = { command: {} };
+ let run: RecordRun = { command: {} };
return actionResult(await computerAction(resources, params.at, params.action, () => {
run = recordCommand(params);
+ if ("watch" in run || "stop" in run) return resources.address(params.at);
if ("views" in run) {
const address = resources.address(params.at);
if (address.kind !== "window" && address.kind !== "display") throw new ComputerRequestError("sequence captures a window or display address.");
return address;
}
return machineAddress(resources, params.at);
- }, async address => "views" in run
- ? (await resources.sequence(address, ctx.cwd, run.views, signal))!
+ }, async address => "watch" in run ? resources.watch(address, ctx.cwd, run.watch, signal)
+ : "stop" in run ? resources.stopWatch(address, ctx.cwd, run.stop, signal)
+ : "views" in run ? (await resources.sequence(address, ctx.cwd, run.views, signal))!
: resources.perform(address, ctx.cwd, JSON.stringify(run.command), signal)));
},
});
diff --git a/src/contact-sheet.ts b/src/contact-sheet.ts
index 40e3985..bf2b0cf 100644
--- a/src/contact-sheet.ts
+++ b/src/contact-sheet.ts
@@ -7,7 +7,13 @@ export interface FrameRegion { x: number; y: number; width: number; height: numb
* How selected frames are returned: one image each (the default), one tiled contact sheet, or one per-pixel
* maximum or minimum composite. `region` crops every frame first; `maxLongEdge` bounds the returned image.
*/
-export interface FrameRendering { as: "sheet" | "max" | "min"; region?: FrameRegion; columns?: number; maxLongEdge?: number }
+export interface FrameRendering { as: "sheet" | "max" | "min"; region?: FrameRegion; columns?: number; maxLongEdge?: number; marks?: readonly FrameMark[] }
+/** An action during the frames, in their timestamps: a sheet labels the frames it covers with its name on an orange strip. */
+export interface FrameMark { label: string; start_ms: number; end_ms: number }
+/** The labels of the actions a frame's time falls in. */
+export function marksAt(marks: readonly FrameMark[] | undefined, timestamp: number): string[] {
+ return (marks ?? []).filter(mark => timestamp >= mark.start_ms && timestamp <= mark.end_ms).map(mark => mark.label);
+}
/** Bounds of a rendered sheet or composite. */
export const RENDER_LIMITS = { columns: 10, maxLongEdge: 4096, defaultLongEdge: 1600 } as const;
@@ -42,9 +48,9 @@ async function decode(frame: SequenceFrame, region: FrameRegion | undefined): Pr
}
const escape = (value: string) => value.replace(/[<>&]/g, char => ({ "<": "&lt;", ">": "&gt;", "&": "&amp;" })[char]!);
-/** A dark label strip with the frame's index and time, drawn as SVG text. */
-function label(text: string, width: number): Buffer {
- return Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${LABEL_HEIGHT}"><rect width="${width}" height="${LABEL_HEIGHT}" fill="#000"/>` +
+/** A label strip with the frame's index and time, drawn as SVG text: dark, or orange while an action ran. */
+function label(text: string, width: number, action = false): Buffer {
+ return Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${LABEL_HEIGHT}"><rect width="${width}" height="${LABEL_HEIGHT}" fill="${action ? "#c2410c" : "#000"}"/>` +
`<text x="4" y="13" font-family="sans-serif" font-size="12" fill="#fff">${escape(text)}</text></svg>`);
}
@@ -67,7 +73,8 @@ async function sheet(frames: readonly SequenceFrame[], rendering: FrameRendering
const png = await sharp(d.data, { raw: { width: d.width, height: d.height, channels: 3 } })
.resize(tile.width, tile.height, { fit: "contain", background: "#202020", kernel: "lanczos3" }).png().toBuffer();
const { x, y } = tiles[i];
- return [{ input: label(`#${frames[i].index} · ${Math.round(frames[i].timestamp_ms)} ms`, tile.width), left: x, top: y - LABEL_HEIGHT },
+ const actions = marksAt(rendering.marks, frames[i].timestamp_ms);
+ return [{ input: label(`#${frames[i].index} · ${Math.round(frames[i].timestamp_ms)} ms${actions.length ? ` · ${actions.join(", ")}` : ""}`, tile.width, actions.length > 0), left: x, top: y - LABEL_HEIGHT },
{ input: png, left: x, top: y }];
}));
const png = await sharp({ create: { width, height, channels: 3, background: "#202020" } }).composite(layers.flat()).png().toBuffer();
diff --git a/src/driver.ts b/src/driver.ts
index 8fc2b9e..dc25e31 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -296,6 +296,13 @@ export class McpDriver implements Driver {
});
return transport;
}
+ /**
+ * Another driver process with this configuration, for captures alongside this one's input. A guest endpoint takes one
+ * connection, so it has none.
+ */
+ companion(): McpDriver | undefined {
+ return this.config.endpoint ? undefined : new McpDriver(this.config, this.version);
+ }
/** One cancellable native call. This method never retries input. */
async call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult> {
if (["get_window_control_state", "manage_window"].includes(name) && !this.windowManagement)
diff --git a/src/provider.ts b/src/provider.ts
index 7358fef..a1788b5 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -165,6 +165,11 @@ export function presentComputerResult(result: ComputerResult, full = false): str
lines.push(`Sequence: ${computerSource("sequence", sequence.sequence_id)}`,
`Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}; pending_slots=${sequence.pending_slots}`);
}
+ if (d.watch) lines.push(d.watch.running
+ ? `Watch: ${d.watch.watching} runs in the background as ${computerSource("sequence", d.watch.sequence_id)}; computer_record watch_stop at that address stops it`
+ : `Watch: ${d.watch.watching} stopped; ${d.watch.changed_frames ?? 0} frames changed${d.watch.files ? `; files in ${d.watch.files.dir}` : ""}`);
+ for (const span of d.watch_spans ?? [])
+ lines.push(`Watch: ${span.label} ran ${Math.round(span.from_ms)}–${Math.round(span.to_ms)} ms into ${computerSource("sequence", span.sequence_id)}; ${span.frames ? `frames ${span.frames.from}–${span.frames.to}` : "no frame captured during it yet"}`);
if (d.process) lines.push(d.process.responding
? `Process: ${d.process.name} (pid ${d.process.pid}) responds; ${d.process.memory_mb} MB memory, ${d.process.cpu_seconds} s CPU`
: `Process: ${d.process.name} (pid ${d.process.pid}) is NOT responding; ${d.process.memory_mb} MB memory. Capture and input cannot recover a hung app: wait and read again, or close it.`);
@@ -187,6 +192,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str
: d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; refs of this sample are current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly.")
: d.dom ? "Act on the window with the computer_* input tools, a ref such as e2 as target. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot."
: d.window_state_id ? "This Read only looks; computer_window changes the window's state."
+ : d.watch && d.status === "observed" ? "A watch only looks: the window's refs and points stay as they were for input."
: d.viewport && d.status === "observed" ? "Act in this viewport with the computer_* input tools at its address; a viewport Read gives no refs."
: d.status === "observed" ? "Refs of this Read are current for the next input to its window; detail, display and frame Reads give none."
: "Native delivery is not verified application state. Never replay unknown input; Read a fresh window overview/state before another action.");
@@ -281,6 +287,10 @@ export interface ComputerReceipt {
viewport?: string;
/** The sandbox address computer_open started. */
sandbox?: string;
+ /** The sequence address of a watch this call started or stopped. */
+ watch?: string;
+ /** The action's span in each running watch of its window, in that watch's milliseconds, with the frames captured during it. */
+ watch_spans?: { sequence: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }[];
/** The new current observation issued by this call, such as a DOM action's next snapshot. */
observation: string | null;
/** Whether the addressed observation can no longer be used for input. */
@@ -323,6 +333,8 @@ export function computerReceipt(source: string, action: string | null, outcome:
window: typeof d.window_ref === "string" ? computerSource("window", d.window_ref) : null,
...(d.viewport ? { viewport: d.viewport.source } : {}),
...(typeof d.sandbox_ref === "string" ? { sandbox: computerSource("sandbox", d.sandbox_ref) } : {}),
+ ...(d.watch ? { watch: computerSource("sequence", d.watch.sequence_id) } : {}),
+ ...(d.watch_spans ? { watch_spans: d.watch_spans.map(span => ({ sequence: computerSource("sequence", span.sequence_id), from_ms: span.from_ms, to_ms: span.to_ms, frames: span.frames })) } : {}),
observation: next ? computerSource("observation", next) : null,
input_consumed: outcome.inputConsumed, application_effect: "unverified",
reason: failed ? computerResultText(outcome.result).split("\n{", 1)[0] : null,
diff --git a/src/session.ts b/src/session.ts
index ef87de5..5a4de06 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -1,7 +1,12 @@
import { randomUUID, createHash } from "node:crypto";
import { nativeDisplay, selectedDisplaySource, type NativeDisplay } from "./selected-displays.ts";
-import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";
-import { renderFrames, type FrameRendering, type RenderedFrames } from "./contact-sheet.ts";
+import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";
+import { changeLines, differs, frameChanges, framesBetween, sheetFrames, WATCH_LIMITS, writeWatchFiles, type WatchFiles, type WatchSpan } from "./watch.ts";
+import type { PixelBox } from "./pixel-diff.ts";
+import { rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { renderFrames, type FrameMark, type FrameRendering, type RenderedFrames } from "./contact-sheet.ts";
import type { GuideMark } from "./image-guides.ts";
import { createUiResource, projectUi, resolveUi, annotateUi, validateUiCapture, type UiResource, type UiRequest, type UiCaptureOptions } from "./ui-resources.ts";
import { isSemanticAction, semanticReceipt } from "./semantic-actions.ts";
@@ -65,6 +70,11 @@ export interface Driver {
overviewKernel?: CuaVisionSettings["kernel"];
call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;
identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;
+ /**
+ * A second driver process on the same desktop for captures that run while this one acts: a driver answers one call
+ * at a time, so a capture sent during a long drag would wait for its end. Absent where only one connection exists.
+ */
+ companion?(): Driver | undefined;
close(): Promise<void>;
}
/** One complete action or bounded known-target sequence, grounded in one window overview. */
@@ -113,6 +123,10 @@ export interface ComputerResult {
captured?: { image: boolean; ui: boolean };
sequence?: ReturnType<typeof sequenceMetadata>;
selected_frames?: Omit<SequenceFrame, "image">[];
+ /** A watch this result started or stopped: what it watches, whether it still runs, the actions it noted and, once stopped, its files. */
+ watch?: { sequence_id: string; watching: string; running: boolean; marks: FrameMark[]; changed_frames?: number; files?: WatchFiles };
+ /** An action's span in each running watch of its window, in the watch's milliseconds, with the frames captured during it. */
+ watch_spans?: WatchSpan[];
/** How a sheet or composite Read placed the selected frames in its one image. */
rendered_frames?: Omit<RenderedFrames, "png">;
/** Outcome of a bounded wait_for/verify; the conditions as requested, in order. */
@@ -148,6 +162,8 @@ interface WindowEntry { ref: string; target: WindowTarget; identity: string; own
seen?: { width: number; height: number };
windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }
interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }
+/** A watch of a window: its growing sequence, when its clock started, the actions noted in it and how to stop it. */
+interface Watching { window_ref: string; watching: string; sequence?: FrameSequence; started: number; marks: FrameMark[]; stop: AbortController; finished: boolean; done: Promise<FrameSequence> }
/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */
export function textMatcher(query?: string): (text: string) => boolean {
if (!query) return () => true;
@@ -189,6 +205,11 @@ export class ComputerSession {
private recordings = 0;
private audioClips = 0;
private readonly sequences = new SequenceStore();
+ /** Watches by sequence id until stopWatch reports them; they capture through `watcher`, a second driver where one exists. */
+ private readonly watches = new Map<string, Watching>();
+ private watcher?: Driver;
+ /** Temporary folders of stopped watches' files, removed when the session closes. */
+ private readonly watchDirs = new Set<string>();
private uiSerial = 0;
private uiImages: Observation[] = [];
/**
@@ -677,6 +698,160 @@ export class ComputerSession {
}
});
}
+
+ /**
+ * Start watching a window, or with `crop` (a viewport's rectangle in full-resolution window pixels) a part of it.
+ * Frames come from a second driver process in the background, outside the session's queue, so the agent acts
+ * meanwhile and a long drag is seen while it runs. Returns once the first frame is in; the sequence then grows until
+ * stopWatch, its time limit, its frame or byte limit, or the session closing.
+ */
+ async startWatch(window_ref: string, watching: string, options: { fps: number; timeout_ms: number; maxLongEdge: number; crop?: PixelBox }, signal?: AbortSignal): Promise<ComputerResult> {
+ const entry = this.entries.get(window_ref);
+ if (!entry || !this.allowed(entry)) return this.result("refused", "Unknown or unavailable window reference. Search its machine's windows again.");
+ const permitted = () => this.entries.get(window_ref) === entry && this.allowed(entry);
+ const driver = this.watcher ??= this.driver.companion?.() ?? this.driver;
+ // Watch frames come from the window's own pixels at a size that gives the crop maxLongEdge pixels on its long edge.
+ let cap = Math.min(FULL_RESOLUTION, options.maxLongEdge);
+ const fullSize = (state: Record<string, unknown>) => {
+ const width = state.screenshot_original_width, height = state.screenshot_original_height;
+ if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;
+ const fit = Math.min(1, FULL_RESOLUTION / Math.max(width, height));
+ return { width: Math.round(width * fit), height: Math.round(height * fit) };
+ };
+ if (options.crop) {
+ let probe: DriverResult;
+ try { probe = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: 64 }, signal); }
+ catch (error) { return this.result("refused", `The watch did not start: ${error instanceof Error ? error.message : error}`); }
+ const full = probe.isError || !probe.structuredContent ? undefined : fullSize(probe.structuredContent);
+ if (!full) return this.result("refused", "The watch did not start: the window could not be captured.");
+ const long = Math.max(options.crop.width, options.crop.height);
+ cap = Math.max(64, Math.min(FULL_RESOLUTION, Math.ceil(Math.max(full.width, full.height) * Math.min(options.maxLongEdge, long) / long)));
+ }
+ const capture = async (active: AbortSignal) => {
+ if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
+ const nativeStarted = performance.now();
+ const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap }, active);
+ const native_completed_at = performance.now();
+ active.throwIfAborted();
+ if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
+ const state = native.structuredContent;
+ if (native.isError || !state || state.status === "refused" || state.effect === "refused") throw new SequenceStop("capture_failed");
+ // Identity is not asked here: its peer takes one request at a time and the session's own calls use it.
+ if ((state.pid !== undefined && state.pid !== entry.target.pid) || (state.window_id !== undefined && state.window_id !== entry.target.window_id))
+ throw new SequenceStop("target_lost");
+ const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");
+ const context = this.overview(state, "watch-frame", false, images);
+ if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024) throw new SequenceStop("capture_failed");
+ let source = this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target });
+ source.semantic_scope = "none"; source.capabilities = { read: true, input: "none", detail: "unsupported" };
+ let image = images[0];
+ const full = options.crop && fullSize(state);
+ if (options.crop && full) {
+ // The viewport follows the window: its rectangle is relative to the window's top-left corner.
+ const kx = context.width / full.width, ky = context.height / full.height, crop = options.crop;
+ const region = { x: Math.round(crop.x * kx), y: Math.round(crop.y * ky), width: Math.max(1, Math.round(crop.width * kx)), height: Math.max(1, Math.round(crop.height * ky)) };
+ if (region.x >= context.width || region.y >= context.height) throw new SequenceStop("capture_failed");
+ ({ image, source } = await cropSequenceFrame(image, source, region, false));
+ }
+ return { image, source, native_completed_at, native_latency_ms: native_completed_at - nativeStarted };
+ };
+ const stop = new AbortController();
+ const watch: Watching = { window_ref, watching, started: 0, marks: [], stop, finished: false, done: Promise.resolve(undefined as unknown as FrameSequence) };
+ watch.done = collectSequence({ duration_ms: options.timeout_ms, fps: options.fps, max_frames: WATCH_LIMITS.max_frames, timeout_ms: options.timeout_ms + 20_000 }, capture,
+ undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, started: (sequence, at) => {
+ watch.sequence = sequence; watch.started = at;
+ sequence.configuration = { ...this.overviewSettings(), maxLongEdge: cap };
+ this.sequences.put(sequence, true);
+ this.watches.set(sequence.sequence_id, watch);
+ } }).then(sequence => {
+ watch.finished = true;
+ if (this.watches.get(sequence.sequence_id) === watch) this.sequences.put(sequence);
+ return sequence;
+ });
+ // Return with the first frame, so a watch that cannot capture says so now; starting the second driver takes a few seconds.
+ const deadline = performance.now() + 30_000;
+ while (!watch.finished && !watch.sequence?.frames.length && performance.now() < deadline && !signal?.aborted) await delay(20);
+ const sequence = watch.sequence!;
+ if (!sequence?.frames.length) {
+ stop.abort();
+ const ended = await watch.done;
+ this.watches.delete(ended.sequence_id);
+ return this.result("refused", `The watch did not start: ${signal?.aborted ? "cancelled" : ended.stop_reason === "stopped" ? "no frame came within 30 s" : ended.stop_reason}.`);
+ }
+ const size = sequence.frames[0].source.image.dimensions;
+ return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +
+ `Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +
+ `It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,
+ undefined, { ...this.addressed(entry), sequence: sequenceMetadata(sequence), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });
+ }
+
+ /** Note an action on a window in each running watch of it: the action's span in the watch's time and the frames captured during it so far. */
+ markWatches(window_ref: string, label: string, start: number, end: number): WatchSpan[] {
+ const spans: WatchSpan[] = [];
+ for (const watch of this.watches.values()) {
+ if (watch.window_ref !== window_ref || watch.finished || !watch.sequence) continue;
+ const mark = { label, start_ms: start - watch.started, end_ms: end - watch.started };
+ watch.marks.push(mark);
+ spans.push({ sequence_id: watch.sequence.sequence_id, label, from_ms: mark.start_ms, to_ms: mark.end_ms, frames: framesBetween(watch.sequence.frames, mark.start_ms, mark.end_ms) });
+ }
+ return spans;
+ }
+ /** Whether this sequence is a watch that has not been stopped with stopWatch yet. */
+ isWatch(id: string): boolean { return this.watches.has(id); }
+
+ /**
+ * Stop a watch: a contact sheet of the frames that changed with the actions marked, change statistics of every
+ * frame, and the frames as files in `dir` (a temporary folder removed when the session closes, unless given), with an
+ * animated WebP when asked. The sequence keeps its frames for frames: Reads.
+ */
+ async stopWatch(id: string, files: { dir?: string; animation?: boolean } = {}): Promise<ComputerResult> {
+ const watch = this.watches.get(id);
+ if (!watch) return this.result("refused", this.sequences.get(id) ? "This sequence is no running watch; Read it with a frames: view." : "No watch of this session has this address. Watches do not survive reload or close.");
+ watch.stop.abort();
+ const sequence = await watch.done;
+ this.watches.delete(id);
+ const entry = this.entries.get(watch.window_ref);
+ if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") {
+ this.sequences.clear();
+ return this.result("refused", "The watched window is no longer accessible; its frames are withheld.");
+ }
+ const reasons: Record<FrameSequence["stop_reason"], string> = { stopped: "stopped", completed: "stopped by itself at its time limit", frame_limit: "stopped by itself at its frame limit",
+ byte_limit: "stopped by itself at its memory limit", timeout: "stopped by itself at its time limit", cancelled: "cancelled", target_lost: "stopped: the window closed",
+ access_revoked: "stopped: access to the window ended", capture_failed: "stopped: a capture failed" };
+ const frames = sequence.frames;
+ const changes = await frameChanges(frames);
+ const distinct = frames.filter((_, i) => differs(frames, i)).length;
+ const lines = [`Watch of ${watch.watching} ${reasons[sequence.stop_reason]}: ${frames.length} frames over ${(sequence.elapsed_ms / 1000).toFixed(1)} s` +
+ `${sequence.actual_fps ? ` at ${sequence.actual_fps.toFixed(1)} FPS` : ""}; ${distinct} differ from the frame before.`];
+ if (watch.marks.length) lines.push("Actions (ms of the watch):", ...watch.marks.map(mark => {
+ const span = framesBetween(frames, mark.start_ms, mark.end_ms);
+ return `- ${mark.label} ${Math.round(mark.start_ms)}–${Math.round(mark.end_ms)} ms, ${span ? `frames ${span.from}–${span.to}` : "no frame during it"}`;
+ }));
+ const moving = changeLines(changes, watch.marks);
+ lines.push(moving.length ? "Biggest changes against the frame before (share of the frame's pixels, box in its pixels):" : "No frame changed against the one before.", ...moving);
+ const content: Content[] = [];
+ let rendered: Omit<RenderedFrames, "png"> | undefined;
+ if (frames.length) {
+ const { png, ...layout } = await renderFrames(sheetFrames(frames, watch.marks), { as: "sheet", marks: watch.marks });
+ rendered = layout;
+ lines.push(`Contact sheet: ${layout.tiles.length} frames that changed, ${layout.columns}x${layout.rows} tiles in time order, each under "#index · time"; an orange label means an action ran then. Frames: ${layout.tiles.map(tile => `#${tile.index}`).join(", ")}.`);
+ content.push({ type: "image", mimeType: "image/png", data: png.toString("base64") });
+ }
+ const dir = files.dir ?? join(tmpdir(), "pi-computer-use-watch", id);
+ let written: WatchFiles | undefined;
+ try {
+ written = await writeWatchFiles(dir, sequence, watch.marks, changes, { watching: watch.watching }, files.animation);
+ if (!files.dir) this.watchDirs.add(dir);
+ lines.push(`Files in ${dir}: ${written.frames} PNG frames (the frames that differ from the one before), watch.json with every frame's time, file, change and actions${written.animation ? ", and watch.webp, an animation in real time" : ""}.` +
+ (files.dir ? "" : " The folder is removed when the computer session closes."));
+ } catch (error) { lines.push(`The frames were not written to ${dir}: ${error instanceof Error ? error.message : error}`); }
+ lines.push("Read the sequence with a frames: view for other frames.");
+ const result = this.result(frames.length ? "observed" : "refused", lines.join("\n"), undefined,
+ { sequence: sequenceMetadata(sequence), ...(rendered ? { rendered_frames: rendered } : {}),
+ watch: { sequence_id: id, watching: watch.watching, running: false, marks: watch.marks, changed_frames: distinct, ...(written ? { files: written } : {}) } });
+ result.content.push(...content);
+ return result;
+ }
/** Enumerate native Windows monitors in YOLO. References grant read access, never input. */
displays(signal?: AbortSignal): Promise<ComputerResult> {
return this.queue(async () => {
@@ -1856,6 +2031,14 @@ export class ComputerSession {
/** Close session-opened browsers (deleting their profiles), detach from attached ones and release the native runtime. Other windows/files are not closed or saved. */
close(): Promise<ComputerResult> {
return this.queue(async () => {
+ const watches = [...this.watches.values()];
+ this.watches.clear();
+ for (const watch of watches) watch.stop.abort();
+ await Promise.allSettled(watches.map(watch => watch.done));
+ if (this.watcher && this.watcher !== this.driver) await this.watcher.close().catch(() => undefined);
+ this.watcher = undefined;
+ await Promise.allSettled([...this.watchDirs].map(dir => rm(dir, { recursive: true, force: true })));
+ this.watchDirs.clear();
this.entries.clear(); this.displayEntries.clear(); this.displayOverview = undefined; this.uiImages = []; this.sequences.clear();
const browsers = [...this.browsers.values()];
this.browsers.clear();
diff --git a/src/temporal-capture.ts b/src/temporal-capture.ts
index 8fc8a8d..03c3501 100644
--- a/src/temporal-capture.ts
+++ b/src/temporal-capture.ts
@@ -23,7 +23,8 @@ export interface FrameSequence {
/** How old the newest frame was when the sequence returned: time from its native completion to the result. */
last_frame_age_ms: number | null;
skipped_slots: number[]; pending_slots: number; payload_bytes: number;
- stop_reason: "completed" | "frame_limit" | "byte_limit" | "cancelled" | "timeout" | "target_lost" | "access_revoked" | "capture_failed";
+ /** stopped: a watch stopped by request, with every frame it captured until then. */
+ stop_reason: "completed" | "stopped" | "frame_limit" | "byte_limit" | "cancelled" | "timeout" | "target_lost" | "access_revoked" | "capture_failed";
frames: SequenceFrame[];
configuration?: CuaVisionSettings;
interval_ms?: { min: number; max: number; mean: number };
@@ -35,12 +36,13 @@ const byteBudget = 48 * 1024 * 1024;
export class SequenceStop extends Error {
constructor(readonly reason: FrameSequence["stop_reason"]) { super(reason); }
}
+/** Bounds of a capture: its length, rate, frame count and total time. */
+export interface CaptureLimits { duration_ms: number; fps: number; max_frames: number; timeout_ms: number }
/** Limits of one sequence: a minute of frames at up to 30 FPS; one Read still selects at most 40 of them. */
export const SEQUENCE_LIMITS = { duration_ms: 60_000, fps: 30, max_frames: 300, timeout_ms: 90_000, selected: 40 } as const;
/** Validate before native dispatch; requested FPS is a sampling goal, never a performance promise. */
-export function validateSequence(options: SequenceOptions): FrameSequence["requested"] {
+export function validateSequence(options: SequenceOptions, L: CaptureLimits = SEQUENCE_LIMITS): FrameSequence["requested"] {
const result = { duration_ms: options.duration_ms, fps: options.fps ?? 4, max_frames: options.max_frames ?? 40, timeout_ms: options.timeout_ms ?? 15000 };
- const L = SEQUENCE_LIMITS;
if (!Number.isSafeInteger(result.duration_ms) || result.duration_ms < 1 || result.duration_ms > L.duration_ms ||
!Number.isFinite(result.fps) || result.fps < 1 || result.fps > L.fps ||
!Number.isSafeInteger(result.max_frames) || result.max_frames < 1 || result.max_frames > L.max_frames ||
@@ -52,15 +54,28 @@ export function validateSequence(options: SequenceOptions): FrameSequence["reque
return result;
}
-/** Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. */
+/** What a long-running capture such as a watch adds: its own limits, a stop request that keeps the frames, and its sequence as soon as it starts. */
+export interface CaptureHooks {
+ limits?: CaptureLimits;
+ /** Ends sampling with stop_reason "stopped"; frames captured until then stay. */
+ stop?: AbortSignal;
+ /** Receives the sequence, which then grows frame by frame, and the clock time its timestamps count from. */
+ started?(sequence: FrameSequence, at: number): void;
+}
+/**
+ * Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. A frame with the
+ * same bytes as the one before shares its image and adds nothing to the byte budget, so a still window costs little.
+ */
export async function collectSequence(options: SequenceOptions,
capture: (signal: AbortSignal) => Promise<{ image: SequenceFrame["image"]; source: ObservationSource; native_completed_at?: number; native_latency_ms?: number }>, signal?: AbortSignal,
- clock: { now(): number; wait(ms: number, active: AbortSignal): Promise<void> } = { now: () => performance.now(), wait: (ms, active) => delay(ms, undefined, { signal: active }) }): Promise<FrameSequence> {
- const requested = validateSequence(options); const started = clock.now();
+ clock: { now(): number; wait(ms: number, active: AbortSignal): Promise<void> } = { now: () => performance.now(), wait: (ms, active) => delay(ms, undefined, { signal: active }) },
+ hooks: CaptureHooks = {}): Promise<FrameSequence> {
+ const requested = validateSequence(options, hooks.limits); const started = clock.now();
const timeout = AbortSignal.timeout(requested.timeout_ms);
- const active = signal ? AbortSignal.any([signal, timeout]) : timeout;
+ const active = AbortSignal.any([timeout, ...(signal ? [signal] : []), ...(hooks.stop ? [hooks.stop] : [])]);
const sequence: FrameSequence = { sequence_id: "seq-" + randomUUID(), requested, started_at: new Date().toISOString(),
elapsed_ms: 0, sample_span_ms: 0, actual_fps: null, last_frame_age_ms: null, skipped_slots: [], pending_slots: 0, payload_bytes: 0, stop_reason: "completed", frames: [] };
+ hooks.started?.(sequence, started);
const interval = 1000 / requested.fps; const slots = Math.ceil(requested.duration_ms / interval);
let slot = 0;
try {
@@ -79,17 +94,21 @@ export async function collectSequence(options: SequenceOptions,
const before = clock.now(); const result = await capture(active); const after = clock.now();
active.throwIfAborted();
const bytes = Buffer.from(result.image.data, "base64");
- if (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget) { sequence.stop_reason = "byte_limit"; break; }
+ const hash = createHash("sha256").update(bytes).digest("hex");
const previous = sequence.frames.at(-1);
const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });
+ const geometryChanged = !!previous && geometry(previous.source) !== geometry(result.source);
+ const repeated = !!previous && previous.hash === hash && !geometryChanged;
+ if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; break; }
sequence.frames.push({ index: sequence.frames.length, slot, timestamp_ms: (result.native_completed_at ?? after) - started, capture_started_ms: before - started,
- capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash: createHash("sha256").update(bytes).digest("hex"), bytes: bytes.length,
- source: result.source, geometry_changed: !!previous && geometry(previous.source) !== geometry(result.source), image: result.image });
- sequence.payload_bytes += bytes.length; slot++;
+ capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,
+ source: result.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result.image });
+ if (!repeated) sequence.payload_bytes += bytes.length;
+ slot++;
while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);
}
} catch (error) {
- sequence.stop_reason = signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";
+ sequence.stop_reason = hooks.stop?.aborted ? "stopped" : signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";
}
sequence.elapsed_ms = clock.now() - started;
sequence.last_frame_age_ms = sequence.frames.length ? sequence.elapsed_ms - sequence.frames.at(-1)!.timestamp_ms : null;
@@ -147,14 +166,21 @@ export async function cropSequenceFrame(image: SequenceFrame["image"], source: O
coordinate_space: "detail_image_pixels" as const, dimensions: { width: applied.width, height: applied.height }, origin,
crop: { requested, applied, coordinate_space: "uncropped_image_pixels" as const } } } };
}
-/** Private bounded frame resources. Reload/close drops pixels; readers must recheck source access. */
+/**
+ * Private bounded frame resources. Reload/close drops pixels; readers must recheck source access. A running watch is
+ * never evicted; once it stops, putting it again makes it an ordinary sequence.
+ */
export class SequenceStore {
private readonly sequences = new Map<string, FrameSequence>();
- put(sequence: FrameSequence): void {
+ private readonly running = new Set<string>();
+ put(sequence: FrameSequence, running = false): void {
+ this.sequences.delete(sequence.sequence_id);
this.sequences.set(sequence.sequence_id, sequence);
- while (this.sequences.size > 3 || [...this.sequences.values()].reduce((sum, s) => sum + s.payload_bytes, 0) > byteBudget)
- this.sequences.delete(this.sequences.keys().next().value!);
+ if (running) this.running.add(sequence.sequence_id); else this.running.delete(sequence.sequence_id);
+ const evictable = () => [...this.sequences.keys()].filter(id => !this.running.has(id));
+ while (evictable().length && (evictable().length > 3 || [...this.sequences.values()].reduce((sum, s) => sum + s.payload_bytes, 0) > byteBudget))
+ this.sequences.delete(evictable()[0]);
}
get(id: string): FrameSequence | undefined { return this.sequences.get(id); }
- clear(): void { this.sequences.clear(); }
+ clear(): void { this.sequences.clear(); this.running.clear(); }
}
diff --git a/src/watch.ts b/src/watch.ts
new file mode 100644
index 0000000..113a0b2
--- /dev/null
+++ b/src/watch.ts
@@ -0,0 +1,133 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import sharp from "sharp";
+import { marksAt, type FrameMark } from "./contact-sheet.ts";
+import { decodeFrame, diffFrames, type PixelBox, type PixelFrame } from "./pixel-diff.ts";
+import type { CaptureLimits, FrameSequence, SequenceFrame } from "./temporal-capture.ts";
+
+/**
+ * A watch captures a window or a viewport in the background while the agent acts, until it is stopped, its time
+ * limit passes or the session closes. Frames that repeat the one before share its image, so a still window costs
+ * little; the frame limit bounds a busy one.
+ */
+export const WATCH_LIMITS: CaptureLimits = { duration_ms: 300_000, fps: 30, max_frames: 3000, timeout_ms: 320_000 };
+/** Defaults of a watch: 10 FPS for a minute, frames at most 800 px on their long edge. */
+export const WATCH_DEFAULTS = { fps: 10, timeout_ms: 60_000, maxLongEdge: 800 } as const;
+/** Frames on the contact sheet a stopped watch returns. */
+export const WATCH_SHEET_FRAMES = 40;
+/** Frames of the animation, and its long edge. */
+export const ANIMATION_LIMITS = { frames: 200, longEdge: 640 } as const;
+
+/** What a receipt says about one watch of the window an action ran on: the action's span in the watch's time and the frames captured in it. */
+export interface WatchSpan { sequence_id: string; label: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }
+
+/** The first and last frame captured within a span of the watch's time; null when none was. */
+export function framesBetween(frames: readonly Pick<SequenceFrame, "index" | "timestamp_ms">[], from: number, to: number): { from: number; to: number } | null {
+ const inside = frames.filter(frame => frame.timestamp_ms >= from && frame.timestamp_ms <= to);
+ return inside.length ? { from: inside[0].index, to: inside.at(-1)!.index } : null;
+}
+
+/** Whether a frame differs from the one before it: the first frame, other bytes or another size. */
+export const differs = (frames: readonly SequenceFrame[], i: number) => i === 0 || frames[i].hash !== frames[i - 1].hash || frames[i].geometry_changed;
+
+/**
+ * The frames of a watch's contact sheet, in time order: every frame that differs from the one before, plus the last.
+ * When there are more than `limit`, the first, the last and those around each action (the first changed frame at or
+ * after it starts, the last during it and the first after it) stay, and the rest are spread evenly over the watch.
+ */
+export function sheetFrames(frames: readonly SequenceFrame[], marks: readonly FrameMark[], limit = WATCH_SHEET_FRAMES): SequenceFrame[] {
+ if (!frames.length) return [];
+ const changed = frames.filter((_, i) => differs(frames, i));
+ if (changed.at(-1) !== frames.at(-1)) changed.push(frames.at(-1)!);
+ if (changed.length <= limit) return changed;
+ const keep = new Set<SequenceFrame>([changed[0], changed.at(-1)!]);
+ for (const mark of marks) {
+ const during = changed.filter(frame => frame.timestamp_ms >= mark.start_ms && frame.timestamp_ms <= mark.end_ms);
+ const after = changed.find(frame => frame.timestamp_ms > mark.end_ms);
+ for (const frame of [during[0], during.at(-1), after]) if (frame && keep.size < limit) keep.add(frame);
+ }
+ const rest = changed.filter(frame => !keep.has(frame));
+ const room = limit - keep.size;
+ for (let i = 0; i < room && rest.length; i++) keep.add(rest[Math.floor((i + 0.5) * rest.length / room)]);
+ return changed.filter(frame => keep.has(frame));
+}
+
+/** How one frame differs from the one before it, in its own pixels: share of changed pixels and the box around them. */
+export interface FrameChange { index: number; timestamp_ms: number; share: number; box?: PixelBox; size_changed?: true }
+
+/** Change statistics of every frame against the one before it; repeated frames changed nothing. */
+export async function frameChanges(frames: readonly SequenceFrame[]): Promise<FrameChange[]> {
+ const changes: FrameChange[] = [];
+ let previous: PixelFrame | undefined;
+ for (let i = 0; i < frames.length; i++) {
+ const frame = frames[i];
+ if (i > 0 && !differs(frames, i)) { changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 }); continue; }
+ const pixels = await decodeFrame(Buffer.from(frame.image.data, "base64"));
+ if (!previous) changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 });
+ else {
+ const diff = diffFrames(previous, pixels);
+ changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: diff.size_changed ? 1 : diff.share,
+ ...(diff.box ? { box: diff.box } : {}), ...(diff.size_changed ? { size_changed: true as const } : {}) });
+ }
+ previous = pixels;
+ }
+ return changes;
+}
+
+/** A short text of the frames that changed most, at most `limit` lines in time order. */
+export function changeLines(changes: readonly FrameChange[], marks: readonly FrameMark[], limit = 20): string[] {
+ const moving = changes.filter(change => change.share > 0);
+ const shown = new Set([...moving].sort((a, b) => b.share - a.share).slice(0, limit));
+ return moving.filter(change => shown.has(change)).map(change => {
+ const actions = marksAt(marks, change.timestamp_ms);
+ const box = change.box ? ` in ${change.box.x},${change.box.y} ${change.box.width}x${change.box.height}` : "";
+ return `- #${change.index} at ${Math.round(change.timestamp_ms)} ms: ${change.size_changed ? "size changed" : `${(change.share * 100).toFixed(1)}%${box}`}${actions.length ? ` (during ${actions.join(", ")})` : ""}`;
+ });
+}
+
+/** The files a stopped watch leaves: each frame that differs from the one before as a PNG, watch.json, and with `animation` watch.webp. */
+export interface WatchFiles { dir: string; frames: number; json: string; animation?: string }
+
+/**
+ * Write a stopped watch's frames for further processing. watch.json lists every frame with its time, its file (shared
+ * by the repeats after it), its change against the frame before and the actions it falls in, and every action.
+ */
+export async function writeWatchFiles(dir: string, sequence: FrameSequence, marks: readonly FrameMark[], changes: readonly FrameChange[],
+ about: Record<string, unknown>, animation = false): Promise<WatchFiles> {
+ await mkdir(dir, { recursive: true });
+ const frames = sequence.frames;
+ const files: string[] = [];
+ let written = 0;
+ for (let i = 0; i < frames.length; i++) {
+ if (differs(frames, i)) {
+ files.push(`frame-${String(frames[i].index).padStart(5, "0")}.png`);
+ await writeFile(join(dir, files[i]), Buffer.from(frames[i].image.data, "base64"));
+ written++;
+ } else files.push(files[i - 1]);
+ }
+ const json = join(dir, "watch.json");
+ await writeFile(json, JSON.stringify({ ...about, sequence_id: sequence.sequence_id, started_at: sequence.started_at, stop_reason: sequence.stop_reason,
+ elapsed_ms: sequence.elapsed_ms, requested: sequence.requested, actual_fps: sequence.actual_fps, skipped_slots: sequence.skipped_slots.length, actions: marks,
+ frames: frames.map((frame, i) => ({ index: frame.index, timestamp_ms: frame.timestamp_ms, file: files[i], width: frame.source.image.dimensions.width,
+ height: frame.source.image.dimensions.height, changed_share: changes[i]?.share ?? null, changed_box: changes[i]?.box ?? null,
+ actions: marksAt(marks, frame.timestamp_ms) })) }, null, 2));
+ return { dir, frames: written, json, ...(animation && frames.length ? { animation: await writeAnimation(join(dir, "watch.webp"), frames) } : {}) };
+}
+
+/**
+ * An animated WebP of the frames that differ from the one before, each shown until the next one, played once in real
+ * time and then looped. More than ANIMATION_LIMITS.frames are spread evenly; all are fitted into one size.
+ */
+async function writeAnimation(path: string, frames: readonly SequenceFrame[]): Promise<string> {
+ let picked = frames.filter((_, i) => differs(frames, i));
+ if (picked.length > ANIMATION_LIMITS.frames)
+ picked = Array.from({ length: ANIMATION_LIMITS.frames }, (_, i) => picked[Math.floor(i * picked.length / ANIMATION_LIMITS.frames)]);
+ const first = picked[0].source.image.dimensions;
+ const scale = Math.min(1, ANIMATION_LIMITS.longEdge / Math.max(first.width!, first.height!));
+ const size = { width: Math.max(1, Math.round(first.width! * scale)), height: Math.max(1, Math.round(first.height! * scale)) };
+ const images = await Promise.all(picked.map(frame => sharp(Buffer.from(frame.image.data, "base64")).removeAlpha()
+ .resize(size.width, size.height, { fit: "contain", background: "#202020" }).png().toBuffer()));
+ const delay = picked.map((frame, i) => Math.max(20, Math.round((picked[i + 1]?.timestamp_ms ?? frame.timestamp_ms + 1000) - frame.timestamp_ms)));
+ await sharp(images, { join: { animated: true } }).webp({ delay, loop: 0, quality: 80 }).toFile(path);
+ return path;
+}
diff --git a/test/fixtures/watch-driver.mjs b/test/fixtures/watch-driver.mjs
new file mode 100644
index 0000000..9e831e1
--- /dev/null
+++ b/test/fixtures/watch-driver.mjs
@@ -0,0 +1,53 @@
+import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
+import sharp from "sharp";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { agentCursor } from "./agent-cursor.mjs";
+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
+
+// A window of 1000x640 with a red square on a lane: a drag moves the square along with the pointer over its duration,
+// like a panorama panned by dragging. Every driver process started with the same log shares the drag through a state
+// file, so a second process captures the square while the first one is still dragging.
+const log = process.argv[2], shared = `${log}.drag.json`;
+const window = { width: 1000, height: 640 };
+const bounds = { x: 80, y: 90, ...window };
+const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });
+const drag = () => { try { return JSON.parse(readFileSync(shared, "utf8")); } catch { return undefined; } };
+/** Where the square is now, in window pixels: at the drag's start before it, moving with it, at its end after it. */
+function square() {
+ const d = drag();
+ if (!d) return 100;
+ const progress = Math.min(1, Math.max(0, (Date.now() - d.start) / Math.max(1, d.duration)));
+ return Math.round(d.from + (d.to - d.from) * progress);
+}
+async function capture(cap) {
+ const scale = Math.min(1, (cap ?? 4096) / Math.max(window.width, window.height));
+ const x = square();
+ const full = await sharp({ create: { ...window, channels: 3, background: "white" } })
+ .composite([{ input: { create: { width: 60, height: 60, channels: 3, background: "red" } }, left: Math.min(window.width - 60, x), top: 170 }]).png().toBuffer();
+ const width = Math.round(window.width * scale), height = Math.round(window.height * scale);
+ return { png: scale === 1 ? full : await sharp(full).resize(width, height).png().toBuffer(), width, height };
+}
+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [] }));
+server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {
+ const cursor = agentCursor(name, args); if (cursor) return cursor;
+ appendFileSync(log, JSON.stringify({ name, arguments: args, pid: process.pid, at: Date.now() }) + "\n");
+ let state; let image;
+ if (name === "fixture_identity") state = { identity: `owned:${args.window_id}` };
+ else if (name === "list_windows") state = { windows: [{ pid: 42, window_id: 91, bounds, title: "Watch fixture" }] };
+ else if (name === "get_window_state") {
+ const shot = await capture(args.max_image_dimension);
+ state = { pid: 42, window_id: 91, window_bounds: bounds, screenshot_width: shot.width, screenshot_height: shot.height,
+ screenshot_original_width: window.width, screenshot_original_height: window.height, capture_id: `capture-${Date.now()}`,
+ elements_complete: true, elements: [] };
+ if (args.include_screenshot !== false) image = shot.png;
+ } else if (name === "drag") {
+ const duration = args.duration_ms ?? 500;
+ writeFileSync(shared, JSON.stringify({ start: Date.now(), duration, from: args.from_x, to: args.to_x }));
+ await new Promise(resolve => setTimeout(resolve, duration));
+ state = { effect: "confirmed", delivery: { mode: "foreground" } };
+ } else throw Error(`Unexpected native call ${name}`);
+ return { structuredContent: state, content: [{ type: "text", text: JSON.stringify(state) },
+ ...(image ? [{ type: "image", mimeType: "image/png", data: image.toString("base64") }] : [])] };
+});
+await server.connect(new StdioServerTransport());
diff --git a/test/temporal-capture.test.ts b/test/temporal-capture.test.ts
index b373c33..599916b 100644
--- a/test/temporal-capture.test.ts
+++ b/test/temporal-capture.test.ts
@@ -15,10 +15,44 @@ test("slow captures skip elapsed slots instead of bursting or inventing timestam
expect(sequence.frames.map(f => f.timestamp_ms)).toEqual([240, 540, 840, 1140]);
expect(sequence.skipped_slots).toEqual([1, 2, 4, 5, 7, 8]);
expect(sequence.sample_span_ms).toBe(900); expect(sequence.actual_fps).toBeCloseTo(1000 / 300);
- expect(sequence.elapsed_ms).toBe(1140); expect(sequence.payload_bytes).toBe(4);
+ // Four identical frames: the repeats share the first one's image, so its byte counts once.
+ expect(sequence.elapsed_ms).toBe(1140); expect(sequence.payload_bytes).toBe(1);
expect(sequence.frames.every(f => f.capture_latency_ms === 240)).toBe(true);
});
+test("a repeated frame shares the previous image and adds no bytes; a new one does", async () => {
+ const datas = ["YQ==", "YQ==", "Yg==", "Yg==", "YQ=="];
+ let i = 0;
+ const sequence = await collectSequence({ duration_ms: 1000, fps: 5 }, async () => frame(datas[i++]), undefined, clock());
+ expect(sequence.frames).toHaveLength(5);
+ expect(sequence.frames[1].image).toBe(sequence.frames[0].image);
+ expect(sequence.frames[3].image).toBe(sequence.frames[2].image);
+ expect(sequence.frames[4].image).not.toBe(sequence.frames[0].image);
+ expect(sequence.payload_bytes).toBe(3);
+});
+
+test("a stop request ends a capture as stopped and keeps its frames; the running sequence is visible from the start", async () => {
+ const c = clock(); const stop = new AbortController(); let seen: unknown; let calls = 0;
+ const sequence = await collectSequence({ duration_ms: 10_000, fps: 10, max_frames: 1000, timeout_ms: 20_000 }, async () => {
+ c.advance(10); if (++calls === 3) stop.abort(); return frame(String(calls));
+ }, undefined, c, { limits: { duration_ms: 300_000, fps: 30, max_frames: 3000, timeout_ms: 320_000 }, stop: stop.signal, started: s => { seen = s; } });
+ expect(seen).toBe(sequence);
+ expect(sequence.stop_reason).toBe("stopped");
+ expect(sequence.frames).toHaveLength(2);
+});
+
+test("a running sequence is never evicted from the store; once settled it is", () => {
+ const store = new SequenceStore();
+ const make = (id: string) => ({ sequence_id: id, payload_bytes: 1, frames: [] }) as never as Parameters<SequenceStore["put"]>[0];
+ const running = make("running");
+ store.put(running, true);
+ for (let i = 0; i < 5; i++) store.put(make(`other-${i}`));
+ expect(store.get("running")).toBe(running);
+ store.put(running);
+ for (let i = 5; i < 8; i++) store.put(make(`other-${i}`));
+ expect(store.get("running")).toBeUndefined();
+});
+
test("cancellation stops sampling and retains only the known completed prefix", async () => {
const c = clock(); const controller = new AbortController(); let calls = 0;
const sequence = await collectSequence({ duration_ms: 1000, fps: 10 }, async () => {
diff --git a/test/watch.integration.test.ts b/test/watch.integration.test.ts
new file mode 100644
index 0000000..8f45ded
--- /dev/null
+++ b/test/watch.integration.test.ts
@@ -0,0 +1,70 @@
+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
+import { expect, test } from "vitest";
+import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";
+
+// LPT-930: a watch through real Pi: frames of a viewport arrive while a drag runs, the drag's receipt names its span
+// in the watch, the newest frame reads while the watch runs, and watch_stop returns the sheet, statistics and files.
+test("real Pi watches a viewport while a drag moves its content and stops with frames of the motion", async () => {
+ await mkdir(".tmp/watch", { recursive: true });
+ const cwd = await mkdtemp(path.resolve(".tmp/watch/pi-contract-"));
+ const log = path.join(cwd, "calls.jsonl");
+ await mkdir(path.join(cwd, ".pi"));
+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({
+ command: process.execPath, args: [path.resolve("test/fixtures/watch-driver.mjs"), log],
+ }));
+ const code = `${SCRIPT_HELPERS}
+const win = address(await windows(), "window");
+await tools.computer_read({path:win,views:["image"]});
+const lane = (await tools.computer_view({viewport:win + "/lane",x:0,y:100,width:1000,height:200})).viewport;
+const started = await tools.computer_record({at:lane,action:"watch",fps:20});
+if (started.outcome !== "observed" || !started.watch) throw Error("Watch did not start: " + JSON.stringify(started).slice(0, 600));
+const dragged = await tools.computer_drag({window:win,from:{x:100,y:200},to:{x:700,y:200},duration_ms:800});
+if (dragged.outcome !== "delivered") throw Error("Drag failed: " + JSON.stringify(dragged).slice(0, 600));
+const latest = await tools.computer_read({path:started.watch,views:["frames:last"]});
+const stopped = await tools.computer_record({at:started.watch,action:"watch_stop",to:"watch-out",video:true});
+if (stopped.outcome !== "observed") throw Error("Stop failed: " + JSON.stringify(stopped).slice(0, 600));
+const again = await tools.computer_record({at:started.watch,action:"watch_stop"});
+text(JSON.stringify({ watch: started.watch, spans: dragged.watch_spans, latest: typeof latest === "string" ? latest : latest.outcome ?? "image", again: again.outcome }));
+`;
+ try {
+ const result = await new PiIntegrationTest({
+ testName: "watch", artifactsDir: testArtifactsDir(import.meta.filename), cwd,
+ extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,
+ tools: ["codemode", "computer_delete"],
+ conversation: [
+ assistantMessage([toolCall({ id: "watch", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),
+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),
+ assistantMessage([text("Done")]),
+ ],
+ }).run("Watch the lane while dragging.");
+ expect(getToolExecution(result, "watch").isError, getToolResultText(result, "watch")).toBe(false);
+ const report = JSON.parse(getToolResultText(result, "watch").match(/\{"watch".*\}/)![0]);
+ // The drag ran 800 ms at 20 FPS: its receipt names the watch and frames captured while it ran.
+ expect(report.spans).toHaveLength(1);
+ expect(report.spans[0].sequence).toBe(report.watch);
+ expect(report.spans[0].to_ms - report.spans[0].from_ms).toBeGreaterThanOrEqual(800);
+ expect(report.spans[0].frames.to - report.spans[0].frames.from).toBeGreaterThanOrEqual(5);
+ // A stopped watch is no longer one.
+ expect(report.again).toBe("refused");
+
+ // The files: watch.json lists every frame; frames during the drag changed as the square moved; the animation exists.
+ const out = path.join(cwd, "watch-out");
+ const json = JSON.parse(await readFile(path.join(out, "watch.json"), "utf8"));
+ expect(json.actions.map((action: { label: string }) => action.label)).toEqual(["drag"]);
+ const during = json.frames.filter((frame: { actions: string[] }) => frame.actions.includes("drag"));
+ expect(during.filter((frame: { changed_share: number }) => frame.changed_share > 0).length).toBeGreaterThanOrEqual(4);
+ // The viewport crops the lane: 1000x200 window pixels, frames at most 800 px wide.
+ expect(json.frames[0].width).toBe(800);
+ expect(json.frames[0].height).toBe(160);
+ for (const frame of json.frames) await access(path.join(out, frame.file));
+ await access(path.join(out, "watch.webp"));
+
+ // Two driver processes: the session's own did the drag, the second one captured the frames.
+ const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
+ const dragPid = calls.find(call => call.name === "drag").pid;
+ const capturePids = new Set(calls.filter(call => call.name === "get_window_state" && call.at > calls.find(c => c.name === "drag").at).map(call => call.pid));
+ expect([...capturePids].some(pid => pid !== dragPid)).toBe(true);
+ } finally { await rm(cwd, { recursive: true, force: true }); }
+});
diff --git a/test/watch.test.ts b/test/watch.test.ts
new file mode 100644
index 0000000..d7eacbb
--- /dev/null
+++ b/test/watch.test.ts
@@ -0,0 +1,26 @@
+import { expect, test } from "vitest";
+import { framesBetween, sheetFrames } from "../src/watch.ts";
+import type { SequenceFrame } from "../src/temporal-capture.ts";
+
+// Frames every 100 ms; `hashes` decides which differ from the one before.
+const frames = (hashes: string[]) => hashes.map((hash, index) => ({ index, timestamp_ms: index * 100, hash, geometry_changed: false }) as SequenceFrame);
+
+test("a watch's sheet shows every frame that changed and the last one", () => {
+ const picked = sheetFrames(frames(["a", "a", "b", "b", "c", "c"]), []);
+ expect(picked.map(frame => frame.index)).toEqual([0, 2, 4, 5]);
+});
+
+test("a sheet over its limit keeps the first, the last and the frames around each action, and spreads the rest", () => {
+ // 100 frames that all differ; a drag from 3000 to 3500 ms.
+ const all = frames(Array.from({ length: 100 }, (_, i) => String(i)));
+ const picked = sheetFrames(all, [{ label: "drag", start_ms: 3000, end_ms: 3500 }], 10).map(frame => frame.index);
+ expect(picked).toHaveLength(10);
+ expect(picked).toEqual([...picked].sort((a, b) => a - b));
+ for (const index of [0, 99, 30, 35, 36]) expect(picked).toContain(index);
+});
+
+test("an action's span names the frames captured during it, or none", () => {
+ const all = frames(["a", "b", "c", "d", "e"]);
+ expect(framesBetween(all, 150, 320)).toEqual({ from: 2, to: 3 });
+ expect(framesBetween(all, 410, 450)).toBeNull();
+});