Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Watch a window or viewport in the background while acting (LPT-930)
alexshpunt-clanker[bot]f74fd94e75bd
diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex c320a31..8c6c9d8 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";import { parseComputerAddress, computerSource, displaySource, viewportSource, ADDRESS_KINDS, NAME, type ComputerAddress } from "./computer-address.ts";import { clipViewport, pngSize, viewportImage, viewportImageSize, viewportPoint, viewportRect, windowPoint, VIEWPORT_CAP, type Viewport } from "./viewport.ts";import { FULL_RESOLUTION } from "./cua-image-settings.ts";+import { WATCH_DEFAULTS } from "./watch.ts";import type { PixelBox } from "./pixel-diff.ts";import { ShortAddresses, SHORT_REFERENCE } from "./short-addresses.ts";import { parseComputerViews, type ComputerViews } from "./computer-views.ts";@@ -979,7 +980,9 @@ export class ComputerResources {if (management && state.details.window_state_kind !== "management")return this.stamp(place, { ...state, content: [{ type: "text", text: "This native runtime can only restore and activate windows, not move, resize, minimize, maximize or focus them." }],details: { ...state.details, status: "refused" } });- return this.stamp(place, management ? await session.manageWindow(address.ref, id, management, signal) : await session.restoreWindow(address.ref, id, signal));+ const started = performance.now();+ const result = management ? await session.manageWindow(address.ref, id, management, signal) : await session.restoreWindow(address.ref, id, signal);+ return this.stamp(place, ComputerResources.watched(result, session.markWatches(address.ref, String(command.action), started, performance.now())));}/*** Run one page operation (navigation, tabs, dialogs, files, page code) in a browser window by its address. The@@ -1010,13 +1013,39 @@ export class ComputerResources {if (viewport && plan.space !== undefined) throw new ComputerRequestError("space does not apply in a viewport: its points are pixels of the viewport's image.");const place = this.place(address);const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);+ const started = performance.now();let result = await this.inputStep(address.ref, session, plan, cwd, signal, viewport);// A follow-up such as Enter after typing runs only after the first part was delivered.if (plan.then && result.details.status === "delivered") {const next = await this.inputStep(address.ref, session, plan.then, cwd, signal, viewport);result = { ...next, content: [...result.content, ...next.content] };}- return this.stamp(place, result);+ return this.stamp(place, ComputerResources.watched(result, session.markWatches(address.ref, plan.label, started, performance.now())));+ }+ /** An action's receipt with its spans in the running watches of its window. */+ private static watched(result: ComputerResult, spans: NonNullable<ComputerResult["details"]["watch_spans"]>): ComputerResult {+ return spans.length ? { ...result, details: { ...result.details, watch_spans: spans } } : result;+ }+ /**+ * Start a watch for computer_record: frames of a window, or of a viewport at its own pixels, captured in the+ * background while the agent acts. The receipt names the sequence address that grows until watch_stop.+ */+ async watch(address: ComputerAddress, cwd: string, options: { fps?: number; timeout_ms?: number; maxLongEdge?: number }, signal?: AbortSignal): Promise<ComputerResult> {+ if (address.kind !== "window" && address.kind !== "viewport")+ throw new ComputerRequestError("watch takes a window address, or a viewport address from computer_view such as computer:game/pano to watch part of a window.");+ const viewport = address.kind === "viewport" ? this.viewport(address) : undefined;+ const place = this.place(address);+ const session = await this.sessionFor(cwd, { window_ref: address.ref }, signal);+ const watching = viewport ? viewportSource(viewport.window_ref, viewport.name) : computerSource("window", address.ref);+ return this.stamp(place, await session.startWatch(address.ref, watching, { fps: options.fps ?? WATCH_DEFAULTS.fps,+ timeout_ms: options.timeout_ms ?? WATCH_DEFAULTS.timeout_ms, maxLongEdge: options.maxLongEdge ?? WATCH_DEFAULTS.maxLongEdge, ...(viewport ? { crop: viewport.rect } : {}) }, signal));+ }+ /** Stop a watch: its contact sheet with actions marked, change statistics and files, in `to` (a workspace folder) when given. */+ async stopWatch(address: ComputerAddress, cwd: string, options: { to?: string; video?: boolean }, signal?: AbortSignal): Promise<ComputerResult> {+ if (address.kind !== "sequence") throw new ComputerRequestError("watch_stop takes the sequence address its watch returned.");+ const place = this.place(address);+ const { session } = await this.routeAddress(cwd, address, signal);+ return this.stamp(place, await session.stopWatch(address.ref, { ...(options.to !== undefined ? { dir: resolve(cwd, options.to) } : {}), animation: options.video === true }));}private async inputStep(window: string, session: ComputerSession, plan: InputPlan, cwd: string, signal?: AbortSignal, viewport?: Viewport): Promise<ComputerResult> {const ref = refKind(plan.target);diff --git a/src/computer-tools.ts b/src/computer-tools.tsindex 879601c..17d8e6b 100644--- a/src/computer-tools.ts+++ b/src/computer-tools.ts@@ -220,16 +220,17 @@ export function clipboardCommand(params: Static<typeof clipboardParameters>): Re}const recordParameters = Type.Object({- at: Type.String({ description: "video_start, video_stop and audio: computer:machine/<name>; sequence: a window or display address." }),- action: Type.Union([Type.Literal("video_start"), Type.Literal("video_stop"), Type.Literal("audio"), Type.Literal("sequence")], { description: "video_start records the machine's main display to video (YOLO) until video_stop or seconds pass; video_stop ends it and can fetch the file; audio records the machine's playback for seconds (YOLO); sequence captures frames of a window or display over duration_ms and returns a sequence address, whose frames Read shows with a frames: view." }),+ at: Type.String({ description: "video_start, video_stop and audio: computer:machine/<name>; sequence: a window or display address; watch: a window address, or a viewport address from computer_view such as computer:game/pano to watch part of a window; watch_stop: the sequence address watch returned." }),+ action: Type.Union([Type.Literal("video_start"), Type.Literal("video_stop"), Type.Literal("audio"), Type.Literal("sequence"), Type.Literal("watch"), Type.Literal("watch_stop")], { description: "video_start records the machine's main display to video (YOLO) until video_stop or seconds pass; video_stop ends it and can fetch the file; audio records the machine's playback for seconds (YOLO); sequence captures frames of a window or display over duration_ms and returns a sequence address, whose frames Read shows with a frames: view; watch starts capturing a window or viewport in the background and returns a sequence address at once, while the agent keeps acting on the window and each action's receipt names its span and frames in the watch; watch_stop ends a watch and returns a contact sheet with the actions marked, per-frame change statistics and the frames as files." }),seconds: Type.Optional(Type.Integer({ minimum: 1, maximum: 600, description: "video_start: recording limit, 1..600; audio: recording length, 1..300." })),- to: Type.Optional(Type.String({ description: "video_stop and audio: workspace path to fetch the recorded file into." })),+ to: Type.Optional(Type.String({ description: "video_stop and audio: workspace path to fetch the recorded file into; watch_stop: workspace folder for the frames, default a temporary folder removed when the computer session closes." })),transcribe: Type.Optional(Type.Boolean({ description: "audio: add a speech transcript made by faster-whisper on this machine; it can be wrong." })),+ video: Type.Optional(Type.Boolean({ description: "watch_stop: also write watch.webp, an animation of the frames in real time." })),duration_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 60000, description: "sequence: capture length." })),- fps: Type.Optional(Type.Integer({ minimum: 1, maximum: 30, description: "sequence: frames per second to aim for, default 4; the result reports the rate reached." })),+ fps: Type.Optional(Type.Integer({ minimum: 1, maximum: 30, description: "sequence and watch: frames per second to aim for, default 4 for sequence and 10 for watch; the result reports the rate reached." })),max_frames: Type.Optional(Type.Integer({ minimum: 1, maximum: 300, description: "sequence: frame limit, default 40." })),- timeout_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 90000, description: "sequence: total time limit, at least duration_ms, default 15000." })),- maxLongEdge: Type.Optional(Type.Integer({ minimum: 64, maximum: 4096, description: "sequence: longest frame edge in pixels." })),+ timeout_ms: Type.Optional(Type.Integer({ minimum: 1, maximum: 300000, description: "sequence: total time limit, at least duration_ms, at most 90000, default 15000; watch: when the watch stops by itself, default 60000." })),+ maxLongEdge: Type.Optional(Type.Integer({ minimum: 64, maximum: 4096, description: "sequence: longest frame edge in pixels; watch: longest frame edge, default 800, at most the viewport's own pixels." })),scale: Type.Optional(Type.Number({ minimum: 0.05, maximum: 1, description: "sequence: frame scale instead of maxLongEdge." })),region: Type.Optional(Type.Object({ x: Type.Integer({ minimum: 0 }), y: Type.Integer({ minimum: 0 }), width: Type.Integer({ minimum: 1 }), height: Type.Integer({ minimum: 1 }) },{ additionalProperties: false, description: "sequence: part of the window or display to capture, in its local pixels." })),@@ -237,11 +238,19 @@ const recordParameters = Type.Object({const SEQUENCE_FIELDS = ["duration_ms", "fps", "max_frames", "timeout_ms", "maxLongEdge", "scale", "region"];-/**- * What computer_record runs: a machine command for video and audio, or the Read sequence view of a window or display.- */-export function recordCommand(params: Static<typeof recordParameters>): { command: Record<string, unknown> } | { views: string[] } {+/** What computer_record runs: a machine command for video and audio, the Read sequence view of a window or display, or a watch start or stop. */+export type RecordRun = { command: Record<string, unknown> } | { views: string[] }+ | { watch: { fps?: number; timeout_ms?: number; maxLongEdge?: number } } | { stop: { to?: string; video?: boolean } };+export function recordCommand(params: Static<typeof recordParameters>): RecordRun {const { at: _at, action, ...fields } = params;+ if (action === "watch") {+ fieldsFor(action, fields, ["fps", "timeout_ms", "maxLongEdge"]);+ return { watch: fields };+ }+ if (action === "watch_stop") {+ fieldsFor(action, fields, ["to", "video"]);+ return { stop: fields };+ }if (action === "sequence") {fieldsFor(action, fields, SEQUENCE_FIELDS, ["duration_ms"]);const views = [`sequence:${JSON.stringify(fields)}`];@@ -586,22 +595,24 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou});pi.registerTool({...common, name: "computer_record", label: "Computer record",- description: "Use computer_record to record a machine's display to video or its playback audio, or to capture frames of a window or display over time for motion and animation.",+ description: "Use computer_record to record a machine's display to video or its playback audio, to capture frames of a window or display over time for motion and animation, or to watch a window or viewport in the background while acting on it, such as panning a panorama or dragging a 3D camera.",parameters: recordParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },...rendered("computer_record", args => `${args.at} ${args.action}${args.seconds ? ` ${args.seconds} s` : args.duration_ms ? ` ${args.duration_ms} ms` : ""}`),async execute(_id, params, signal, _update, ctx) {- let run: ReturnType<typeof recordCommand> = { command: {} };+ let run: RecordRun = { command: {} };return actionResult(await computerAction(resources, params.at, params.action, () => {run = recordCommand(params);+ if ("watch" in run || "stop" in run) return resources.address(params.at);if ("views" in run) {const address = resources.address(params.at);if (address.kind !== "window" && address.kind !== "display") throw new ComputerRequestError("sequence captures a window or display address.");return address;}return machineAddress(resources, params.at);- }, async address => "views" in run- ? (await resources.sequence(address, ctx.cwd, run.views, signal))!+ }, async address => "watch" in run ? resources.watch(address, ctx.cwd, run.watch, signal)+ : "stop" in run ? resources.stopWatch(address, ctx.cwd, run.stop, signal)+ : "views" in run ? (await resources.sequence(address, ctx.cwd, run.views, signal))!: resources.perform(address, ctx.cwd, JSON.stringify(run.command), signal)));},});diff --git a/src/contact-sheet.ts b/src/contact-sheet.tsindex 40e3985..bf2b0cf 100644--- a/src/contact-sheet.ts+++ b/src/contact-sheet.ts@@ -7,7 +7,13 @@ export interface FrameRegion { x: number; y: number; width: number; height: numb* How selected frames are returned: one image each (the default), one tiled contact sheet, or one per-pixel* maximum or minimum composite. `region` crops every frame first; `maxLongEdge` bounds the returned image.*/-export interface FrameRendering { as: "sheet" | "max" | "min"; region?: FrameRegion; columns?: number; maxLongEdge?: number }+export interface FrameRendering { as: "sheet" | "max" | "min"; region?: FrameRegion; columns?: number; maxLongEdge?: number; marks?: readonly FrameMark[] }+/** An action during the frames, in their timestamps: a sheet labels the frames it covers with its name on an orange strip. */+export interface FrameMark { label: string; start_ms: number; end_ms: number }+/** The labels of the actions a frame's time falls in. */+export function marksAt(marks: readonly FrameMark[] | undefined, timestamp: number): string[] {+ return (marks ?? []).filter(mark => timestamp >= mark.start_ms && timestamp <= mark.end_ms).map(mark => mark.label);+}/** Bounds of a rendered sheet or composite. */export const RENDER_LIMITS = { columns: 10, maxLongEdge: 4096, defaultLongEdge: 1600 } as const;@@ -42,9 +48,9 @@ async function decode(frame: SequenceFrame, region: FrameRegion | undefined): Pr}const escape = (value: string) => value.replace(/[<>&]/g, char => ({ "<": "<", ">": ">", "&": "&" })[char]!);-/** A dark label strip with the frame's index and time, drawn as SVG text. */-function label(text: string, width: number): Buffer {- return Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${LABEL_HEIGHT}"><rect width="${width}" height="${LABEL_HEIGHT}" fill="#000"/>` ++/** A label strip with the frame's index and time, drawn as SVG text: dark, or orange while an action ran. */+function label(text: string, width: number, action = false): Buffer {+ return Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${LABEL_HEIGHT}"><rect width="${width}" height="${LABEL_HEIGHT}" fill="${action ? "#c2410c" : "#000"}"/>` +`<text x="4" y="13" font-family="sans-serif" font-size="12" fill="#fff">${escape(text)}</text></svg>`);}@@ -67,7 +73,8 @@ async function sheet(frames: readonly SequenceFrame[], rendering: FrameRenderingconst png = await sharp(d.data, { raw: { width: d.width, height: d.height, channels: 3 } }).resize(tile.width, tile.height, { fit: "contain", background: "#202020", kernel: "lanczos3" }).png().toBuffer();const { x, y } = tiles[i];- return [{ input: label(`#${frames[i].index} · ${Math.round(frames[i].timestamp_ms)} ms`, tile.width), left: x, top: y - LABEL_HEIGHT },+ const actions = marksAt(rendering.marks, frames[i].timestamp_ms);+ return [{ input: label(`#${frames[i].index} · ${Math.round(frames[i].timestamp_ms)} ms${actions.length ? ` · ${actions.join(", ")}` : ""}`, tile.width, actions.length > 0), left: x, top: y - LABEL_HEIGHT },{ input: png, left: x, top: y }];}));const png = await sharp({ create: { width, height, channels: 3, background: "#202020" } }).composite(layers.flat()).png().toBuffer();diff --git a/src/driver.ts b/src/driver.tsindex 8fc2b9e..dc25e31 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -296,6 +296,13 @@ export class McpDriver implements Driver {});return transport;}+ /**+ * Another driver process with this configuration, for captures alongside this one's input. A guest endpoint takes one+ * connection, so it has none.+ */+ companion(): McpDriver | undefined {+ return this.config.endpoint ? undefined : new McpDriver(this.config, this.version);+ }/** One cancellable native call. This method never retries input. */async call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult> {if (["get_window_control_state", "manage_window"].includes(name) && !this.windowManagement)diff --git a/src/provider.ts b/src/provider.tsindex 7358fef..a1788b5 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -165,6 +165,11 @@ export function presentComputerResult(result: ComputerResult, full = false): strlines.push(`Sequence: ${computerSource("sequence", sequence.sequence_id)}`,`Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}; pending_slots=${sequence.pending_slots}`);}+ if (d.watch) lines.push(d.watch.running+ ? `Watch: ${d.watch.watching} runs in the background as ${computerSource("sequence", d.watch.sequence_id)}; computer_record watch_stop at that address stops it`+ : `Watch: ${d.watch.watching} stopped; ${d.watch.changed_frames ?? 0} frames changed${d.watch.files ? `; files in ${d.watch.files.dir}` : ""}`);+ for (const span of d.watch_spans ?? [])+ lines.push(`Watch: ${span.label} ran ${Math.round(span.from_ms)}–${Math.round(span.to_ms)} ms into ${computerSource("sequence", span.sequence_id)}; ${span.frames ? `frames ${span.frames.from}–${span.frames.to}` : "no frame captured during it yet"}`);if (d.process) lines.push(d.process.responding? `Process: ${d.process.name} (pid ${d.process.pid}) responds; ${d.process.memory_mb} MB memory, ${d.process.cpu_seconds} s CPU`: `Process: ${d.process.name} (pid ${d.process.pid}) is NOT responding; ${d.process.memory_mb} MB memory. Capture and input cannot recover a hung app: wait and read again, or close it.`);@@ -187,6 +192,7 @@ export function presentComputerResult(result: ComputerResult, full = false): str: d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; refs of this sample are current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly."): d.dom ? "Act on the window with the computer_* input tools, a ref such as e2 as target. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot.": d.window_state_id ? "This Read only looks; computer_window changes the window's state."+ : d.watch && d.status === "observed" ? "A watch only looks: the window's refs and points stay as they were for input.": d.viewport && d.status === "observed" ? "Act in this viewport with the computer_* input tools at its address; a viewport Read gives no refs.": d.status === "observed" ? "Refs of this Read are current for the next input to its window; detail, display and frame Reads give none.": "Native delivery is not verified application state. Never replay unknown input; Read a fresh window overview/state before another action.");@@ -281,6 +287,10 @@ export interface ComputerReceipt {viewport?: string;/** The sandbox address computer_open started. */sandbox?: string;+ /** The sequence address of a watch this call started or stopped. */+ watch?: string;+ /** The action's span in each running watch of its window, in that watch's milliseconds, with the frames captured during it. */+ watch_spans?: { sequence: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }[];/** The new current observation issued by this call, such as a DOM action's next snapshot. */observation: string | null;/** Whether the addressed observation can no longer be used for input. */@@ -323,6 +333,8 @@ export function computerReceipt(source: string, action: string | null, outcome:window: typeof d.window_ref === "string" ? computerSource("window", d.window_ref) : null,...(d.viewport ? { viewport: d.viewport.source } : {}),...(typeof d.sandbox_ref === "string" ? { sandbox: computerSource("sandbox", d.sandbox_ref) } : {}),+ ...(d.watch ? { watch: computerSource("sequence", d.watch.sequence_id) } : {}),+ ...(d.watch_spans ? { watch_spans: d.watch_spans.map(span => ({ sequence: computerSource("sequence", span.sequence_id), from_ms: span.from_ms, to_ms: span.to_ms, frames: span.frames })) } : {}),observation: next ? computerSource("observation", next) : null,input_consumed: outcome.inputConsumed, application_effect: "unverified",reason: failed ? computerResultText(outcome.result).split("\n{", 1)[0] : null,diff --git a/src/session.ts b/src/session.tsindex ef87de5..5a4de06 100644--- a/src/session.ts+++ b/src/session.ts@@ -1,7 +1,12 @@import { randomUUID, createHash } from "node:crypto";import { nativeDisplay, selectedDisplaySource, type NativeDisplay } from "./selected-displays.ts";-import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";-import { renderFrames, type FrameRendering, type RenderedFrames } from "./contact-sheet.ts";+import { collectSequence, cropSequenceFrame, frameSummary, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";+import { changeLines, differs, frameChanges, framesBetween, sheetFrames, WATCH_LIMITS, writeWatchFiles, type WatchFiles, type WatchSpan } from "./watch.ts";+import type { PixelBox } from "./pixel-diff.ts";+import { rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { renderFrames, type FrameMark, type FrameRendering, type RenderedFrames } from "./contact-sheet.ts";import type { GuideMark } from "./image-guides.ts";import { createUiResource, projectUi, resolveUi, annotateUi, validateUiCapture, type UiResource, type UiRequest, type UiCaptureOptions } from "./ui-resources.ts";import { isSemanticAction, semanticReceipt } from "./semantic-actions.ts";@@ -65,6 +70,11 @@ export interface Driver {overviewKernel?: CuaVisionSettings["kernel"];call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;+ /**+ * A second driver process on the same desktop for captures that run while this one acts: a driver answers one call+ * at a time, so a capture sent during a long drag would wait for its end. Absent where only one connection exists.+ */+ companion?(): Driver | undefined;close(): Promise<void>;}/** One complete action or bounded known-target sequence, grounded in one window overview. */@@ -113,6 +123,10 @@ export interface ComputerResult {captured?: { image: boolean; ui: boolean };sequence?: ReturnType<typeof sequenceMetadata>;selected_frames?: Omit<SequenceFrame, "image">[];+ /** A watch this result started or stopped: what it watches, whether it still runs, the actions it noted and, once stopped, its files. */+ watch?: { sequence_id: string; watching: string; running: boolean; marks: FrameMark[]; changed_frames?: number; files?: WatchFiles };+ /** An action's span in each running watch of its window, in the watch's milliseconds, with the frames captured during it. */+ watch_spans?: WatchSpan[];/** How a sheet or composite Read placed the selected frames in its one image. */rendered_frames?: Omit<RenderedFrames, "png">;/** Outcome of a bounded wait_for/verify; the conditions as requested, in order. */@@ -148,6 +162,8 @@ interface WindowEntry { ref: string; target: WindowTarget; identity: string; ownseen?: { width: number; height: number };windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }+/** A watch of a window: its growing sequence, when its clock started, the actions noted in it and how to stop it. */+interface Watching { window_ref: string; watching: string; sequence?: FrameSequence; started: number; marks: FrameMark[]; stop: AbortController; finished: boolean; done: Promise<FrameSequence> }/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */export function textMatcher(query?: string): (text: string) => boolean {if (!query) return () => true;@@ -189,6 +205,11 @@ export class ComputerSession {private recordings = 0;private audioClips = 0;private readonly sequences = new SequenceStore();+ /** Watches by sequence id until stopWatch reports them; they capture through `watcher`, a second driver where one exists. */+ private readonly watches = new Map<string, Watching>();+ private watcher?: Driver;+ /** Temporary folders of stopped watches' files, removed when the session closes. */+ private readonly watchDirs = new Set<string>();private uiSerial = 0;private uiImages: Observation[] = [];/**@@ -677,6 +698,160 @@ export class ComputerSession {}});}++ /**+ * Start watching a window, or with `crop` (a viewport's rectangle in full-resolution window pixels) a part of it.+ * Frames come from a second driver process in the background, outside the session's queue, so the agent acts+ * meanwhile and a long drag is seen while it runs. Returns once the first frame is in; the sequence then grows until+ * stopWatch, its time limit, its frame or byte limit, or the session closing.+ */+ async startWatch(window_ref: string, watching: string, options: { fps: number; timeout_ms: number; maxLongEdge: number; crop?: PixelBox }, signal?: AbortSignal): Promise<ComputerResult> {+ const entry = this.entries.get(window_ref);+ if (!entry || !this.allowed(entry)) return this.result("refused", "Unknown or unavailable window reference. Search its machine's windows again.");+ const permitted = () => this.entries.get(window_ref) === entry && this.allowed(entry);+ const driver = this.watcher ??= this.driver.companion?.() ?? this.driver;+ // Watch frames come from the window's own pixels at a size that gives the crop maxLongEdge pixels on its long edge.+ let cap = Math.min(FULL_RESOLUTION, options.maxLongEdge);+ const fullSize = (state: Record<string, unknown>) => {+ const width = state.screenshot_original_width, height = state.screenshot_original_height;+ if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;+ const fit = Math.min(1, FULL_RESOLUTION / Math.max(width, height));+ return { width: Math.round(width * fit), height: Math.round(height * fit) };+ };+ if (options.crop) {+ let probe: DriverResult;+ try { probe = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: 64 }, signal); }+ catch (error) { return this.result("refused", `The watch did not start: ${error instanceof Error ? error.message : error}`); }+ const full = probe.isError || !probe.structuredContent ? undefined : fullSize(probe.structuredContent);+ if (!full) return this.result("refused", "The watch did not start: the window could not be captured.");+ const long = Math.max(options.crop.width, options.crop.height);+ cap = Math.max(64, Math.min(FULL_RESOLUTION, Math.ceil(Math.max(full.width, full.height) * Math.min(options.maxLongEdge, long) / long)));+ }+ const capture = async (active: AbortSignal) => {+ if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");+ const nativeStarted = performance.now();+ const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap }, active);+ const native_completed_at = performance.now();+ active.throwIfAborted();+ if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");+ const state = native.structuredContent;+ if (native.isError || !state || state.status === "refused" || state.effect === "refused") throw new SequenceStop("capture_failed");+ // Identity is not asked here: its peer takes one request at a time and the session's own calls use it.+ if ((state.pid !== undefined && state.pid !== entry.target.pid) || (state.window_id !== undefined && state.window_id !== entry.target.window_id))+ throw new SequenceStop("target_lost");+ const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");+ const context = this.overview(state, "watch-frame", false, images);+ if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024) throw new SequenceStop("capture_failed");+ let source = this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target });+ source.semantic_scope = "none"; source.capabilities = { read: true, input: "none", detail: "unsupported" };+ let image = images[0];+ const full = options.crop && fullSize(state);+ if (options.crop && full) {+ // The viewport follows the window: its rectangle is relative to the window's top-left corner.+ const kx = context.width / full.width, ky = context.height / full.height, crop = options.crop;+ const region = { x: Math.round(crop.x * kx), y: Math.round(crop.y * ky), width: Math.max(1, Math.round(crop.width * kx)), height: Math.max(1, Math.round(crop.height * ky)) };+ if (region.x >= context.width || region.y >= context.height) throw new SequenceStop("capture_failed");+ ({ image, source } = await cropSequenceFrame(image, source, region, false));+ }+ return { image, source, native_completed_at, native_latency_ms: native_completed_at - nativeStarted };+ };+ const stop = new AbortController();+ const watch: Watching = { window_ref, watching, started: 0, marks: [], stop, finished: false, done: Promise.resolve(undefined as unknown as FrameSequence) };+ watch.done = collectSequence({ duration_ms: options.timeout_ms, fps: options.fps, max_frames: WATCH_LIMITS.max_frames, timeout_ms: options.timeout_ms + 20_000 }, capture,+ undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, started: (sequence, at) => {+ watch.sequence = sequence; watch.started = at;+ sequence.configuration = { ...this.overviewSettings(), maxLongEdge: cap };+ this.sequences.put(sequence, true);+ this.watches.set(sequence.sequence_id, watch);+ } }).then(sequence => {+ watch.finished = true;+ if (this.watches.get(sequence.sequence_id) === watch) this.sequences.put(sequence);+ return sequence;+ });+ // Return with the first frame, so a watch that cannot capture says so now; starting the second driver takes a few seconds.+ const deadline = performance.now() + 30_000;+ while (!watch.finished && !watch.sequence?.frames.length && performance.now() < deadline && !signal?.aborted) await delay(20);+ const sequence = watch.sequence!;+ if (!sequence?.frames.length) {+ stop.abort();+ const ended = await watch.done;+ this.watches.delete(ended.sequence_id);+ return this.result("refused", `The watch did not start: ${signal?.aborted ? "cancelled" : ended.stop_reason === "stopped" ? "no frame came within 30 s" : ended.stop_reason}.`);+ }+ const size = sequence.frames[0].source.image.dimensions;+ return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` ++ `Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` ++ `It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,+ undefined, { ...this.addressed(entry), sequence: sequenceMetadata(sequence), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });+ }++ /** Note an action on a window in each running watch of it: the action's span in the watch's time and the frames captured during it so far. */+ markWatches(window_ref: string, label: string, start: number, end: number): WatchSpan[] {+ const spans: WatchSpan[] = [];+ for (const watch of this.watches.values()) {+ if (watch.window_ref !== window_ref || watch.finished || !watch.sequence) continue;+ const mark = { label, start_ms: start - watch.started, end_ms: end - watch.started };+ watch.marks.push(mark);+ spans.push({ sequence_id: watch.sequence.sequence_id, label, from_ms: mark.start_ms, to_ms: mark.end_ms, frames: framesBetween(watch.sequence.frames, mark.start_ms, mark.end_ms) });+ }+ return spans;+ }+ /** Whether this sequence is a watch that has not been stopped with stopWatch yet. */+ isWatch(id: string): boolean { return this.watches.has(id); }++ /**+ * Stop a watch: a contact sheet of the frames that changed with the actions marked, change statistics of every+ * frame, and the frames as files in `dir` (a temporary folder removed when the session closes, unless given), with an+ * animated WebP when asked. The sequence keeps its frames for frames: Reads.+ */+ async stopWatch(id: string, files: { dir?: string; animation?: boolean } = {}): Promise<ComputerResult> {+ const watch = this.watches.get(id);+ if (!watch) return this.result("refused", this.sequences.get(id) ? "This sequence is no running watch; Read it with a frames: view." : "No watch of this session has this address. Watches do not survive reload or close.");+ watch.stop.abort();+ const sequence = await watch.done;+ this.watches.delete(id);+ const entry = this.entries.get(watch.window_ref);+ if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") {+ this.sequences.clear();+ return this.result("refused", "The watched window is no longer accessible; its frames are withheld.");+ }+ const reasons: Record<FrameSequence["stop_reason"], string> = { stopped: "stopped", completed: "stopped by itself at its time limit", frame_limit: "stopped by itself at its frame limit",+ byte_limit: "stopped by itself at its memory limit", timeout: "stopped by itself at its time limit", cancelled: "cancelled", target_lost: "stopped: the window closed",+ access_revoked: "stopped: access to the window ended", capture_failed: "stopped: a capture failed" };+ const frames = sequence.frames;+ const changes = await frameChanges(frames);+ const distinct = frames.filter((_, i) => differs(frames, i)).length;+ const lines = [`Watch of ${watch.watching} ${reasons[sequence.stop_reason]}: ${frames.length} frames over ${(sequence.elapsed_ms / 1000).toFixed(1)} s` ++ `${sequence.actual_fps ? ` at ${sequence.actual_fps.toFixed(1)} FPS` : ""}; ${distinct} differ from the frame before.`];+ if (watch.marks.length) lines.push("Actions (ms of the watch):", ...watch.marks.map(mark => {+ const span = framesBetween(frames, mark.start_ms, mark.end_ms);+ return `- ${mark.label} ${Math.round(mark.start_ms)}–${Math.round(mark.end_ms)} ms, ${span ? `frames ${span.from}–${span.to}` : "no frame during it"}`;+ }));+ const moving = changeLines(changes, watch.marks);+ lines.push(moving.length ? "Biggest changes against the frame before (share of the frame's pixels, box in its pixels):" : "No frame changed against the one before.", ...moving);+ const content: Content[] = [];+ let rendered: Omit<RenderedFrames, "png"> | undefined;+ if (frames.length) {+ const { png, ...layout } = await renderFrames(sheetFrames(frames, watch.marks), { as: "sheet", marks: watch.marks });+ rendered = layout;+ lines.push(`Contact sheet: ${layout.tiles.length} frames that changed, ${layout.columns}x${layout.rows} tiles in time order, each under "#index · time"; an orange label means an action ran then. Frames: ${layout.tiles.map(tile => `#${tile.index}`).join(", ")}.`);+ content.push({ type: "image", mimeType: "image/png", data: png.toString("base64") });+ }+ const dir = files.dir ?? join(tmpdir(), "pi-computer-use-watch", id);+ let written: WatchFiles | undefined;+ try {+ written = await writeWatchFiles(dir, sequence, watch.marks, changes, { watching: watch.watching }, files.animation);+ if (!files.dir) this.watchDirs.add(dir);+ lines.push(`Files in ${dir}: ${written.frames} PNG frames (the frames that differ from the one before), watch.json with every frame's time, file, change and actions${written.animation ? ", and watch.webp, an animation in real time" : ""}.` ++ (files.dir ? "" : " The folder is removed when the computer session closes."));+ } catch (error) { lines.push(`The frames were not written to ${dir}: ${error instanceof Error ? error.message : error}`); }+ lines.push("Read the sequence with a frames: view for other frames.");+ const result = this.result(frames.length ? "observed" : "refused", lines.join("\n"), undefined,+ { sequence: sequenceMetadata(sequence), ...(rendered ? { rendered_frames: rendered } : {}),+ watch: { sequence_id: id, watching: watch.watching, running: false, marks: watch.marks, changed_frames: distinct, ...(written ? { files: written } : {}) } });+ result.content.push(...content);+ return result;+ }/** Enumerate native Windows monitors in YOLO. References grant read access, never input. */displays(signal?: AbortSignal): Promise<ComputerResult> {return this.queue(async () => {@@ -1856,6 +2031,14 @@ export class ComputerSession {/** Close session-opened browsers (deleting their profiles), detach from attached ones and release the native runtime. Other windows/files are not closed or saved. */close(): Promise<ComputerResult> {return this.queue(async () => {+ const watches = [...this.watches.values()];+ this.watches.clear();+ for (const watch of watches) watch.stop.abort();+ await Promise.allSettled(watches.map(watch => watch.done));+ if (this.watcher && this.watcher !== this.driver) await this.watcher.close().catch(() => undefined);+ this.watcher = undefined;+ await Promise.allSettled([...this.watchDirs].map(dir => rm(dir, { recursive: true, force: true })));+ this.watchDirs.clear();this.entries.clear(); this.displayEntries.clear(); this.displayOverview = undefined; this.uiImages = []; this.sequences.clear();const browsers = [...this.browsers.values()];this.browsers.clear();diff --git a/src/temporal-capture.ts b/src/temporal-capture.tsindex 8fc8a8d..03c3501 100644--- a/src/temporal-capture.ts+++ b/src/temporal-capture.ts@@ -23,7 +23,8 @@ export interface FrameSequence {/** How old the newest frame was when the sequence returned: time from its native completion to the result. */last_frame_age_ms: number | null;skipped_slots: number[]; pending_slots: number; payload_bytes: number;- stop_reason: "completed" | "frame_limit" | "byte_limit" | "cancelled" | "timeout" | "target_lost" | "access_revoked" | "capture_failed";+ /** stopped: a watch stopped by request, with every frame it captured until then. */+ stop_reason: "completed" | "stopped" | "frame_limit" | "byte_limit" | "cancelled" | "timeout" | "target_lost" | "access_revoked" | "capture_failed";frames: SequenceFrame[];configuration?: CuaVisionSettings;interval_ms?: { min: number; max: number; mean: number };@@ -35,12 +36,13 @@ const byteBudget = 48 * 1024 * 1024;export class SequenceStop extends Error {constructor(readonly reason: FrameSequence["stop_reason"]) { super(reason); }}+/** Bounds of a capture: its length, rate, frame count and total time. */+export interface CaptureLimits { duration_ms: number; fps: number; max_frames: number; timeout_ms: number }/** Limits of one sequence: a minute of frames at up to 30 FPS; one Read still selects at most 40 of them. */export const SEQUENCE_LIMITS = { duration_ms: 60_000, fps: 30, max_frames: 300, timeout_ms: 90_000, selected: 40 } as const;/** Validate before native dispatch; requested FPS is a sampling goal, never a performance promise. */-export function validateSequence(options: SequenceOptions): FrameSequence["requested"] {+export function validateSequence(options: SequenceOptions, L: CaptureLimits = SEQUENCE_LIMITS): FrameSequence["requested"] {const result = { duration_ms: options.duration_ms, fps: options.fps ?? 4, max_frames: options.max_frames ?? 40, timeout_ms: options.timeout_ms ?? 15000 };- const L = SEQUENCE_LIMITS;if (!Number.isSafeInteger(result.duration_ms) || result.duration_ms < 1 || result.duration_ms > L.duration_ms ||!Number.isFinite(result.fps) || result.fps < 1 || result.fps > L.fps ||!Number.isSafeInteger(result.max_frames) || result.max_frames < 1 || result.max_frames > L.max_frames ||@@ -52,15 +54,28 @@ export function validateSequence(options: SequenceOptions): FrameSequence["requereturn result;}-/** Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. */+/** What a long-running capture such as a watch adds: its own limits, a stop request that keeps the frames, and its sequence as soon as it starts. */+export interface CaptureHooks {+ limits?: CaptureLimits;+ /** Ends sampling with stop_reason "stopped"; frames captured until then stay. */+ stop?: AbortSignal;+ /** Receives the sequence, which then grows frame by frame, and the clock time its timestamps count from. */+ started?(sequence: FrameSequence, at: number): void;+}+/**+ * Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. A frame with the+ * same bytes as the one before shares its image and adds nothing to the byte budget, so a still window costs little.+ */export async function collectSequence(options: SequenceOptions,capture: (signal: AbortSignal) => Promise<{ image: SequenceFrame["image"]; source: ObservationSource; native_completed_at?: number; native_latency_ms?: number }>, signal?: AbortSignal,- clock: { now(): number; wait(ms: number, active: AbortSignal): Promise<void> } = { now: () => performance.now(), wait: (ms, active) => delay(ms, undefined, { signal: active }) }): Promise<FrameSequence> {- const requested = validateSequence(options); const started = clock.now();+ clock: { now(): number; wait(ms: number, active: AbortSignal): Promise<void> } = { now: () => performance.now(), wait: (ms, active) => delay(ms, undefined, { signal: active }) },+ hooks: CaptureHooks = {}): Promise<FrameSequence> {+ const requested = validateSequence(options, hooks.limits); const started = clock.now();const timeout = AbortSignal.timeout(requested.timeout_ms);- const active = signal ? AbortSignal.any([signal, timeout]) : timeout;+ const active = AbortSignal.any([timeout, ...(signal ? [signal] : []), ...(hooks.stop ? [hooks.stop] : [])]);const sequence: FrameSequence = { sequence_id: "seq-" + randomUUID(), requested, started_at: new Date().toISOString(),elapsed_ms: 0, sample_span_ms: 0, actual_fps: null, last_frame_age_ms: null, skipped_slots: [], pending_slots: 0, payload_bytes: 0, stop_reason: "completed", frames: [] };+ hooks.started?.(sequence, started);const interval = 1000 / requested.fps; const slots = Math.ceil(requested.duration_ms / interval);let slot = 0;try {@@ -79,17 +94,21 @@ export async function collectSequence(options: SequenceOptions,const before = clock.now(); const result = await capture(active); const after = clock.now();active.throwIfAborted();const bytes = Buffer.from(result.image.data, "base64");- if (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget) { sequence.stop_reason = "byte_limit"; break; }+ const hash = createHash("sha256").update(bytes).digest("hex");const previous = sequence.frames.at(-1);const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });+ const geometryChanged = !!previous && geometry(previous.source) !== geometry(result.source);+ const repeated = !!previous && previous.hash === hash && !geometryChanged;+ if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; break; }sequence.frames.push({ index: sequence.frames.length, slot, timestamp_ms: (result.native_completed_at ?? after) - started, capture_started_ms: before - started,- capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash: createHash("sha256").update(bytes).digest("hex"), bytes: bytes.length,- source: result.source, geometry_changed: !!previous && geometry(previous.source) !== geometry(result.source), image: result.image });- sequence.payload_bytes += bytes.length; slot++;+ capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,+ source: result.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result.image });+ if (!repeated) sequence.payload_bytes += bytes.length;+ slot++;while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);}} catch (error) {- sequence.stop_reason = signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";+ sequence.stop_reason = hooks.stop?.aborted ? "stopped" : signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";}sequence.elapsed_ms = clock.now() - started;sequence.last_frame_age_ms = sequence.frames.length ? sequence.elapsed_ms - sequence.frames.at(-1)!.timestamp_ms : null;@@ -147,14 +166,21 @@ export async function cropSequenceFrame(image: SequenceFrame["image"], source: Ocoordinate_space: "detail_image_pixels" as const, dimensions: { width: applied.width, height: applied.height }, origin,crop: { requested, applied, coordinate_space: "uncropped_image_pixels" as const } } } };}-/** Private bounded frame resources. Reload/close drops pixels; readers must recheck source access. */+/**+ * Private bounded frame resources. Reload/close drops pixels; readers must recheck source access. A running watch is+ * never evicted; once it stops, putting it again makes it an ordinary sequence.+ */export class SequenceStore {private readonly sequences = new Map<string, FrameSequence>();- put(sequence: FrameSequence): void {+ private readonly running = new Set<string>();+ put(sequence: FrameSequence, running = false): void {+ this.sequences.delete(sequence.sequence_id);this.sequences.set(sequence.sequence_id, sequence);- while (this.sequences.size > 3 || [...this.sequences.values()].reduce((sum, s) => sum + s.payload_bytes, 0) > byteBudget)- this.sequences.delete(this.sequences.keys().next().value!);+ if (running) this.running.add(sequence.sequence_id); else this.running.delete(sequence.sequence_id);+ const evictable = () => [...this.sequences.keys()].filter(id => !this.running.has(id));+ while (evictable().length && (evictable().length > 3 || [...this.sequences.values()].reduce((sum, s) => sum + s.payload_bytes, 0) > byteBudget))+ this.sequences.delete(evictable()[0]);}get(id: string): FrameSequence | undefined { return this.sequences.get(id); }- clear(): void { this.sequences.clear(); }+ clear(): void { this.sequences.clear(); this.running.clear(); }}diff --git a/src/watch.ts b/src/watch.tsnew file mode 100644index 0000000..113a0b2--- /dev/null+++ b/src/watch.ts@@ -0,0 +1,133 @@+import { mkdir, writeFile } from "node:fs/promises";+import { join } from "node:path";+import sharp from "sharp";+import { marksAt, type FrameMark } from "./contact-sheet.ts";+import { decodeFrame, diffFrames, type PixelBox, type PixelFrame } from "./pixel-diff.ts";+import type { CaptureLimits, FrameSequence, SequenceFrame } from "./temporal-capture.ts";++/**+ * A watch captures a window or a viewport in the background while the agent acts, until it is stopped, its time+ * limit passes or the session closes. Frames that repeat the one before share its image, so a still window costs+ * little; the frame limit bounds a busy one.+ */+export const WATCH_LIMITS: CaptureLimits = { duration_ms: 300_000, fps: 30, max_frames: 3000, timeout_ms: 320_000 };+/** Defaults of a watch: 10 FPS for a minute, frames at most 800 px on their long edge. */+export const WATCH_DEFAULTS = { fps: 10, timeout_ms: 60_000, maxLongEdge: 800 } as const;+/** Frames on the contact sheet a stopped watch returns. */+export const WATCH_SHEET_FRAMES = 40;+/** Frames of the animation, and its long edge. */+export const ANIMATION_LIMITS = { frames: 200, longEdge: 640 } as const;++/** What a receipt says about one watch of the window an action ran on: the action's span in the watch's time and the frames captured in it. */+export interface WatchSpan { sequence_id: string; label: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }++/** The first and last frame captured within a span of the watch's time; null when none was. */+export function framesBetween(frames: readonly Pick<SequenceFrame, "index" | "timestamp_ms">[], from: number, to: number): { from: number; to: number } | null {+ const inside = frames.filter(frame => frame.timestamp_ms >= from && frame.timestamp_ms <= to);+ return inside.length ? { from: inside[0].index, to: inside.at(-1)!.index } : null;+}++/** Whether a frame differs from the one before it: the first frame, other bytes or another size. */+export const differs = (frames: readonly SequenceFrame[], i: number) => i === 0 || frames[i].hash !== frames[i - 1].hash || frames[i].geometry_changed;++/**+ * The frames of a watch's contact sheet, in time order: every frame that differs from the one before, plus the last.+ * When there are more than `limit`, the first, the last and those around each action (the first changed frame at or+ * after it starts, the last during it and the first after it) stay, and the rest are spread evenly over the watch.+ */+export function sheetFrames(frames: readonly SequenceFrame[], marks: readonly FrameMark[], limit = WATCH_SHEET_FRAMES): SequenceFrame[] {+ if (!frames.length) return [];+ const changed = frames.filter((_, i) => differs(frames, i));+ if (changed.at(-1) !== frames.at(-1)) changed.push(frames.at(-1)!);+ if (changed.length <= limit) return changed;+ const keep = new Set<SequenceFrame>([changed[0], changed.at(-1)!]);+ for (const mark of marks) {+ const during = changed.filter(frame => frame.timestamp_ms >= mark.start_ms && frame.timestamp_ms <= mark.end_ms);+ const after = changed.find(frame => frame.timestamp_ms > mark.end_ms);+ for (const frame of [during[0], during.at(-1), after]) if (frame && keep.size < limit) keep.add(frame);+ }+ const rest = changed.filter(frame => !keep.has(frame));+ const room = limit - keep.size;+ for (let i = 0; i < room && rest.length; i++) keep.add(rest[Math.floor((i + 0.5) * rest.length / room)]);+ return changed.filter(frame => keep.has(frame));+}++/** How one frame differs from the one before it, in its own pixels: share of changed pixels and the box around them. */+export interface FrameChange { index: number; timestamp_ms: number; share: number; box?: PixelBox; size_changed?: true }++/** Change statistics of every frame against the one before it; repeated frames changed nothing. */+export async function frameChanges(frames: readonly SequenceFrame[]): Promise<FrameChange[]> {+ const changes: FrameChange[] = [];+ let previous: PixelFrame | undefined;+ for (let i = 0; i < frames.length; i++) {+ const frame = frames[i];+ if (i > 0 && !differs(frames, i)) { changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 }); continue; }+ const pixels = await decodeFrame(Buffer.from(frame.image.data, "base64"));+ if (!previous) changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 });+ else {+ const diff = diffFrames(previous, pixels);+ changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: diff.size_changed ? 1 : diff.share,+ ...(diff.box ? { box: diff.box } : {}), ...(diff.size_changed ? { size_changed: true as const } : {}) });+ }+ previous = pixels;+ }+ return changes;+}++/** A short text of the frames that changed most, at most `limit` lines in time order. */+export function changeLines(changes: readonly FrameChange[], marks: readonly FrameMark[], limit = 20): string[] {+ const moving = changes.filter(change => change.share > 0);+ const shown = new Set([...moving].sort((a, b) => b.share - a.share).slice(0, limit));+ return moving.filter(change => shown.has(change)).map(change => {+ const actions = marksAt(marks, change.timestamp_ms);+ const box = change.box ? ` in ${change.box.x},${change.box.y} ${change.box.width}x${change.box.height}` : "";+ return `- #${change.index} at ${Math.round(change.timestamp_ms)} ms: ${change.size_changed ? "size changed" : `${(change.share * 100).toFixed(1)}%${box}`}${actions.length ? ` (during ${actions.join(", ")})` : ""}`;+ });+}++/** The files a stopped watch leaves: each frame that differs from the one before as a PNG, watch.json, and with `animation` watch.webp. */+export interface WatchFiles { dir: string; frames: number; json: string; animation?: string }++/**+ * Write a stopped watch's frames for further processing. watch.json lists every frame with its time, its file (shared+ * by the repeats after it), its change against the frame before and the actions it falls in, and every action.+ */+export async function writeWatchFiles(dir: string, sequence: FrameSequence, marks: readonly FrameMark[], changes: readonly FrameChange[],+ about: Record<string, unknown>, animation = false): Promise<WatchFiles> {+ await mkdir(dir, { recursive: true });+ const frames = sequence.frames;+ const files: string[] = [];+ let written = 0;+ for (let i = 0; i < frames.length; i++) {+ if (differs(frames, i)) {+ files.push(`frame-${String(frames[i].index).padStart(5, "0")}.png`);+ await writeFile(join(dir, files[i]), Buffer.from(frames[i].image.data, "base64"));+ written++;+ } else files.push(files[i - 1]);+ }+ const json = join(dir, "watch.json");+ await writeFile(json, JSON.stringify({ ...about, sequence_id: sequence.sequence_id, started_at: sequence.started_at, stop_reason: sequence.stop_reason,+ elapsed_ms: sequence.elapsed_ms, requested: sequence.requested, actual_fps: sequence.actual_fps, skipped_slots: sequence.skipped_slots.length, actions: marks,+ frames: frames.map((frame, i) => ({ index: frame.index, timestamp_ms: frame.timestamp_ms, file: files[i], width: frame.source.image.dimensions.width,+ height: frame.source.image.dimensions.height, changed_share: changes[i]?.share ?? null, changed_box: changes[i]?.box ?? null,+ actions: marksAt(marks, frame.timestamp_ms) })) }, null, 2));+ return { dir, frames: written, json, ...(animation && frames.length ? { animation: await writeAnimation(join(dir, "watch.webp"), frames) } : {}) };+}++/**+ * An animated WebP of the frames that differ from the one before, each shown until the next one, played once in real+ * time and then looped. More than ANIMATION_LIMITS.frames are spread evenly; all are fitted into one size.+ */+async function writeAnimation(path: string, frames: readonly SequenceFrame[]): Promise<string> {+ let picked = frames.filter((_, i) => differs(frames, i));+ if (picked.length > ANIMATION_LIMITS.frames)+ picked = Array.from({ length: ANIMATION_LIMITS.frames }, (_, i) => picked[Math.floor(i * picked.length / ANIMATION_LIMITS.frames)]);+ const first = picked[0].source.image.dimensions;+ const scale = Math.min(1, ANIMATION_LIMITS.longEdge / Math.max(first.width!, first.height!));+ const size = { width: Math.max(1, Math.round(first.width! * scale)), height: Math.max(1, Math.round(first.height! * scale)) };+ const images = await Promise.all(picked.map(frame => sharp(Buffer.from(frame.image.data, "base64")).removeAlpha()+ .resize(size.width, size.height, { fit: "contain", background: "#202020" }).png().toBuffer()));+ const delay = picked.map((frame, i) => Math.max(20, Math.round((picked[i + 1]?.timestamp_ms ?? frame.timestamp_ms + 1000) - frame.timestamp_ms)));+ await sharp(images, { join: { animated: true } }).webp({ delay, loop: 0, quality: 80 }).toFile(path);+ return path;+}diff --git a/test/fixtures/watch-driver.mjs b/test/fixtures/watch-driver.mjsnew file mode 100644index 0000000..9e831e1--- /dev/null+++ b/test/fixtures/watch-driver.mjs@@ -0,0 +1,53 @@+import { appendFileSync, readFileSync, writeFileSync } from "node:fs";+import sharp from "sharp";+import { Server } from "@modelcontextprotocol/sdk/server/index.js";+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";+import { agentCursor } from "./agent-cursor.mjs";+import { CallToolRequestSchema, ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";++// A window of 1000x640 with a red square on a lane: a drag moves the square along with the pointer over its duration,+// like a panorama panned by dragging. Every driver process started with the same log shares the drag through a state+// file, so a second process captures the square while the first one is still dragging.+const log = process.argv[2], shared = `${log}.drag.json`;+const window = { width: 1000, height: 640 };+const bounds = { x: 80, y: 90, ...window };+const server = new Server({ name: "cua-driver", version: "0.33.2" }, { capabilities: { tools: {} } });+const drag = () => { try { return JSON.parse(readFileSync(shared, "utf8")); } catch { return undefined; } };+/** Where the square is now, in window pixels: at the drag's start before it, moving with it, at its end after it. */+function square() {+ const d = drag();+ if (!d) return 100;+ const progress = Math.min(1, Math.max(0, (Date.now() - d.start) / Math.max(1, d.duration)));+ return Math.round(d.from + (d.to - d.from) * progress);+}+async function capture(cap) {+ const scale = Math.min(1, (cap ?? 4096) / Math.max(window.width, window.height));+ const x = square();+ const full = await sharp({ create: { ...window, channels: 3, background: "white" } })+ .composite([{ input: { create: { width: 60, height: 60, channels: 3, background: "red" } }, left: Math.min(window.width - 60, x), top: 170 }]).png().toBuffer();+ const width = Math.round(window.width * scale), height = Math.round(window.height * scale);+ return { png: scale === 1 ? full : await sharp(full).resize(width, height).png().toBuffer(), width, height };+}+server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: [] }));+server.setRequestHandler(CallToolRequestSchema, async ({ params: { name, arguments: args = {} } }) => {+ const cursor = agentCursor(name, args); if (cursor) return cursor;+ appendFileSync(log, JSON.stringify({ name, arguments: args, pid: process.pid, at: Date.now() }) + "\n");+ let state; let image;+ if (name === "fixture_identity") state = { identity: `owned:${args.window_id}` };+ else if (name === "list_windows") state = { windows: [{ pid: 42, window_id: 91, bounds, title: "Watch fixture" }] };+ else if (name === "get_window_state") {+ const shot = await capture(args.max_image_dimension);+ state = { pid: 42, window_id: 91, window_bounds: bounds, screenshot_width: shot.width, screenshot_height: shot.height,+ screenshot_original_width: window.width, screenshot_original_height: window.height, capture_id: `capture-${Date.now()}`,+ elements_complete: true, elements: [] };+ if (args.include_screenshot !== false) image = shot.png;+ } else if (name === "drag") {+ const duration = args.duration_ms ?? 500;+ writeFileSync(shared, JSON.stringify({ start: Date.now(), duration, from: args.from_x, to: args.to_x }));+ await new Promise(resolve => setTimeout(resolve, duration));+ state = { effect: "confirmed", delivery: { mode: "foreground" } };+ } else throw Error(`Unexpected native call ${name}`);+ return { structuredContent: state, content: [{ type: "text", text: JSON.stringify(state) },+ ...(image ? [{ type: "image", mimeType: "image/png", data: image.toString("base64") }] : [])] };+});+await server.connect(new StdioServerTransport());diff --git a/test/temporal-capture.test.ts b/test/temporal-capture.test.tsindex b373c33..599916b 100644--- a/test/temporal-capture.test.ts+++ b/test/temporal-capture.test.ts@@ -15,10 +15,44 @@ test("slow captures skip elapsed slots instead of bursting or inventing timestamexpect(sequence.frames.map(f => f.timestamp_ms)).toEqual([240, 540, 840, 1140]);expect(sequence.skipped_slots).toEqual([1, 2, 4, 5, 7, 8]);expect(sequence.sample_span_ms).toBe(900); expect(sequence.actual_fps).toBeCloseTo(1000 / 300);- expect(sequence.elapsed_ms).toBe(1140); expect(sequence.payload_bytes).toBe(4);+ // Four identical frames: the repeats share the first one's image, so its byte counts once.+ expect(sequence.elapsed_ms).toBe(1140); expect(sequence.payload_bytes).toBe(1);expect(sequence.frames.every(f => f.capture_latency_ms === 240)).toBe(true);});+test("a repeated frame shares the previous image and adds no bytes; a new one does", async () => {+ const datas = ["YQ==", "YQ==", "Yg==", "Yg==", "YQ=="];+ let i = 0;+ const sequence = await collectSequence({ duration_ms: 1000, fps: 5 }, async () => frame(datas[i++]), undefined, clock());+ expect(sequence.frames).toHaveLength(5);+ expect(sequence.frames[1].image).toBe(sequence.frames[0].image);+ expect(sequence.frames[3].image).toBe(sequence.frames[2].image);+ expect(sequence.frames[4].image).not.toBe(sequence.frames[0].image);+ expect(sequence.payload_bytes).toBe(3);+});++test("a stop request ends a capture as stopped and keeps its frames; the running sequence is visible from the start", async () => {+ const c = clock(); const stop = new AbortController(); let seen: unknown; let calls = 0;+ const sequence = await collectSequence({ duration_ms: 10_000, fps: 10, max_frames: 1000, timeout_ms: 20_000 }, async () => {+ c.advance(10); if (++calls === 3) stop.abort(); return frame(String(calls));+ }, undefined, c, { limits: { duration_ms: 300_000, fps: 30, max_frames: 3000, timeout_ms: 320_000 }, stop: stop.signal, started: s => { seen = s; } });+ expect(seen).toBe(sequence);+ expect(sequence.stop_reason).toBe("stopped");+ expect(sequence.frames).toHaveLength(2);+});++test("a running sequence is never evicted from the store; once settled it is", () => {+ const store = new SequenceStore();+ const make = (id: string) => ({ sequence_id: id, payload_bytes: 1, frames: [] }) as never as Parameters<SequenceStore["put"]>[0];+ const running = make("running");+ store.put(running, true);+ for (let i = 0; i < 5; i++) store.put(make(`other-${i}`));+ expect(store.get("running")).toBe(running);+ store.put(running);+ for (let i = 5; i < 8; i++) store.put(make(`other-${i}`));+ expect(store.get("running")).toBeUndefined();+});+test("cancellation stops sampling and retains only the known completed prefix", async () => {const c = clock(); const controller = new AbortController(); let calls = 0;const sequence = await collectSequence({ duration_ms: 1000, fps: 10 }, async () => {diff --git a/test/watch.integration.test.ts b/test/watch.integration.test.tsnew file mode 100644index 0000000..8f45ded--- /dev/null+++ b/test/watch.integration.test.ts@@ -0,0 +1,70 @@+import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";+import path from "node:path";+import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";+import { expect, test } from "vitest";+import { SCRIPT_HELPERS } from "./fixtures/standalone-computer.ts";++// LPT-930: a watch through real Pi: frames of a viewport arrive while a drag runs, the drag's receipt names its span+// in the watch, the newest frame reads while the watch runs, and watch_stop returns the sheet, statistics and files.+test("real Pi watches a viewport while a drag moves its content and stops with frames of the motion", async () => {+ await mkdir(".tmp/watch", { recursive: true });+ const cwd = await mkdtemp(path.resolve(".tmp/watch/pi-contract-"));+ const log = path.join(cwd, "calls.jsonl");+ await mkdir(path.join(cwd, ".pi"));+ await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({+ command: process.execPath, args: [path.resolve("test/fixtures/watch-driver.mjs"), log],+ }));+ const code = `${SCRIPT_HELPERS}+const win = address(await windows(), "window");+await tools.computer_read({path:win,views:["image"]});+const lane = (await tools.computer_view({viewport:win + "/lane",x:0,y:100,width:1000,height:200})).viewport;+const started = await tools.computer_record({at:lane,action:"watch",fps:20});+if (started.outcome !== "observed" || !started.watch) throw Error("Watch did not start: " + JSON.stringify(started).slice(0, 600));+const dragged = await tools.computer_drag({window:win,from:{x:100,y:200},to:{x:700,y:200},duration_ms:800});+if (dragged.outcome !== "delivered") throw Error("Drag failed: " + JSON.stringify(dragged).slice(0, 600));+const latest = await tools.computer_read({path:started.watch,views:["frames:last"]});+const stopped = await tools.computer_record({at:started.watch,action:"watch_stop",to:"watch-out",video:true});+if (stopped.outcome !== "observed") throw Error("Stop failed: " + JSON.stringify(stopped).slice(0, 600));+const again = await tools.computer_record({at:started.watch,action:"watch_stop"});+text(JSON.stringify({ watch: started.watch, spans: dragged.watch_spans, latest: typeof latest === "string" ? latest : latest.outcome ?? "image", again: again.outcome }));+`;+ try {+ const result = await new PiIntegrationTest({+ testName: "watch", artifactsDir: testArtifactsDir(import.meta.filename), cwd,+ extensions: [path.resolve("test/fixtures/display-extension.ts")], isolateUserResources: true,+ tools: ["codemode", "computer_delete"],+ conversation: [+ assistantMessage([toolCall({ id: "watch", name: "codemode", arguments: { code } })], { stopReason: "toolUse" }),+ assistantMessage([toolCall({ id: "close", name: "computer_delete", arguments: { path: "computer:session" } })], { stopReason: "toolUse" }),+ assistantMessage([text("Done")]),+ ],+ }).run("Watch the lane while dragging.");+ expect(getToolExecution(result, "watch").isError, getToolResultText(result, "watch")).toBe(false);+ const report = JSON.parse(getToolResultText(result, "watch").match(/\{"watch".*\}/)![0]);+ // The drag ran 800 ms at 20 FPS: its receipt names the watch and frames captured while it ran.+ expect(report.spans).toHaveLength(1);+ expect(report.spans[0].sequence).toBe(report.watch);+ expect(report.spans[0].to_ms - report.spans[0].from_ms).toBeGreaterThanOrEqual(800);+ expect(report.spans[0].frames.to - report.spans[0].frames.from).toBeGreaterThanOrEqual(5);+ // A stopped watch is no longer one.+ expect(report.again).toBe("refused");++ // The files: watch.json lists every frame; frames during the drag changed as the square moved; the animation exists.+ const out = path.join(cwd, "watch-out");+ const json = JSON.parse(await readFile(path.join(out, "watch.json"), "utf8"));+ expect(json.actions.map((action: { label: string }) => action.label)).toEqual(["drag"]);+ const during = json.frames.filter((frame: { actions: string[] }) => frame.actions.includes("drag"));+ expect(during.filter((frame: { changed_share: number }) => frame.changed_share > 0).length).toBeGreaterThanOrEqual(4);+ // The viewport crops the lane: 1000x200 window pixels, frames at most 800 px wide.+ expect(json.frames[0].width).toBe(800);+ expect(json.frames[0].height).toBe(160);+ for (const frame of json.frames) await access(path.join(out, frame.file));+ await access(path.join(out, "watch.webp"));++ // Two driver processes: the session's own did the drag, the second one captured the frames.+ const calls = (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));+ const dragPid = calls.find(call => call.name === "drag").pid;+ const capturePids = new Set(calls.filter(call => call.name === "get_window_state" && call.at > calls.find(c => c.name === "drag").at).map(call => call.pid));+ expect([...capturePids].some(pid => pid !== dragPid)).toBe(true);+ } finally { await rm(cwd, { recursive: true, force: true }); }+});diff --git a/test/watch.test.ts b/test/watch.test.tsnew file mode 100644index 0000000..d7eacbb--- /dev/null+++ b/test/watch.test.ts@@ -0,0 +1,26 @@+import { expect, test } from "vitest";+import { framesBetween, sheetFrames } from "../src/watch.ts";+import type { SequenceFrame } from "../src/temporal-capture.ts";++// Frames every 100 ms; `hashes` decides which differ from the one before.+const frames = (hashes: string[]) => hashes.map((hash, index) => ({ index, timestamp_ms: index * 100, hash, geometry_changed: false }) as SequenceFrame);++test("a watch's sheet shows every frame that changed and the last one", () => {+ const picked = sheetFrames(frames(["a", "a", "b", "b", "c", "c"]), []);+ expect(picked.map(frame => frame.index)).toEqual([0, 2, 4, 5]);+});++test("a sheet over its limit keeps the first, the last and the frames around each action, and spreads the rest", () => {+ // 100 frames that all differ; a drag from 3000 to 3500 ms.+ const all = frames(Array.from({ length: 100 }, (_, i) => String(i)));+ const picked = sheetFrames(all, [{ label: "drag", start_ms: 3000, end_ms: 3500 }], 10).map(frame => frame.index);+ expect(picked).toHaveLength(10);+ expect(picked).toEqual([...picked].sort((a, b) => a - b));+ for (const index of [0, 99, 30, 35, 36]) expect(picked).toContain(index);+});++test("an action's span names the frames captured during it, or none", () => {+ const all = frames(["a", "b", "c", "d", "e"]);+ expect(framesBetween(all, 150, 320)).toEqual({ from: 2, to: 3 });+ expect(framesBetween(all, 410, 450)).toBeNull();+});