Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { createHash, randomUUID } from "node:crypto";
import sharp from "sharp";
import { setTimeout as delay } from "node:timers/promises";
import type { Content } from "./session.ts";
import type { ObservationSource } from "./observation-source.ts";
import type { CuaVisionSettings } from "./cua-image-settings.ts";
/** Bounded native sampling. Region uses pixels in the first uncropped frame at the frozen image cap. */
export interface SequenceOptions {
duration_ms: number; fps?: number; max_frames?: number; timeout_ms?: number;
maxLongEdge?: number; scale?: number; region?: { x: number; y: number; width: number; height: number };
}
/** A stored frame is read-only. Native completion is the timestamp boundary, not compositor presentation. */
export interface SequenceFrame {
index: number; slot: number; timestamp_ms: number; capture_started_ms: number; capture_latency_ms: number; sample_latency_ms: number;
hash: string; bytes: number; source: ObservationSource; geometry_changed: boolean;
image: Extract<Content, { type: "image" }>;
}
/** No images are included in the public capture metadata. Partial prefixes never mean full success. */
export interface FrameSequence {
sequence_id: string; requested: { duration_ms: number; fps: number; max_frames: number; timeout_ms: number };
started_at: string; elapsed_ms: number; sample_span_ms: number; actual_fps: number | null;
/** How old the newest frame was when the sequence returned: time from its native completion to the result. */
last_frame_age_ms: number | null;
skipped_slots: number[]; pending_slots: number; payload_bytes: number;
/** stopped: a watch stopped by request, with every frame it captured until then. */
stop_reason: "completed" | "stopped" | "frame_limit" | "byte_limit" | "cancelled" | "timeout" | "target_lost" | "access_revoked" | "capture_failed";
frames: SequenceFrame[];
configuration?: CuaVisionSettings;
interval_ms?: { min: number; max: number; mean: number };
}
/** Inclusive range and indexes refer to captured frame indexes, not scheduled sampling slots. */
export type FrameSelection = "all" | "first" | "last" | "endpoints" | "changed" | { indexes: number[] } | { from: number; to: number };
const byteBudget = 48 * 1024 * 1024;
/** A scoped capture can stop without exposing backend diagnostics or falling back to another target. */
export class SequenceStop extends Error {
constructor(readonly reason: FrameSequence["stop_reason"]) { super(reason); }
}
/** Bounds of a capture: its length, rate, frame count and total time. */
export interface CaptureLimits { duration_ms: number; fps: number; max_frames: number; timeout_ms: number }
/** Limits of one sequence: a minute of frames at up to 30 FPS; one Read still selects at most 40 of them. */
export const SEQUENCE_LIMITS = { duration_ms: 60_000, fps: 30, max_frames: 300, timeout_ms: 90_000, selected: 40 } as const;
/** Validate before native dispatch; requested FPS is a sampling goal, never a performance promise. */
export function validateSequence(options: SequenceOptions, L: CaptureLimits = SEQUENCE_LIMITS): FrameSequence["requested"] {
const result = { duration_ms: options.duration_ms, fps: options.fps ?? 4, max_frames: options.max_frames ?? 40, timeout_ms: options.timeout_ms ?? 15000 };
if (!Number.isSafeInteger(result.duration_ms) || result.duration_ms < 1 || result.duration_ms > L.duration_ms ||
!Number.isFinite(result.fps) || result.fps < 1 || result.fps > L.fps ||
!Number.isSafeInteger(result.max_frames) || result.max_frames < 1 || result.max_frames > L.max_frames ||
!Number.isSafeInteger(result.timeout_ms) || result.timeout_ms < result.duration_ms || result.timeout_ms > L.timeout_ms)
throw Error(`Use duration_ms 1..${L.duration_ms}, fps 1..${L.fps}, max_frames 1..${L.max_frames} and timeout_ms duration..${L.timeout_ms}.`);
const r = options.region;
if (r && (![r.x, r.y, r.width, r.height].every(Number.isSafeInteger) || r.x < 0 || r.y < 0 || r.width < 1 || r.height < 1))
throw Error("Region uses positive integer dimensions and non-negative local pixel coordinates.");
return result;
}
/** What a long-running capture such as a watch adds: its own limits, a stop request that keeps the frames, and its sequence as soon as it starts. */
export interface CaptureHooks {
limits?: CaptureLimits;
/** Ends sampling with stop_reason "stopped"; frames captured until then stay. */
stop?: AbortSignal;
/** Receives the sequence, which then grows frame by frame, and the clock time its timestamps count from. */
started?(sequence: FrameSequence, at: number): void;
/**
* How many captures may run at once (default 1). A capture slower than the frame interval then overlaps the next
* slot's instead of making it skip; frames keep the order their captures started in.
*/
concurrency?: number;
}
/**
* Sampling skips missed slots instead of bursting; with `concurrency` above 1 a slow capture overlaps the next slot's.
* The clock seam allows deterministic timing contract checks. A frame with the same bytes as the one before shares its
* image and adds nothing to the byte budget, so a still window costs little.
*/
export async function collectSequence(options: SequenceOptions,
capture: (signal: AbortSignal) => Promise<{ image: SequenceFrame["image"]; source: ObservationSource; native_completed_at?: number; native_latency_ms?: number }>, signal?: AbortSignal,
clock: { now(): number; wait(ms: number, active: AbortSignal): Promise<void> } = { now: () => performance.now(), wait: (ms, active) => delay(ms, undefined, { signal: active }) },
hooks: CaptureHooks = {}): Promise<FrameSequence> {
const requested = validateSequence(options, hooks.limits); const started = clock.now();
const timeout = AbortSignal.timeout(requested.timeout_ms);
const active = AbortSignal.any([timeout, ...(signal ? [signal] : []), ...(hooks.stop ? [hooks.stop] : [])]);
const sequence: FrameSequence = { sequence_id: "seq-" + randomUUID(), requested, started_at: new Date().toISOString(),
elapsed_ms: 0, sample_span_ms: 0, actual_fps: null, last_frame_age_ms: null, skipped_slots: [], pending_slots: 0, payload_bytes: 0, stop_reason: "completed", frames: [] };
hooks.started?.(sequence, started);
const interval = 1000 / requested.fps; const slots = Math.ceil(requested.duration_ms / interval);
const concurrency = Math.max(1, Math.floor(hooks.concurrency ?? 1));
type Captured = Awaited<ReturnType<typeof capture>>;
// Captures in flight, oldest first. Each settles into a value or an error, so none rejects unobserved.
const pending: { slot: number; before: number; done: Promise<{ result?: Captured; error?: unknown; after: number }> }[] = [];
/** Take the oldest capture's frame. False when the sequence must end. */
let ended = false;
const settle = async (): Promise<boolean> => {
const { slot: at, before, done } = pending.shift()!;
const { result, error, after } = await done;
if (error !== undefined) throw error;
active.throwIfAborted();
const bytes = Buffer.from(result!.image.data, "base64");
const hash = createHash("sha256").update(bytes).digest("hex");
const previous = sequence.frames.at(-1);
const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });
const geometryChanged = !!previous && geometry(previous.source) !== geometry(result!.source);
const repeated = !!previous && previous.hash === hash && !geometryChanged;
if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; ended = true; return false; }
sequence.frames.push({ index: sequence.frames.length, slot: at, timestamp_ms: (result!.native_completed_at ?? after) - started, capture_started_ms: before - started,
capture_latency_ms: result!.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,
source: result!.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result!.image });
if (!repeated) sequence.payload_bytes += bytes.length;
return true;
};
let slot = 0;
try {
sampling: while (slot < slots) {
if (active.aborted) throw new SequenceStop(signal?.aborted ? "cancelled" : "timeout");
if (sequence.frames.length + pending.length >= requested.max_frames) {
if (pending.length) { if (!await settle()) break; continue; }
sequence.stop_reason = "frame_limit"; break;
}
// All capture slots are busy: the oldest capture's frame comes first.
if (pending.length >= concurrency) { if (!await settle()) break; continue; }
const wait = started + slot * interval - clock.now();
if (wait > 0) await clock.wait(wait, active);
active.throwIfAborted();
// A delayed wake can miss more than one scheduled slot. Never catch up with a burst.
while (slot < slots && (slot + 1) * interval <= clock.now() - started) sequence.skipped_slots.push(slot++);
if (slot > 0 && clock.now() - started >= requested.duration_ms) {
while (slot < slots) sequence.skipped_slots.push(slot++);
}
if (slot >= slots) break;
const before = clock.now();
pending.push({ slot, before, done: capture(active).then(result => ({ result, after: clock.now() }), error => ({ error: error ?? new Error("capture failed"), after: clock.now() })) });
slot++;
// One capture at a time waits for its frame before the next slot, as sequential sampling always has.
if (concurrency === 1) {
if (!await settle()) break sampling;
while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);
}
}
// Captures still running when sampling ends deliver their frames, unless a limit ended it.
while (pending.length && !ended) if (!await settle()) break;
} catch (error) {
sequence.stop_reason = hooks.stop?.aborted ? "stopped" : signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";
}
sequence.elapsed_ms = clock.now() - started;
sequence.last_frame_age_ms = sequence.frames.length ? sequence.elapsed_ms - sequence.frames.at(-1)!.timestamp_ms : null;
// A stop request ends a watch whose duration is only its safety limit: the slots after it were never due.
sequence.pending_slots = sequence.stop_reason === "stopped" ? 0 : slots - sequence.frames.length - sequence.skipped_slots.length;
sequence.sample_span_ms = sequence.frames.length > 1 ? sequence.frames.at(-1)!.timestamp_ms - sequence.frames[0].timestamp_ms : 0;
sequence.actual_fps = sequence.sample_span_ms > 0 ? (sequence.frames.length - 1) * 1000 / sequence.sample_span_ms : null;
const intervals = sequence.frames.slice(1).map((frame, i) => frame.timestamp_ms - sequence.frames[i].timestamp_ms);
if (intervals.length) sequence.interval_ms = { min: Math.min(...intervals), max: Math.max(...intervals), mean: sequence.sample_span_ms / intervals.length };
return sequence;
}
/** Small valid-JSON text projection; full per-frame source geometry remains in structured results. */
export function frameSummary(frame: Omit<SequenceFrame, "image">) {
return { index: frame.index, slot: frame.slot, timestamp_ms: frame.timestamp_ms, capture_latency_ms: frame.capture_latency_ms,
sample_latency_ms: frame.sample_latency_ms, hash: frame.hash, bytes: frame.bytes, geometry_changed: frame.geometry_changed,
dimensions: frame.source.image.dimensions, native_bounds: frame.source.image.native_bounds };
}
/** Remove pixel payloads for tool metadata, trace and context. */
export function sequenceMetadata(sequence: FrameSequence) {
return { ...sequence, frames: sequence.frames.map(({ image: _image, ...frame }) => frame),
timestamp_boundary: "native_call_completed", input: "none", retention: "session-local; newest 3 sequences within 48 MiB" };
}
/** A sequence still collecting: no stop reason or slot counts yet. */
export type RunningSequenceMetadata = Omit<ReturnType<typeof sequenceMetadata>, "stop_reason" | "pending_slots" | "skipped_slots" | "interval_ms">
& { running: true; stop_reason?: never; pending_slots?: never; skipped_slots?: never; interval_ms?: never };
/**
* Metadata of a sequence that is still collecting, such as a running watch, `elapsed_ms` after its start: the frames
* so far and the rate they reach. A stop reason and slot counts exist only once it ends.
*/
export function runningSequenceMetadata(sequence: FrameSequence, elapsed_ms: number): RunningSequenceMetadata {
const { stop_reason: _stop, pending_slots: _pending, skipped_slots: _skipped, interval_ms: _intervals, ...metadata } = sequenceMetadata(sequence);
const frames = sequence.frames, span = frames.length > 1 ? frames.at(-1)!.timestamp_ms - frames[0].timestamp_ms : 0;
return { ...metadata, running: true as const, elapsed_ms, sample_span_ms: span, actual_fps: span > 0 ? (frames.length - 1) * 1000 / span : null,
last_frame_age_ms: frames.length ? elapsed_ms - frames.at(-1)!.timestamp_ms : null };
}
/** Select an ordered subset; changed means exact byte changes, not a model judgement. */
export function selectFrames(sequence: FrameSequence, selection: FrameSelection): SequenceFrame[] {
const frames = sequence.frames;
if (selection === "all") {
if (frames.length > SEQUENCE_LIMITS.selected) throw Error(`This sequence has ${frames.length} frames; select at most ${SEQUENCE_LIMITS.selected} with indexes, a range or changed.`);
return frames;
}
if (selection === "first") return frames.slice(0, 1);
if (selection === "last") return frames.slice(-1);
if (selection === "endpoints") return frames.filter((_, i) => i === 0 || i === frames.length - 1);
if (selection === "changed") return frames.filter((f, i) => i === 0 || f.hash !== frames[i - 1].hash || f.geometry_changed);
if (!selection || typeof selection !== "object") throw Error("Choose all, first, last, endpoints, changed, indexes or an inclusive range.");
const indexes = "indexes" in selection ? selection.indexes : Array.from({ length: selection.to - selection.from + 1 }, (_, i) => selection.from + i);
if (!Array.isArray(indexes) || indexes.length < 1 || indexes.length > SEQUENCE_LIMITS.selected ||
indexes.some(i => !Number.isSafeInteger(i) || i < 0 || i >= frames.length) || new Set(indexes).size !== indexes.length)
throw Error("Selection needs unique valid captured-frame indexes or an inclusive from..to range.");
return frames.filter(f => indexes.includes(f.index));
}
/** Crop fresh CUA frame pixels only; resize clips the fixed first-frame region instead of expanding it. */
export async function cropSequenceFrame(image: SequenceFrame["image"], source: ObservationSource, requested: NonNullable<SequenceOptions["region"]>, first: boolean) {
const width = source.image.dimensions.width!; const height = source.image.dimensions.height!;
if (first && (requested.x + requested.width > width || requested.y + requested.height > height))
throw Error("Region must fit the first uncropped frame at the requested image size.");
const applied = { ...requested, width: Math.min(requested.width, width - requested.x), height: Math.min(requested.height, height - requested.y) };
if (applied.width <= 0 || applied.height <= 0) throw Error("Resized frame no longer contains the region.");
const bytes = await sharp(Buffer.from(image.data, "base64"), { limitInputPixels: 32 * 1024 * 1024 })
.extract({ left: applied.x, top: applied.y, width: applied.width, height: applied.height }).png().toBuffer();
const native = source.image.original_dimensions;
const origin = native ? { x: requested.x * native.width / width, y: requested.y * native.height / height,
relative_to: source.kind === "display" ? "primary_display" as const : "window" as const } : null;
return { image: { ...image, data: bytes.toString("base64") }, source: { ...source, image: { ...source.image,
scope: source.kind === "display" ? "primary_display_region" as const : "native_window_zoom" as const,
coordinate_space: "detail_image_pixels" as const, dimensions: { width: applied.width, height: applied.height }, origin,
crop: { requested, applied, coordinate_space: "uncropped_image_pixels" as const } } } };
}
/**
* Private bounded frame resources. Reload/close drops pixels; readers must recheck source access. A running watch is
* never evicted; once it stops, putting it again makes it an ordinary sequence.
*/
export class SequenceStore {
private readonly sequences = new Map<string, FrameSequence>();
private readonly running = new Set<string>();
put(sequence: FrameSequence, running = false): void {
this.sequences.delete(sequence.sequence_id);
this.sequences.set(sequence.sequence_id, sequence);
if (running) this.running.add(sequence.sequence_id); else this.running.delete(sequence.sequence_id);
const evictable = () => [...this.sequences.keys()].filter(id => !this.running.has(id));
while (evictable().length && (evictable().length > 3 || [...this.sequences.values()].reduce((sum, s) => sum + s.payload_bytes, 0) > byteBudget))
this.sequences.delete(evictable()[0]);
}
get(id: string): FrameSequence | undefined { return this.sequences.get(id); }
clear(): void { this.sequences.clear(); this.running.clear(); }
}