Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import sharp from "sharp";
import { marksAt, type FrameMark } from "./contact-sheet.ts";
import { decodeFrame, diffFrames, type PixelBox, type PixelFrame } from "./pixel-diff.ts";
import type { CaptureLimits, FrameSequence, SequenceFrame } from "./temporal-capture.ts";
/**
* A watch captures a window or a viewport in the background while the agent acts, until it is stopped, its time
* limit passes or the session closes. Frames that repeat the one before share its image, so a still window costs
* little; the frame limit bounds a busy one.
*/
export const WATCH_LIMITS: CaptureLimits = { duration_ms: 300_000, fps: 30, max_frames: 3000, timeout_ms: 320_000 };
/** Defaults of a watch: 10 FPS for a minute, frames at most 800 px on their long edge. */
export const WATCH_DEFAULTS = { fps: 10, timeout_ms: 60_000, maxLongEdge: 800 } as const;
/** Frames on the contact sheet a stopped watch returns. */
export const WATCH_SHEET_FRAMES = 40;
/** Frames of the animation, and its long edge. */
export const ANIMATION_LIMITS = { frames: 200, longEdge: 640 } as const;
/** What a receipt says about one watch of the window an action ran on: the action's span in the watch's time and the frames captured in it. */
export interface WatchSpan { sequence_id: string; label: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }
/** The first and last frame captured within a span of the watch's time; null when none was. */
export function framesBetween(frames: readonly Pick<SequenceFrame, "index" | "timestamp_ms">[], from: number, to: number): { from: number; to: number } | null {
const inside = frames.filter(frame => frame.timestamp_ms >= from && frame.timestamp_ms <= to);
return inside.length ? { from: inside[0].index, to: inside.at(-1)!.index } : null;
}
/** Whether a frame differs from the one before it: the first frame, other bytes or another size. */
export const differs = (frames: readonly SequenceFrame[], i: number) => i === 0 || frames[i].hash !== frames[i - 1].hash || frames[i].geometry_changed;
/**
* The frames of a watch's contact sheet, in time order: every frame that differs from the one before, plus the last.
* When there are more than `limit`, the first, the last and those around each action (the first changed frame at or
* after it starts, the last during it and the first after it) stay, and the rest are spread evenly over the watch.
*/
export function sheetFrames(frames: readonly SequenceFrame[], marks: readonly FrameMark[], limit = WATCH_SHEET_FRAMES): SequenceFrame[] {
if (!frames.length) return [];
const changed = frames.filter((_, i) => differs(frames, i));
if (changed.at(-1) !== frames.at(-1)) changed.push(frames.at(-1)!);
if (changed.length <= limit) return changed;
const keep = new Set<SequenceFrame>([changed[0], changed.at(-1)!]);
for (const mark of marks) {
const during = changed.filter(frame => frame.timestamp_ms >= mark.start_ms && frame.timestamp_ms <= mark.end_ms);
const after = changed.find(frame => frame.timestamp_ms > mark.end_ms);
for (const frame of [during[0], during.at(-1), after]) if (frame && keep.size < limit) keep.add(frame);
}
const rest = changed.filter(frame => !keep.has(frame));
const room = limit - keep.size;
for (let i = 0; i < room && rest.length; i++) keep.add(rest[Math.floor((i + 0.5) * rest.length / room)]);
return changed.filter(frame => keep.has(frame));
}
/** How one frame differs from the one before it, in its own pixels: share of changed pixels and the box around them. */
export interface FrameChange { index: number; timestamp_ms: number; share: number; box?: PixelBox; size_changed?: true }
/** Change statistics of every frame against the one before it; repeated frames changed nothing. */
export async function frameChanges(frames: readonly SequenceFrame[]): Promise<FrameChange[]> {
const changes: FrameChange[] = [];
let previous: PixelFrame | undefined;
for (let i = 0; i < frames.length; i++) {
const frame = frames[i];
if (i > 0 && !differs(frames, i)) { changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 }); continue; }
const pixels = await decodeFrame(Buffer.from(frame.image.data, "base64"));
if (!previous) changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: 0 });
else {
const diff = diffFrames(previous, pixels);
changes.push({ index: frame.index, timestamp_ms: frame.timestamp_ms, share: diff.size_changed ? 1 : diff.share,
...(diff.box ? { box: diff.box } : {}), ...(diff.size_changed ? { size_changed: true as const } : {}) });
}
previous = pixels;
}
return changes;
}
/** A short text of the frames that changed most, at most `limit` lines in time order. */
export function changeLines(changes: readonly FrameChange[], marks: readonly FrameMark[], limit = 20): string[] {
const moving = changes.filter(change => change.share > 0);
const shown = new Set([...moving].sort((a, b) => b.share - a.share).slice(0, limit));
return moving.filter(change => shown.has(change)).map(change => {
const actions = marksAt(marks, change.timestamp_ms);
const box = change.box ? ` in ${change.box.x},${change.box.y} ${change.box.width}x${change.box.height}` : "";
return `- #${change.index} at ${Math.round(change.timestamp_ms)} ms: ${change.size_changed ? "size changed" : `${(change.share * 100).toFixed(1)}%${box}`}${actions.length ? ` (during ${actions.join(", ")})` : ""}`;
});
}
/** The files a stopped watch leaves: each frame that differs from the one before as a PNG, watch.json, and with `animation` watch.webp. */
export interface WatchFiles { dir: string; frames: number; json: string; animation?: string }
/**
* Write a stopped watch's frames for further processing. watch.json lists every frame with its time, its file (shared
* by the repeats after it), its change against the frame before and the actions it falls in, and every action.
*/
export async function writeWatchFiles(dir: string, sequence: FrameSequence, marks: readonly FrameMark[], changes: readonly FrameChange[],
about: Record<string, unknown>, animation = false): Promise<WatchFiles> {
await mkdir(dir, { recursive: true });
const frames = sequence.frames;
const files: string[] = [];
let written = 0;
for (let i = 0; i < frames.length; i++) {
if (differs(frames, i)) {
files.push(`frame-${String(frames[i].index).padStart(5, "0")}.png`);
await writeFile(join(dir, files[i]), Buffer.from(frames[i].image.data, "base64"));
written++;
} else files.push(files[i - 1]);
}
const json = join(dir, "watch.json");
await writeFile(json, JSON.stringify({ ...about, sequence_id: sequence.sequence_id, started_at: sequence.started_at, stop_reason: sequence.stop_reason,
elapsed_ms: sequence.elapsed_ms, requested: sequence.requested, actual_fps: sequence.actual_fps, skipped_slots: sequence.skipped_slots.length, actions: marks,
frames: frames.map((frame, i) => ({ index: frame.index, timestamp_ms: frame.timestamp_ms, file: files[i], width: frame.source.image.dimensions.width,
height: frame.source.image.dimensions.height, changed_share: changes[i]?.share ?? null, changed_box: changes[i]?.box ?? null,
actions: marksAt(marks, frame.timestamp_ms) })) }, null, 2));
return { dir, frames: written, json, ...(animation && frames.length ? { animation: await writeAnimation(join(dir, "watch.webp"), frames) } : {}) };
}
/**
* An animated WebP of the frames that differ from the one before, each shown until the next one, played once in real
* time and then looped. More than ANIMATION_LIMITS.frames are spread evenly; all are fitted into one size.
*/
async function writeAnimation(path: string, frames: readonly SequenceFrame[]): Promise<string> {
let picked = frames.filter((_, i) => differs(frames, i));
if (picked.length > ANIMATION_LIMITS.frames)
picked = Array.from({ length: ANIMATION_LIMITS.frames }, (_, i) => picked[Math.floor(i * picked.length / ANIMATION_LIMITS.frames)]);
const first = picked[0].source.image.dimensions;
const scale = Math.min(1, ANIMATION_LIMITS.longEdge / Math.max(first.width!, first.height!));
const size = { width: Math.max(1, Math.round(first.width! * scale)), height: Math.max(1, Math.round(first.height! * scale)) };
const images = await Promise.all(picked.map(frame => sharp(Buffer.from(frame.image.data, "base64")).removeAlpha()
.resize(size.width, size.height, { fit: "contain", background: "#202020" }).png().toBuffer()));
const delay = picked.map((frame, i) => Math.max(20, Math.round((picked[i + 1]?.timestamp_ms ?? frame.timestamp_ms + 1000) - frame.timestamp_ms)));
await sharp(images, { join: { animated: true } }).webp({ delay, loop: 0, quality: 80 }).toFile(path);
return path;
}