Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Pin runtime 2437a85e: a viewport watch gets only the viewport from the driver and overlaps slow captures (LPT-950)
alexshpunt-clanker[bot]cfb6d99a2893
diff --git a/src/driver.ts b/src/driver.tsindex f17278d..a4f9a58 100644--- a/src/driver.ts+++ b/src/driver.ts@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {} as const;/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */export const WINDOW_MANAGEMENT_RUNTIME = {- source: "d9f380f68550691eec806dc45fdbc60f6d066de6", version: "0.37.0",- sha256: "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",+ source: "2437a85e339e1be2a0ab5b9c736227d1139127a3", version: "0.37.0",+ sha256: "00e4014073feddadd19f763cb59884cbe20c2f77d5c59c269784e0b15b733bf8",} as const;const forkRuntimes = {"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,@@ -161,6 +161,8 @@ export class McpDriver implements Driver {get windowRestore(): boolean { return ["windows-window-restore", "windows-window-management", "windows-desktop"].includes(this.config.runtime ?? ""); }/** Old profiles cannot borrow state-bound management from matching version numbers. */get windowManagement(): boolean { return ["windows-window-management", "windows-desktop"].includes(this.config.runtime ?? ""); }+ /** Display-only window reads with a crop arrived in the windows-desktop runtime 2437a85e. */+ get windowCrop(): boolean { return this.config.runtime === "windows-desktop"; }/** Overview metadata follows the native implementation, not an agent-chosen filter. */get overviewKernel(): "bilinear" | "lanczos3" { return ["windows-fast-bilinear", "windows-direct-process", "windows-persistent-wgc", "windows-native-pointer", "windows-coordinate-scroll", "windows-window-restore", "windows-window-management", "windows-desktop"].includes(this.config.runtime ?? "") ? "bilinear" : "lanczos3"; }constructor(private readonly config: ComputerConfig, private readonly version: "0.33.2" = "0.33.2") {diff --git a/src/observation-source.ts b/src/observation-source.tsindex 13edb5c..13efd49 100644--- a/src/observation-source.ts+++ b/src/observation-source.ts@@ -29,7 +29,7 @@ export interface ObservationSource extends SourceIdentity {original_dimensions: { width: number; height: number } | null;origin: { x: number; y: number; relative_to: "primary_display" | "display" | "window" } | null;native_bounds: unknown;- crop?: { requested: { x: number; y: number; width: number; height: number }; applied: { x: number; y: number; width: number; height: number }; coordinate_space: "uncropped_image_pixels" };+ crop?: { requested: { x: number; y: number; width: number; height: number }; applied: { x: number; y: number; width: number; height: number }; /** Old crops count in the uncropped image; a driver crop counts in the window's physical pixels. */ coordinate_space: "uncropped_image_pixels" | "window_physical_pixels" };};semantic_scope: "none" | "native_window";scale: { native_scale_factor: number | null; screenshot_scale: number | null; dpi: null; device_pixel_ratio: null };diff --git a/src/runtime-pins.json b/src/runtime-pins.jsonindex 0b359bf..4996762 100644--- a/src/runtime-pins.json+++ b/src/runtime-pins.json@@ -1,12 +1,12 @@{"format": 2,- "source": "d9f380f68550691eec806dc45fdbc60f6d066de6",+ "source": "2437a85e339e1be2a0ab5b9c736227d1139127a3","version": "0.37.0",- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-d9f380f6/",+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-2437a85e/","files": {- "cua-driver.exe": "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",- "cua-driver-uia.exe": "4da50c45a51a205c65625d352a95a6a937e914ac51426c81f439d128d8e2cf5d",- "cua-cursor-theme.exe": "bd201820ed990ed6b590bb289387138d9c0411649817c4870c4cf58e306dba4e",+ "cua-driver.exe": "00e4014073feddadd19f763cb59884cbe20c2f77d5c59c269784e0b15b733bf8",+ "cua-driver-uia.exe": "d61670d198ba7be4d867ac23b2483999cad6d44c3d4b0809c38340bc4a307c82",+ "cua-cursor-theme.exe": "cddecd9b9a47d2cab2c494b17ff18f7a507ebd6f392f31621346312055064cc0","LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41","THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"}diff --git a/src/session.ts b/src/session.tsindex e4555e0..d84e58e 100644--- a/src/session.ts+++ b/src/session.ts@@ -69,6 +69,8 @@ export interface Driver {windowManagement?: boolean;/** Fixed overview filter of the selected native runtime; never a tool-controlled resize switch. */overviewKernel?: CuaVisionSettings["kernel"];+ /** A display-only window read can return one rectangle of the window, encoding only that rectangle. */+ windowCrop?: boolean;call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;/**@@ -193,6 +195,23 @@ const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");+/** Watch captures in flight at once. A third measured no faster on ZinPC: part of each driver call is serial. */+const WATCH_CONCURRENCY = 2;+/** A watch frame the driver cropped to its viewport: the image is the viewport, at most `cap` on its long edge. */+function nativeCropFrame(state: Record<string, unknown>, images: SequenceFrame["image"][], requested: PixelBox, cap: number, base: ObservationSource) {+ const applied = state.screenshot_crop as PixelBox | undefined;+ const width = state.screenshot_width, height = state.screenshot_height;+ const originalWidth = state.screenshot_original_width, originalHeight = state.screenshot_original_height;+ if (!images[0] || images[0].data.length > 16 * 1024 * 1024 || !applied || typeof width !== "number" || typeof height !== "number"+ || typeof originalWidth !== "number" || typeof originalHeight !== "number" || Math.max(width, height) > cap)+ throw new SequenceStop("capture_failed");+ const source: ObservationSource = { ...base, semantic_scope: "none", capabilities: { read: true, input: "none", detail: "unsupported" },+ image: { ...base.image, scope: "native_window_zoom", coordinate_space: "detail_image_pixels", dimensions: { width, height },+ original_dimensions: { width: originalWidth, height: originalHeight }, origin: { x: applied.x, y: applied.y, relative_to: "window" },+ crop: { requested, applied, coordinate_space: "window_physical_pixels" } } };+ return { image: images[0], source };+}+interface NamedWindow { pid: number; window_id: number; title: string }interface CoveringWindow extends NamedWindow { relation: string }/** The window a click result names as in front after the click closed its own window. */@@ -731,6 +750,8 @@ export class ComputerSession {const driver = this.watcher ??= this.driver.companion?.() ?? this.driver;// Watch frames come from the window's own pixels at a size that gives the crop maxLongEdge pixels on its long edge.let cap = Math.min(FULL_RESOLUTION, options.maxLongEdge);+ // With a native crop the driver copies and encodes only the viewport, in the window's physical pixels.+ let nativeCrop: PixelBox | undefined;const fullSize = (state: Record<string, unknown>) => {const width = state.screenshot_original_width, height = state.screenshot_original_height;if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;@@ -745,11 +766,18 @@ export class ComputerSession {if (!full) return this.result("refused", "The watch did not start: the window could not be captured.");const long = Math.max(options.crop.width, options.crop.height);cap = Math.max(64, Math.min(FULL_RESOLUTION, Math.ceil(Math.max(full.width, full.height) * Math.min(options.maxLongEdge, long) / long)));+ if (driver.windowCrop) {+ const kx = (probe.structuredContent!.screenshot_original_width as number) / full.width;+ const ky = (probe.structuredContent!.screenshot_original_height as number) / full.height, crop = options.crop;+ nativeCrop = { x: Math.floor(crop.x * kx), y: Math.floor(crop.y * ky), width: Math.max(1, Math.ceil(crop.width * kx)), height: Math.max(1, Math.ceil(crop.height * ky)) };+ cap = Math.max(64, Math.min(FULL_RESOLUTION, options.maxLongEdge, long));+ }}const capture = async (active: AbortSignal) => {if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");const nativeStarted = performance.now();- const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap }, active);+ const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap,+ ...(nativeCrop ? { display_only: true, crop: nativeCrop } : {}) }, active);const native_completed_at = performance.now();active.throwIfAborted();if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");@@ -759,6 +787,8 @@ export class ComputerSession {if ((state.pid !== undefined && state.pid !== entry.target.pid) || (state.window_id !== undefined && state.window_id !== entry.target.window_id))throw new SequenceStop("target_lost");const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");+ if (nativeCrop) return { ...nativeCropFrame(state, images, nativeCrop, cap, this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target })),+ native_completed_at, native_latency_ms: native_completed_at - nativeStarted };const context = this.overview(state, "watch-frame", false, images);if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024) throw new SequenceStop("capture_failed");let source = this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target });@@ -776,8 +806,9 @@ export class ComputerSession {};const stop = new AbortController();const watch: Watching = { window_ref, watching, started: 0, marks: [], stop, finished: false, done: Promise.resolve(undefined as unknown as FrameSequence) };+ // A frame of a large window takes longer than a frame interval; a second capture in flight keeps the rate up.watch.done = collectSequence({ duration_ms: options.timeout_ms, fps: options.fps, max_frames: WATCH_LIMITS.max_frames, timeout_ms: options.timeout_ms + 20_000 }, capture,- undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, started: (sequence, at) => {+ undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, concurrency: WATCH_CONCURRENCY, started: (sequence, at) => {watch.sequence = sequence; watch.started = at;sequence.configuration = { ...this.overviewSettings(), maxLongEdge: cap };this.sequences.put(sequence, true);diff --git a/src/temporal-capture.ts b/src/temporal-capture.tsindex 8b42852..6440e0b 100644--- a/src/temporal-capture.ts+++ b/src/temporal-capture.ts@@ -61,10 +61,16 @@ export interface CaptureHooks {stop?: AbortSignal;/** Receives the sequence, which then grows frame by frame, and the clock time its timestamps count from. */started?(sequence: FrameSequence, at: number): void;+ /**+ * How many captures may run at once (default 1). A capture slower than the frame interval then overlaps the next+ * slot's instead of making it skip; frames keep the order their captures started in.+ */+ concurrency?: number;}/**- * Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. A frame with the- * same bytes as the one before shares its image and adds nothing to the byte budget, so a still window costs little.+ * Sampling skips missed slots instead of bursting; with `concurrency` above 1 a slow capture overlaps the next slot's.+ * The clock seam allows deterministic timing contract checks. A frame with the same bytes as the one before shares its+ * image and adds nothing to the byte budget, so a still window costs little.*/export async function collectSequence(options: SequenceOptions,capture: (signal: AbortSignal) => Promise<{ image: SequenceFrame["image"]; source: ObservationSource; native_completed_at?: number; native_latency_ms?: number }>, signal?: AbortSignal,@@ -77,11 +83,40 @@ export async function collectSequence(options: SequenceOptions,elapsed_ms: 0, sample_span_ms: 0, actual_fps: null, last_frame_age_ms: null, skipped_slots: [], pending_slots: 0, payload_bytes: 0, stop_reason: "completed", frames: [] };hooks.started?.(sequence, started);const interval = 1000 / requested.fps; const slots = Math.ceil(requested.duration_ms / interval);+ const concurrency = Math.max(1, Math.floor(hooks.concurrency ?? 1));+ type Captured = Awaited<ReturnType<typeof capture>>;+ // Captures in flight, oldest first. Each settles into a value or an error, so none rejects unobserved.+ const pending: { slot: number; before: number; done: Promise<{ result?: Captured; error?: unknown; after: number }> }[] = [];+ /** Take the oldest capture's frame. False when the sequence must end. */+ let ended = false;+ const settle = async (): Promise<boolean> => {+ const { slot: at, before, done } = pending.shift()!;+ const { result, error, after } = await done;+ if (error !== undefined) throw error;+ active.throwIfAborted();+ const bytes = Buffer.from(result!.image.data, "base64");+ const hash = createHash("sha256").update(bytes).digest("hex");+ const previous = sequence.frames.at(-1);+ const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });+ const geometryChanged = !!previous && geometry(previous.source) !== geometry(result!.source);+ const repeated = !!previous && previous.hash === hash && !geometryChanged;+ if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; ended = true; return false; }+ sequence.frames.push({ index: sequence.frames.length, slot: at, timestamp_ms: (result!.native_completed_at ?? after) - started, capture_started_ms: before - started,+ capture_latency_ms: result!.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,+ source: result!.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result!.image });+ if (!repeated) sequence.payload_bytes += bytes.length;+ return true;+ };let slot = 0;try {- while (slot < slots) {+ sampling: while (slot < slots) {if (active.aborted) throw new SequenceStop(signal?.aborted ? "cancelled" : "timeout");- if (sequence.frames.length >= requested.max_frames) { sequence.stop_reason = "frame_limit"; break; }+ if (sequence.frames.length + pending.length >= requested.max_frames) {+ if (pending.length) { if (!await settle()) break; continue; }+ sequence.stop_reason = "frame_limit"; break;+ }+ // All capture slots are busy: the oldest capture's frame comes first.+ if (pending.length >= concurrency) { if (!await settle()) break; continue; }const wait = started + slot * interval - clock.now();if (wait > 0) await clock.wait(wait, active);active.throwIfAborted();@@ -91,22 +126,17 @@ export async function collectSequence(options: SequenceOptions,while (slot < slots) sequence.skipped_slots.push(slot++);}if (slot >= slots) break;- const before = clock.now(); const result = await capture(active); const after = clock.now();- active.throwIfAborted();- const bytes = Buffer.from(result.image.data, "base64");- const hash = createHash("sha256").update(bytes).digest("hex");- const previous = sequence.frames.at(-1);- const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });- const geometryChanged = !!previous && geometry(previous.source) !== geometry(result.source);- const repeated = !!previous && previous.hash === hash && !geometryChanged;- if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; break; }- sequence.frames.push({ index: sequence.frames.length, slot, timestamp_ms: (result.native_completed_at ?? after) - started, capture_started_ms: before - started,- capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,- source: result.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result.image });- if (!repeated) sequence.payload_bytes += bytes.length;+ const before = clock.now();+ pending.push({ slot, before, done: capture(active).then(result => ({ result, after: clock.now() }), error => ({ error: error ?? new Error("capture failed"), after: clock.now() })) });slot++;- while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);+ // One capture at a time waits for its frame before the next slot, as sequential sampling always has.+ if (concurrency === 1) {+ if (!await settle()) break sampling;+ while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);+ }}+ // Captures still running when sampling ends deliver their frames, unless a limit ended it.+ while (pending.length && !ended) if (!await settle()) break;} catch (error) {sequence.stop_reason = hooks.stop?.aborted ? "stopped" : signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";}