Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Pin runtime 2437a85e: a viewport watch gets only the viewport from the driver and overlaps slow captures (LPT-950)

alexshpunt-clanker[bot]cfb6d99a2893
diff --git a/src/driver.ts b/src/driver.ts
index f17278d..a4f9a58 100644
--- a/src/driver.ts
+++ b/src/driver.ts
@@ -86,8 +86,8 @@ export const WINDOW_RESTORE_RUNTIME = {
} as const;
/** Window management plus every earlier native capability; `windows-desktop` selects these bytes. Browser pages use Playwright, not this runtime. */
export const WINDOW_MANAGEMENT_RUNTIME = {
- source: "d9f380f68550691eec806dc45fdbc60f6d066de6", version: "0.37.0",
- sha256: "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",
+ source: "2437a85e339e1be2a0ab5b9c736227d1139127a3", version: "0.37.0",
+ sha256: "00e4014073feddadd19f763cb59884cbe20c2f77d5c59c269784e0b15b733bf8",
} as const;
const forkRuntimes = {
"windows-desktop": WINDOW_MANAGEMENT_RUNTIME,
@@ -161,6 +161,8 @@ export class McpDriver implements Driver {
get windowRestore(): boolean { return ["windows-window-restore", "windows-window-management", "windows-desktop"].includes(this.config.runtime ?? ""); }
/** Old profiles cannot borrow state-bound management from matching version numbers. */
get windowManagement(): boolean { return ["windows-window-management", "windows-desktop"].includes(this.config.runtime ?? ""); }
+ /** Display-only window reads with a crop arrived in the windows-desktop runtime 2437a85e. */
+ get windowCrop(): boolean { return this.config.runtime === "windows-desktop"; }
/** Overview metadata follows the native implementation, not an agent-chosen filter. */
get overviewKernel(): "bilinear" | "lanczos3" { return ["windows-fast-bilinear", "windows-direct-process", "windows-persistent-wgc", "windows-native-pointer", "windows-coordinate-scroll", "windows-window-restore", "windows-window-management", "windows-desktop"].includes(this.config.runtime ?? "") ? "bilinear" : "lanczos3"; }
constructor(private readonly config: ComputerConfig, private readonly version: "0.33.2" = "0.33.2") {
diff --git a/src/observation-source.ts b/src/observation-source.ts
index 13edb5c..13efd49 100644
--- a/src/observation-source.ts
+++ b/src/observation-source.ts
@@ -29,7 +29,7 @@ export interface ObservationSource extends SourceIdentity {
original_dimensions: { width: number; height: number } | null;
origin: { x: number; y: number; relative_to: "primary_display" | "display" | "window" } | null;
native_bounds: unknown;
- crop?: { requested: { x: number; y: number; width: number; height: number }; applied: { x: number; y: number; width: number; height: number }; coordinate_space: "uncropped_image_pixels" };
+ crop?: { requested: { x: number; y: number; width: number; height: number }; applied: { x: number; y: number; width: number; height: number }; /** Old crops count in the uncropped image; a driver crop counts in the window's physical pixels. */ coordinate_space: "uncropped_image_pixels" | "window_physical_pixels" };
};
semantic_scope: "none" | "native_window";
scale: { native_scale_factor: number | null; screenshot_scale: number | null; dpi: null; device_pixel_ratio: null };
diff --git a/src/runtime-pins.json b/src/runtime-pins.json
index 0b359bf..4996762 100644
--- a/src/runtime-pins.json
+++ b/src/runtime-pins.json
@@ -1,12 +1,12 @@
{
"format": 2,
- "source": "d9f380f68550691eec806dc45fdbc60f6d066de6",
+ "source": "2437a85e339e1be2a0ab5b9c736227d1139127a3",
"version": "0.37.0",
- "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-d9f380f6/",
+ "url": "https://github.com/alexshpunt/cua/releases/download/pi-runtime-0.37.0-2437a85e/",
"files": {
- "cua-driver.exe": "057eecd1150fa462d9838e7475a35690d29a05fb2783ed1c91083a2605a973eb",
- "cua-driver-uia.exe": "4da50c45a51a205c65625d352a95a6a937e914ac51426c81f439d128d8e2cf5d",
- "cua-cursor-theme.exe": "bd201820ed990ed6b590bb289387138d9c0411649817c4870c4cf58e306dba4e",
+ "cua-driver.exe": "00e4014073feddadd19f763cb59884cbe20c2f77d5c59c269784e0b15b733bf8",
+ "cua-driver-uia.exe": "d61670d198ba7be4d867ac23b2483999cad6d44c3d4b0809c38340bc4a307c82",
+ "cua-cursor-theme.exe": "cddecd9b9a47d2cab2c494b17ff18f7a507ebd6f392f31621346312055064cc0",
"LICENSE.md": "d0a32419a44fa38d5023d1431dfcbedd1457eb5e7b2e6c7c87ca064facb41b41",
"THIRD_PARTY_NOTICES.md": "88a24f59293f03b3e03655e8144bcfadc9861bef1182cfeea91639e11f573fd1"
}
diff --git a/src/session.ts b/src/session.ts
index e4555e0..d84e58e 100644
--- a/src/session.ts
+++ b/src/session.ts
@@ -69,6 +69,8 @@ export interface Driver {
windowManagement?: boolean;
/** Fixed overview filter of the selected native runtime; never a tool-controlled resize switch. */
overviewKernel?: CuaVisionSettings["kernel"];
+ /** A display-only window read can return one rectangle of the window, encoding only that rectangle. */
+ windowCrop?: boolean;
call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;
identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;
/**
@@ -193,6 +195,23 @@ const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+
/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */
const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");
+/** Watch captures in flight at once. A third measured no faster on ZinPC: part of each driver call is serial. */
+const WATCH_CONCURRENCY = 2;
+/** A watch frame the driver cropped to its viewport: the image is the viewport, at most `cap` on its long edge. */
+function nativeCropFrame(state: Record<string, unknown>, images: SequenceFrame["image"][], requested: PixelBox, cap: number, base: ObservationSource) {
+ const applied = state.screenshot_crop as PixelBox | undefined;
+ const width = state.screenshot_width, height = state.screenshot_height;
+ const originalWidth = state.screenshot_original_width, originalHeight = state.screenshot_original_height;
+ if (!images[0] || images[0].data.length > 16 * 1024 * 1024 || !applied || typeof width !== "number" || typeof height !== "number"
+ || typeof originalWidth !== "number" || typeof originalHeight !== "number" || Math.max(width, height) > cap)
+ throw new SequenceStop("capture_failed");
+ const source: ObservationSource = { ...base, semantic_scope: "none", capabilities: { read: true, input: "none", detail: "unsupported" },
+ image: { ...base.image, scope: "native_window_zoom", coordinate_space: "detail_image_pixels", dimensions: { width, height },
+ original_dimensions: { width: originalWidth, height: originalHeight }, origin: { x: applied.x, y: applied.y, relative_to: "window" },
+ crop: { requested, applied, coordinate_space: "window_physical_pixels" } } };
+ return { image: images[0], source };
+}
+
interface NamedWindow { pid: number; window_id: number; title: string }
interface CoveringWindow extends NamedWindow { relation: string }
/** The window a click result names as in front after the click closed its own window. */
@@ -731,6 +750,8 @@ export class ComputerSession {
const driver = this.watcher ??= this.driver.companion?.() ?? this.driver;
// Watch frames come from the window's own pixels at a size that gives the crop maxLongEdge pixels on its long edge.
let cap = Math.min(FULL_RESOLUTION, options.maxLongEdge);
+ // With a native crop the driver copies and encodes only the viewport, in the window's physical pixels.
+ let nativeCrop: PixelBox | undefined;
const fullSize = (state: Record<string, unknown>) => {
const width = state.screenshot_original_width, height = state.screenshot_original_height;
if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;
@@ -745,11 +766,18 @@ export class ComputerSession {
if (!full) return this.result("refused", "The watch did not start: the window could not be captured.");
const long = Math.max(options.crop.width, options.crop.height);
cap = Math.max(64, Math.min(FULL_RESOLUTION, Math.ceil(Math.max(full.width, full.height) * Math.min(options.maxLongEdge, long) / long)));
+ if (driver.windowCrop) {
+ const kx = (probe.structuredContent!.screenshot_original_width as number) / full.width;
+ const ky = (probe.structuredContent!.screenshot_original_height as number) / full.height, crop = options.crop;
+ nativeCrop = { x: Math.floor(crop.x * kx), y: Math.floor(crop.y * ky), width: Math.max(1, Math.ceil(crop.width * kx)), height: Math.max(1, Math.ceil(crop.height * ky)) };
+ cap = Math.max(64, Math.min(FULL_RESOLUTION, options.maxLongEdge, long));
+ }
}
const capture = async (active: AbortSignal) => {
if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
const nativeStarted = performance.now();
- const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap }, active);
+ const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap,
+ ...(nativeCrop ? { display_only: true, crop: nativeCrop } : {}) }, active);
const native_completed_at = performance.now();
active.throwIfAborted();
if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
@@ -759,6 +787,8 @@ export class ComputerSession {
if ((state.pid !== undefined && state.pid !== entry.target.pid) || (state.window_id !== undefined && state.window_id !== entry.target.window_id))
throw new SequenceStop("target_lost");
const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");
+ if (nativeCrop) return { ...nativeCropFrame(state, images, nativeCrop, cap, this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target })),
+ native_completed_at, native_latency_ms: native_completed_at - nativeStarted };
const context = this.overview(state, "watch-frame", false, images);
if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024) throw new SequenceStop("capture_failed");
let source = this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target });
@@ -776,8 +806,9 @@ export class ComputerSession {
};
const stop = new AbortController();
const watch: Watching = { window_ref, watching, started: 0, marks: [], stop, finished: false, done: Promise.resolve(undefined as unknown as FrameSequence) };
+ // A frame of a large window takes longer than a frame interval; a second capture in flight keeps the rate up.
watch.done = collectSequence({ duration_ms: options.timeout_ms, fps: options.fps, max_frames: WATCH_LIMITS.max_frames, timeout_ms: options.timeout_ms + 20_000 }, capture,
- undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, started: (sequence, at) => {
+ undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, concurrency: WATCH_CONCURRENCY, started: (sequence, at) => {
watch.sequence = sequence; watch.started = at;
sequence.configuration = { ...this.overviewSettings(), maxLongEdge: cap };
this.sequences.put(sequence, true);
diff --git a/src/temporal-capture.ts b/src/temporal-capture.ts
index 8b42852..6440e0b 100644
--- a/src/temporal-capture.ts
+++ b/src/temporal-capture.ts
@@ -61,10 +61,16 @@ export interface CaptureHooks {
stop?: AbortSignal;
/** Receives the sequence, which then grows frame by frame, and the clock time its timestamps count from. */
started?(sequence: FrameSequence, at: number): void;
+ /**
+ * How many captures may run at once (default 1). A capture slower than the frame interval then overlaps the next
+ * slot's instead of making it skip; frames keep the order their captures started in.
+ */
+ concurrency?: number;
}
/**
- * Sequential sampling skips missed slots. The clock seam allows deterministic timing contract checks. A frame with the
- * same bytes as the one before shares its image and adds nothing to the byte budget, so a still window costs little.
+ * Sampling skips missed slots instead of bursting; with `concurrency` above 1 a slow capture overlaps the next slot's.
+ * The clock seam allows deterministic timing contract checks. A frame with the same bytes as the one before shares its
+ * image and adds nothing to the byte budget, so a still window costs little.
*/
export async function collectSequence(options: SequenceOptions,
capture: (signal: AbortSignal) => Promise<{ image: SequenceFrame["image"]; source: ObservationSource; native_completed_at?: number; native_latency_ms?: number }>, signal?: AbortSignal,
@@ -77,11 +83,40 @@ export async function collectSequence(options: SequenceOptions,
elapsed_ms: 0, sample_span_ms: 0, actual_fps: null, last_frame_age_ms: null, skipped_slots: [], pending_slots: 0, payload_bytes: 0, stop_reason: "completed", frames: [] };
hooks.started?.(sequence, started);
const interval = 1000 / requested.fps; const slots = Math.ceil(requested.duration_ms / interval);
+ const concurrency = Math.max(1, Math.floor(hooks.concurrency ?? 1));
+ type Captured = Awaited<ReturnType<typeof capture>>;
+ // Captures in flight, oldest first. Each settles into a value or an error, so none rejects unobserved.
+ const pending: { slot: number; before: number; done: Promise<{ result?: Captured; error?: unknown; after: number }> }[] = [];
+ /** Take the oldest capture's frame. False when the sequence must end. */
+ let ended = false;
+ const settle = async (): Promise<boolean> => {
+ const { slot: at, before, done } = pending.shift()!;
+ const { result, error, after } = await done;
+ if (error !== undefined) throw error;
+ active.throwIfAborted();
+ const bytes = Buffer.from(result!.image.data, "base64");
+ const hash = createHash("sha256").update(bytes).digest("hex");
+ const previous = sequence.frames.at(-1);
+ const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });
+ const geometryChanged = !!previous && geometry(previous.source) !== geometry(result!.source);
+ const repeated = !!previous && previous.hash === hash && !geometryChanged;
+ if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; ended = true; return false; }
+ sequence.frames.push({ index: sequence.frames.length, slot: at, timestamp_ms: (result!.native_completed_at ?? after) - started, capture_started_ms: before - started,
+ capture_latency_ms: result!.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,
+ source: result!.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result!.image });
+ if (!repeated) sequence.payload_bytes += bytes.length;
+ return true;
+ };
let slot = 0;
try {
- while (slot < slots) {
+ sampling: while (slot < slots) {
if (active.aborted) throw new SequenceStop(signal?.aborted ? "cancelled" : "timeout");
- if (sequence.frames.length >= requested.max_frames) { sequence.stop_reason = "frame_limit"; break; }
+ if (sequence.frames.length + pending.length >= requested.max_frames) {
+ if (pending.length) { if (!await settle()) break; continue; }
+ sequence.stop_reason = "frame_limit"; break;
+ }
+ // All capture slots are busy: the oldest capture's frame comes first.
+ if (pending.length >= concurrency) { if (!await settle()) break; continue; }
const wait = started + slot * interval - clock.now();
if (wait > 0) await clock.wait(wait, active);
active.throwIfAborted();
@@ -91,22 +126,17 @@ export async function collectSequence(options: SequenceOptions,
while (slot < slots) sequence.skipped_slots.push(slot++);
}
if (slot >= slots) break;
- const before = clock.now(); const result = await capture(active); const after = clock.now();
- active.throwIfAborted();
- const bytes = Buffer.from(result.image.data, "base64");
- const hash = createHash("sha256").update(bytes).digest("hex");
- const previous = sequence.frames.at(-1);
- const geometry = (source: ObservationSource) => JSON.stringify({ image: source.image, scale: source.scale });
- const geometryChanged = !!previous && geometry(previous.source) !== geometry(result.source);
- const repeated = !!previous && previous.hash === hash && !geometryChanged;
- if (!repeated && (bytes.length > 12 * 1024 * 1024 || sequence.payload_bytes + bytes.length > byteBudget)) { sequence.stop_reason = "byte_limit"; break; }
- sequence.frames.push({ index: sequence.frames.length, slot, timestamp_ms: (result.native_completed_at ?? after) - started, capture_started_ms: before - started,
- capture_latency_ms: result.native_latency_ms ?? after - before, sample_latency_ms: after - before, hash, bytes: bytes.length,
- source: result.source, geometry_changed: geometryChanged, image: repeated ? previous.image : result.image });
- if (!repeated) sequence.payload_bytes += bytes.length;
+ const before = clock.now();
+ pending.push({ slot, before, done: capture(active).then(result => ({ result, after: clock.now() }), error => ({ error: error ?? new Error("capture failed"), after: clock.now() })) });
slot++;
- while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);
+ // One capture at a time waits for its frame before the next slot, as sequential sampling always has.
+ if (concurrency === 1) {
+ if (!await settle()) break sampling;
+ while (slot < slots && slot * interval < clock.now() - started) sequence.skipped_slots.push(slot++);
+ }
}
+ // Captures still running when sampling ends deliver their frames, unless a limit ended it.
+ while (pending.length && !ended) if (!await settle()) break;
} catch (error) {
sequence.stop_reason = hooks.stop?.aborted ? "stopped" : signal?.aborted ? "cancelled" : timeout.aborted ? "timeout" : error instanceof SequenceStop ? error.reason : "capture_failed";
}