Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Tests: a viewport watch of a large window keeps its frame rate (LPT-950)

alexshpunt-clanker[bot]d892a6307eba
diff --git a/test/live/watch.live.ts b/test/live/watch.live.ts
index c25476d..54ec125 100644
--- a/test/live/watch.live.ts
+++ b/test/live/watch.live.ts
@@ -15,7 +15,7 @@ afterAll(async () => { await mkdir(out, { recursive: true }); await writeFile(`$
const image = (result: ComputerResult) => Buffer.from((result.content.find(block => block.type === "image") as { data: string }).data, "base64");
/** A form with a 600x300 view of a wide strip of numbered colour bands that follows the pointer while the left button is held. */
-function panoramaForm(title: string): string[] {
+function panoramaForm(title: string, maximized = false): string[] {
const script = `Add-Type -AssemblyName System.Windows.Forms, System.Drawing
$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.Width = 660; $f.Height = 380
$v = New-Object Windows.Forms.Panel; $v.Left = 10; $v.Top = 10; $v.Width = 600; $v.Height = 300; $v.BackColor = 'Black'
@@ -29,7 +29,7 @@ for ($i = 0; $i -lt 16; $i++) {
$p = New-Object Windows.Forms.PictureBox; $p.Image = $bmp; $p.Width = 2400; $p.Height = 300; $p.Left = 0; $p.AccessibleName = 'Pano'
$p.Add_MouseDown({ $script:mx = [Windows.Forms.Control]::MousePosition.X; $script:left = $p.Left })
$p.Add_MouseMove({ if ([Windows.Forms.Control]::MouseButtons -eq 'Left') { $p.Left = [Math]::Min(0, [Math]::Max(-1800, $script:left + [Windows.Forms.Control]::MousePosition.X - $script:mx)) } })
-$v.Controls.Add($p); $f.Controls.Add($v); [void]$f.ShowDialog()`;
+$v.Controls.Add($p); $f.Controls.Add($v); ${maximized ? "$f.WindowState = 'Maximized'; " : ""}[void]$f.ShowDialog()`;
return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];
}
@@ -94,3 +94,34 @@ test("a watch of a viewport sees the panorama move during a drag and stops with
await live.resources.delete(window, live.cwd).catch(() => undefined);
}
}, 180_000);
+
+// LPT-950: a viewport of a maximized window: the driver encodes only the viewport, and a second capture overlaps a slow
+// one, so the watch keeps at least 12 of 15 FPS during a drag. Before, every frame was the whole window: 5 FPS at 4K.
+test("a watch of a viewport in a maximized window keeps its frame rate during a drag", async () => {
+ const title = `pi-watch-max-${Date.now()}`;
+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, panoramaForm(title, true), `regex:^${title}$`);
+ try {
+ await new Promise(resolve => setTimeout(resolve, 1500));
+ await live.resources.manageWindow(requireComputerAddress(window), live.cwd, windowCommand({ window, action: "focus" }));
+ const overview = await live.read(window, ["full"]);
+ const box = overview.details.ui!.elements.find(e => e.name === "Pano")!.bounds.window_image!;
+ const state = overview.details.state as { screenshot_width: number; screenshot_original_width: number };
+ const k = state.screenshot_width / state.screenshot_original_width;
+ const rect = { x: box.x, y: box.y, width: Math.round(600 * k), height: box.height };
+ const view = requireComputerAddress(`${window}/pano`);
+ expect((await live.resources.view(view, live.cwd, rect)).details.status).toBe("observed");
+ const watch = await live.resources.watch(view, live.cwd, { fps: 15 });
+ expect(watch.details.status, presentComputerResult(watch)).toBe("observed");
+ const from = { x: rect.x + Math.round(rect.width * 0.8), y: rect.y + Math.round(rect.height / 2) };
+ const dragged = await live.resources.input(requireComputerAddress(window), live.cwd, dragPlan({ window, from, to: { x: from.x - Math.round(300 * k), y: from.y }, duration_ms: 2000 }));
+ expect(dragged.details.status, presentComputerResult(dragged)).toBe("delivered");
+ const stopped = await live.resources.stopWatch(requireComputerAddress(`computer:sequence/${watch.details.watch!.sequence_id}`), live.cwd, {});
+ const sequence = stopped.details.sequence!;
+ report.push(`maximized window\n${(stopped.content[0] as { text: string }).text.split("\n")[0]}\ncapture latency ${JSON.stringify(sequence.frames.map(f => Math.round(f.capture_latency_ms)))}`);
+ expect(sequence.actual_fps).toBeGreaterThanOrEqual(12);
+ expect(sequence.frames[0].source.image.crop?.coordinate_space).toBe("window_physical_pixels");
+ } finally {
+ const { pid } = (await live.read(window, ["state"])).details.target as { pid: number };
+ await live.kill(pid);
+ }
+}, 180_000);
diff --git a/test/temporal-capture.test.ts b/test/temporal-capture.test.ts
index 6d8d3d1..464d7a4 100644
--- a/test/temporal-capture.test.ts
+++ b/test/temporal-capture.test.ts
@@ -99,3 +99,41 @@ test("selection keeps temporal order and compares bytes, not semantic claims", a
for (let i = 0; i < 3; i++) store.put({ ...sequence, sequence_id: `other-${i}` });
expect(store.get(sequence.sequence_id)).toBeUndefined(); store.clear(); expect(store.get("other-2")).toBeUndefined();
});
+
+// LPT-950: a capture that takes longer than a frame interval runs alongside the next one, at most `concurrency` at a
+// time; frames stay in the order their captures started.
+test("captures slower than the frame interval overlap up to the concurrency limit and keep their order", async () => {
+ const run = async (concurrency?: number) => {
+ let inFlight = 0, most = 0, calls = 0;
+ const sequence = await collectSequence({ duration_ms: 1200, fps: 25 }, async () => {
+ const call = ++calls;
+ inFlight++; most = Math.max(most, inFlight);
+ // Later captures finish faster, so completion order differs from start order.
+ await new Promise(resolve => setTimeout(resolve, call % 2 ? 110 : 80));
+ inFlight--;
+ return frame(Buffer.from(String(call)).toString("base64"));
+ }, undefined, undefined, concurrency ? { concurrency } : {});
+ return { sequence, most };
+ };
+ const serial = await run(), paired = await run(2);
+ expect(serial.most).toBe(1);
+ expect(paired.most).toBe(2);
+ expect(paired.sequence.frames.length).toBeGreaterThanOrEqual(serial.sequence.frames.length * 1.5);
+ const slots = paired.sequence.frames.map(f => f.slot);
+ expect(slots).toEqual([...slots].sort((a, b) => a - b));
+ const starts = paired.sequence.frames.map(f => f.capture_started_ms);
+ expect(starts).toEqual([...starts].sort((a, b) => a - b));
+ expect(paired.sequence.frames.map(f => f.index)).toEqual(paired.sequence.frames.map((_, i) => i));
+});
+
+test("a failed capture among overlapping ones ends the sequence with the frames before it", async () => {
+ let calls = 0;
+ const sequence = await collectSequence({ duration_ms: 2000, fps: 25 }, async () => {
+ const call = ++calls;
+ await new Promise(resolve => setTimeout(resolve, 100));
+ if (call === 4) throw new Error("window closed");
+ return frame(Buffer.from(String(call)).toString("base64"));
+ }, undefined, undefined, { concurrency: 2 });
+ expect(sequence.stop_reason).toBe("capture_failed");
+ expect(sequence.frames).toHaveLength(3);
+});
diff --git a/test/watch-crop.test.ts b/test/watch-crop.test.ts
new file mode 100644
index 0000000..9a4f6ee
--- /dev/null
+++ b/test/watch-crop.test.ts
@@ -0,0 +1,58 @@
+import sharp from "sharp";
+import { expect, test, vi } from "vitest";
+import { ComputerSession, type Driver } from "../src/session.ts";
+
+// LPT-950: a watch of a viewport asks the driver for the viewport only, in the window's physical pixels, so the driver
+// encodes just that rectangle instead of the whole window; two captures may be in flight at once.
+const target = { pid: 42, window_id: 100 };
+const png = (width: number, height: number) => sharp({ create: { width, height, channels: 3, background: "#336699" } }).png().toBuffer();
+
+test("a viewport watch sends its rectangle as the driver crop and records the frames as cropped", async () => {
+ let windows: object[] = [];
+ const calls: Record<string, unknown>[] = [];
+ let inFlight = 0, most = 0;
+ const driver: Driver = {
+ windowCrop: true,
+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),
+ call: vi.fn<Driver["call"]>(async (name, args) => {
+ if (name === "list_windows") return { content: [], structuredContent: { windows } };
+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Big", launch_path: "C:\\Big\\big.exe" }] } };
+ if (name === "launch_app") { windows = [{ ...target, title: "Big" }]; return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] }; }
+ if (name !== "get_window_state") return { content: [], structuredContent: {} };
+ calls.push(args);
+ const full = { screenshot_original_width: 8192, screenshot_original_height: 5120 };
+ if (!args.crop) {
+ const cap = Number(args.max_image_dimension), width = Math.min(cap, 8192), height = Math.round(width * 5120 / 8192);
+ return { content: [{ type: "image", mimeType: "image/png", data: (await png(width, height)).toString("base64") }],
+ structuredContent: { ...target, ...full, screenshot_width: width, screenshot_height: height, snapshot_id: "s1", elements: [] } };
+ }
+ inFlight++; most = Math.max(most, inFlight);
+ await new Promise(resolve => setTimeout(resolve, 120));
+ inFlight--;
+ const crop = args.crop as { x: number; y: number; width: number; height: number };
+ const k = Math.min(1, Number(args.max_image_dimension) / Math.max(crop.width, crop.height));
+ const width = Math.round(crop.width * k), height = Math.round(crop.height * k);
+ return { content: [{ type: "image", mimeType: "image/png", data: (await png(width, height)).toString("base64") }],
+ structuredContent: { ...target, ...full, screenshot_width: width, screenshot_height: height, screenshot_crop: crop, display_only: true } };
+ }),
+ close: vi.fn(async () => {}),
+ };
+ const session = new ComputerSession(driver, () => "yolo");
+ const ref = (await session.open("Big")).details.window_ref!;
+ // The viewport in pixels of the window's full-size image, which is at most 4096 px: this 8192 px window counts at 1/2.
+ const started = await session.startWatch(ref, "viewport", { fps: 15, timeout_ms: 1500, maxLongEdge: 800, crop: { x: 100, y: 200, width: 400, height: 300 } });
+ expect(started.details.status).toBe("observed");
+ await new Promise(resolve => setTimeout(resolve, 900));
+ const stopped = await session.stopWatch(started.details.watch!.sequence_id);
+ const cropped = calls.filter(call => call.crop);
+ // The rectangle in the window's physical pixels; display-only, so the image never becomes an input source.
+ expect(cropped[0]).toMatchObject({ display_only: true, include_accessibility_tree: false, crop: { x: 200, y: 400, width: 800, height: 600 } });
+ // The viewport keeps its size: 400x300 at most 800 on its long edge.
+ expect(cropped[0].max_image_dimension).toBe(400);
+ const frames = stopped.details.sequence!.frames;
+ expect(frames[0].source.image).toMatchObject({ scope: "native_window_zoom", dimensions: { width: 400, height: 300 },
+ origin: { x: 200, y: 400, relative_to: "window" }, crop: { coordinate_space: "window_physical_pixels", applied: { x: 200, y: 400, width: 800, height: 600 } } });
+ // Captures of 120 ms at 15 FPS overlap, two at a time.
+ expect(most).toBe(2);
+ expect(frames.length).toBeGreaterThanOrEqual(9);
+});