Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Tests: a viewport watch of a large window keeps its frame rate (LPT-950)
alexshpunt-clanker[bot]d892a6307eba
diff --git a/test/live/watch.live.ts b/test/live/watch.live.tsindex c25476d..54ec125 100644--- a/test/live/watch.live.ts+++ b/test/live/watch.live.ts@@ -15,7 +15,7 @@ afterAll(async () => { await mkdir(out, { recursive: true }); await writeFile(`$const image = (result: ComputerResult) => Buffer.from((result.content.find(block => block.type === "image") as { data: string }).data, "base64");/** A form with a 600x300 view of a wide strip of numbered colour bands that follows the pointer while the left button is held. */-function panoramaForm(title: string): string[] {+function panoramaForm(title: string, maximized = false): string[] {const script = `Add-Type -AssemblyName System.Windows.Forms, System.Drawing$f = New-Object Windows.Forms.Form; $f.Text = '${title}'; $f.Width = 660; $f.Height = 380$v = New-Object Windows.Forms.Panel; $v.Left = 10; $v.Top = 10; $v.Width = 600; $v.Height = 300; $v.BackColor = 'Black'@@ -29,7 +29,7 @@ for ($i = 0; $i -lt 16; $i++) {$p = New-Object Windows.Forms.PictureBox; $p.Image = $bmp; $p.Width = 2400; $p.Height = 300; $p.Left = 0; $p.AccessibleName = 'Pano'$p.Add_MouseDown({ $script:mx = [Windows.Forms.Control]::MousePosition.X; $script:left = $p.Left })$p.Add_MouseMove({ if ([Windows.Forms.Control]::MouseButtons -eq 'Left') { $p.Left = [Math]::Min(0, [Math]::Max(-1800, $script:left + [Windows.Forms.Control]::MousePosition.X - $script:mx)) } })-$v.Controls.Add($p); $f.Controls.Add($v); [void]$f.ShowDialog()`;+$v.Controls.Add($p); $f.Controls.Add($v); ${maximized ? "$f.WindowState = 'Maximized'; " : ""}[void]$f.ShowDialog()`;return ["-NoProfile", "-WindowStyle", "Hidden", "-EncodedCommand", Buffer.from(script, "utf16le").toString("base64")];}@@ -94,3 +94,34 @@ test("a watch of a viewport sees the panorama move during a drag and stops withawait live.resources.delete(window, live.cwd).catch(() => undefined);}}, 180_000);++// LPT-950: a viewport of a maximized window: the driver encodes only the viewport, and a second capture overlaps a slow+// one, so the watch keeps at least 12 of 15 FPS during a drag. Before, every frame was the whole window: 5 FPS at 4K.+test("a watch of a viewport in a maximized window keeps its frame rate during a drag", async () => {+ const title = `pi-watch-max-${Date.now()}`;+ const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, panoramaForm(title, true), `regex:^${title}$`);+ try {+ await new Promise(resolve => setTimeout(resolve, 1500));+ await live.resources.manageWindow(requireComputerAddress(window), live.cwd, windowCommand({ window, action: "focus" }));+ const overview = await live.read(window, ["full"]);+ const box = overview.details.ui!.elements.find(e => e.name === "Pano")!.bounds.window_image!;+ const state = overview.details.state as { screenshot_width: number; screenshot_original_width: number };+ const k = state.screenshot_width / state.screenshot_original_width;+ const rect = { x: box.x, y: box.y, width: Math.round(600 * k), height: box.height };+ const view = requireComputerAddress(`${window}/pano`);+ expect((await live.resources.view(view, live.cwd, rect)).details.status).toBe("observed");+ const watch = await live.resources.watch(view, live.cwd, { fps: 15 });+ expect(watch.details.status, presentComputerResult(watch)).toBe("observed");+ const from = { x: rect.x + Math.round(rect.width * 0.8), y: rect.y + Math.round(rect.height / 2) };+ const dragged = await live.resources.input(requireComputerAddress(window), live.cwd, dragPlan({ window, from, to: { x: from.x - Math.round(300 * k), y: from.y }, duration_ms: 2000 }));+ expect(dragged.details.status, presentComputerResult(dragged)).toBe("delivered");+ const stopped = await live.resources.stopWatch(requireComputerAddress(`computer:sequence/${watch.details.watch!.sequence_id}`), live.cwd, {});+ const sequence = stopped.details.sequence!;+ report.push(`maximized window\n${(stopped.content[0] as { text: string }).text.split("\n")[0]}\ncapture latency ${JSON.stringify(sequence.frames.map(f => Math.round(f.capture_latency_ms)))}`);+ expect(sequence.actual_fps).toBeGreaterThanOrEqual(12);+ expect(sequence.frames[0].source.image.crop?.coordinate_space).toBe("window_physical_pixels");+ } finally {+ const { pid } = (await live.read(window, ["state"])).details.target as { pid: number };+ await live.kill(pid);+ }+}, 180_000);diff --git a/test/temporal-capture.test.ts b/test/temporal-capture.test.tsindex 6d8d3d1..464d7a4 100644--- a/test/temporal-capture.test.ts+++ b/test/temporal-capture.test.ts@@ -99,3 +99,41 @@ test("selection keeps temporal order and compares bytes, not semantic claims", afor (let i = 0; i < 3; i++) store.put({ ...sequence, sequence_id: `other-${i}` });expect(store.get(sequence.sequence_id)).toBeUndefined(); store.clear(); expect(store.get("other-2")).toBeUndefined();});++// LPT-950: a capture that takes longer than a frame interval runs alongside the next one, at most `concurrency` at a+// time; frames stay in the order their captures started.+test("captures slower than the frame interval overlap up to the concurrency limit and keep their order", async () => {+ const run = async (concurrency?: number) => {+ let inFlight = 0, most = 0, calls = 0;+ const sequence = await collectSequence({ duration_ms: 1200, fps: 25 }, async () => {+ const call = ++calls;+ inFlight++; most = Math.max(most, inFlight);+ // Later captures finish faster, so completion order differs from start order.+ await new Promise(resolve => setTimeout(resolve, call % 2 ? 110 : 80));+ inFlight--;+ return frame(Buffer.from(String(call)).toString("base64"));+ }, undefined, undefined, concurrency ? { concurrency } : {});+ return { sequence, most };+ };+ const serial = await run(), paired = await run(2);+ expect(serial.most).toBe(1);+ expect(paired.most).toBe(2);+ expect(paired.sequence.frames.length).toBeGreaterThanOrEqual(serial.sequence.frames.length * 1.5);+ const slots = paired.sequence.frames.map(f => f.slot);+ expect(slots).toEqual([...slots].sort((a, b) => a - b));+ const starts = paired.sequence.frames.map(f => f.capture_started_ms);+ expect(starts).toEqual([...starts].sort((a, b) => a - b));+ expect(paired.sequence.frames.map(f => f.index)).toEqual(paired.sequence.frames.map((_, i) => i));+});++test("a failed capture among overlapping ones ends the sequence with the frames before it", async () => {+ let calls = 0;+ const sequence = await collectSequence({ duration_ms: 2000, fps: 25 }, async () => {+ const call = ++calls;+ await new Promise(resolve => setTimeout(resolve, 100));+ if (call === 4) throw new Error("window closed");+ return frame(Buffer.from(String(call)).toString("base64"));+ }, undefined, undefined, { concurrency: 2 });+ expect(sequence.stop_reason).toBe("capture_failed");+ expect(sequence.frames).toHaveLength(3);+});diff --git a/test/watch-crop.test.ts b/test/watch-crop.test.tsnew file mode 100644index 0000000..9a4f6ee--- /dev/null+++ b/test/watch-crop.test.ts@@ -0,0 +1,58 @@+import sharp from "sharp";+import { expect, test, vi } from "vitest";+import { ComputerSession, type Driver } from "../src/session.ts";++// LPT-950: a watch of a viewport asks the driver for the viewport only, in the window's physical pixels, so the driver+// encodes just that rectangle instead of the whole window; two captures may be in flight at once.+const target = { pid: 42, window_id: 100 };+const png = (width: number, height: number) => sharp({ create: { width, height, channels: 3, background: "#336699" } }).png().toBuffer();++test("a viewport watch sends its rectangle as the driver crop and records the frames as cropped", async () => {+ let windows: object[] = [];+ const calls: Record<string, unknown>[] = [];+ let inFlight = 0, most = 0;+ const driver: Driver = {+ windowCrop: true,+ identity: vi.fn(async window => `${window.pid}:${window.window_id}`),+ call: vi.fn<Driver["call"]>(async (name, args) => {+ if (name === "list_windows") return { content: [], structuredContent: { windows } };+ if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: "Big", launch_path: "C:\\Big\\big.exe" }] } };+ if (name === "launch_app") { windows = [{ ...target, title: "Big" }]; return { content: [{ type: "text", text: JSON.stringify({ pid: 42, windows }) }] }; }+ if (name !== "get_window_state") return { content: [], structuredContent: {} };+ calls.push(args);+ const full = { screenshot_original_width: 8192, screenshot_original_height: 5120 };+ if (!args.crop) {+ const cap = Number(args.max_image_dimension), width = Math.min(cap, 8192), height = Math.round(width * 5120 / 8192);+ return { content: [{ type: "image", mimeType: "image/png", data: (await png(width, height)).toString("base64") }],+ structuredContent: { ...target, ...full, screenshot_width: width, screenshot_height: height, snapshot_id: "s1", elements: [] } };+ }+ inFlight++; most = Math.max(most, inFlight);+ await new Promise(resolve => setTimeout(resolve, 120));+ inFlight--;+ const crop = args.crop as { x: number; y: number; width: number; height: number };+ const k = Math.min(1, Number(args.max_image_dimension) / Math.max(crop.width, crop.height));+ const width = Math.round(crop.width * k), height = Math.round(crop.height * k);+ return { content: [{ type: "image", mimeType: "image/png", data: (await png(width, height)).toString("base64") }],+ structuredContent: { ...target, ...full, screenshot_width: width, screenshot_height: height, screenshot_crop: crop, display_only: true } };+ }),+ close: vi.fn(async () => {}),+ };+ const session = new ComputerSession(driver, () => "yolo");+ const ref = (await session.open("Big")).details.window_ref!;+ // The viewport in pixels of the window's full-size image, which is at most 4096 px: this 8192 px window counts at 1/2.+ const started = await session.startWatch(ref, "viewport", { fps: 15, timeout_ms: 1500, maxLongEdge: 800, crop: { x: 100, y: 200, width: 400, height: 300 } });+ expect(started.details.status).toBe("observed");+ await new Promise(resolve => setTimeout(resolve, 900));+ const stopped = await session.stopWatch(started.details.watch!.sequence_id);+ const cropped = calls.filter(call => call.crop);+ // The rectangle in the window's physical pixels; display-only, so the image never becomes an input source.+ expect(cropped[0]).toMatchObject({ display_only: true, include_accessibility_tree: false, crop: { x: 200, y: 400, width: 800, height: 600 } });+ // The viewport keeps its size: 400x300 at most 800 on its long edge.+ expect(cropped[0].max_image_dimension).toBe(400);+ const frames = stopped.details.sequence!.frames;+ expect(frames[0].source.image).toMatchObject({ scope: "native_window_zoom", dimensions: { width: 400, height: 300 },+ origin: { x: 200, y: 400, relative_to: "window" }, crop: { coordinate_space: "window_physical_pixels", applied: { x: 200, y: 400, width: 800, height: 600 } } });+ // Captures of 120 ms at 15 FPS overlap, two at a time.+ expect(most).toBe(2);+ expect(frames.length).toBeGreaterThanOrEqual(9);+});