Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

test(computer): cover native visuals and human-only settings

Aleksandr Shpuntenko45d0ec6eda44
diff --git a/test/fixtures/visual-commands-extension.ts b/test/fixtures/visual-commands-extension.ts
new file mode 100644
index 0000000..5fcc654
--- /dev/null
+++ b/test/fixtures/visual-commands-extension.ts
@@ -0,0 +1,24 @@
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { CuaImageSettings } from "../../src/cua-image-settings.ts";
+import { registerImageSettingsCommand } from "../../src/image-settings-command.ts";
+import { VisualSettings } from "../../src/native-visuals.ts";
+import { registerVisualSettingsCommand } from "../../src/visual-settings-command.ts";
+
+/** Real Pi owns command dispatch, session entries and tree navigation; this fixture only reads the outcome. */
+export default function(pi: ExtensionAPI): void {
+ const visuals = new VisualSettings(); const images = new CuaImageSettings();
+ registerVisualSettingsCommand(pi, visuals); registerImageSettingsCommand(pi, images);
+ pi.registerCommand("visual-probe", {
+ description: "Save the current test preferences.",
+ handler: async (_args, ctx) => {
+ await writeFile(path.join(ctx.cwd, "preferences.json"), JSON.stringify({ visuals: visuals.current(), images: images.settings(),
+ branch: ctx.sessionManager.getBranch() }));
+ },
+ });
+ pi.registerCommand("visual-navigate", {
+ description: "Navigate the test session without inference.",
+ handler: async (target, ctx) => { await ctx.navigateTree(target.trim(), { summarize: false }); },
+ });
+}
diff --git a/test/image-settings-command.test.ts b/test/image-settings-command.test.ts
new file mode 100644
index 0000000..f74c7e5
--- /dev/null
+++ b/test/image-settings-command.test.ts
@@ -0,0 +1,43 @@
+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { expect, test, vi } from "vitest";
+import { CuaImageSettings } from "../src/cua-image-settings.ts";
+import { registerImageSettingsCommand } from "../src/image-settings-command.ts";
+
+type Command = { handler(args: string, ctx: ExtensionCommandContext): Promise<void> };
+function fixture() {
+ const commands = new Map<string, Command>();
+ const images = new CuaImageSettings(() => "bilinear");
+ const notify = vi.fn();
+ registerImageSettingsCommand({ registerCommand: (name: string, command: Command) => commands.set(name, command) } as unknown as ExtensionAPI, images);
+ const ctx = { hasUI: true, ui: { notify } } as unknown as ExtensionCommandContext;
+ return { images, notify, run: (input: string) => commands.get("computer-images")!.handler(input, ctx), commands };
+}
+
+test("the human image command changes future native captures without a filter switch", async () => {
+ const f = fixture(); const old = f.images.settings();
+ await f.run("cap 800"); await f.run("scale 0.5"); await f.run("hints off"); await f.run("guides all");
+ expect(f.images.settings()).toMatchObject({ kernel: "bilinear", maxLongEdge: 800, scale: 0.5, hints: false,
+ guides: { grid: true, action: true }, version: 4 });
+ expect(old).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });
+ expect(old.hints).toBeUndefined(); expect(old.guides).toBeUndefined();
+ await f.run("reset"); expect(f.images.settings()).toEqual(old);
+});
+
+test("bad image command arguments preserve the full settings tuple", async () => {
+ const f = fixture(); await f.run("cap 800"); const saved = f.images.settings();
+ for (const input of ["cap 63", "cap 800.5", "cap Infinity", "scale 0", "scale NaN", "hints false", "hints off extra",
+ "guides on", "kernel lanczos3", "mode yolo", "reset extra"]) {
+ await f.run(input); expect(f.images.settings()).toEqual(saved);
+ expect(f.notify.mock.calls.at(-1)?.[1]).toBe("warning");
+ }
+});
+
+test("showing image settings does not change captures and instances stay independent", async () => {
+ const first = fixture(); const second = fixture();
+ await first.run("cap 600"); await first.run("guides grid");
+ const saved = first.images.settings(); await first.run(""); await first.run("show");
+ expect(first.images.settings()).toEqual(saved);
+ expect(second.images.settings()).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });
+ expect(first.commands.size).toBe(1);
+ expect(first.notify.mock.calls.at(-1)?.[0]).toContain("600");
+});
diff --git a/test/native-visuals.test.ts b/test/native-visuals.test.ts
new file mode 100644
index 0000000..82cae1a
--- /dev/null
+++ b/test/native-visuals.test.ts
@@ -0,0 +1,196 @@
+import { expect, test, vi } from "vitest";
+import { NativeVisuals, VisualSettings, notifyHighlight } from "../src/native-visuals.ts";
+import sharp from "sharp";
+import { ComputerSession, type Driver, type DriverResult } from "../src/session.ts";
+import type { InputCommand } from "../src/complete-input.ts";
+import { WebError, type WebAction, type WebBrowser, type WebLauncher } from "../src/web-browser.ts";
+
+const frame = { target: { window_id: 101, pid: 42 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`,
+ bounds: { x: -100, y: 20, width: 800, height: 600 } };
+const tick = async () => { for (let n = 0; n < 8; n++) await Promise.resolve(); };
+
+test("presentation returns immediately while native admission is pending, with only the newest pending image retained", async () => {
+ const sent: { name: string; args: Record<string, unknown> }[] = [];
+ let release!: () => void;
+ const visuals = new NativeVisuals(async (name, args) => { sent.push({ name, args }); await new Promise<void>(r => { release = r; }); return true; });
+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" });
+ await tick();
+ for (let n = 1; n <= 100; n++) await visuals.flash({ ...frame, mark: { kind: "action", point: { x: n, y: 50 } } });
+ expect(sent).toHaveLength(1);
+ release(); await tick();
+ expect(sent).toHaveLength(2);
+ expect(sent[1].args).toMatchObject({ generation: 101, point: { x: 100, y: 50 } });
+ release(); await tick(); visuals.close();
+});
+
+test("previews contain the supplied output image and validated physical source, never a new capture", async () => {
+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
+ const image = { type: "image" as const, mimeType: "image/png", data: "aGVsbG8=" };
+ await notifyHighlight(visuals.flash, { ...frame, image }); await tick();
+ expect(send).toHaveBeenCalledExactlyOnceWith("present_agent_activity", {
+ generation: 1, kind: "overview", source: { window_id: 101, pid: 42, identity: frame.identity, bounds: frame.bounds },
+ image: { data: image.data, mime_type: image.mimeType },
+ });
+ visuals.close();
+});
+
+test("invalid or out-of-source geometry is cleared without sending image content", async () => {
+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
+ const result = await notifyHighlight(visuals.flash, { ...frame, identity: frame.identity.replace(":42:", ":99:") });
+ await tick();
+ expect(result?.status).toBe("unavailable");
+ expect(send.mock.calls).toEqual([["present_agent_activity", { kind: "clear", generation: 1 }]]);
+ visuals.close();
+});
+
+test("off discards queued images and old cancellation cannot clear a newer event", async () => {
+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const settings = new VisualSettings(); const visuals = new NativeVisuals(send, settings);
+ const abort = new AbortController();
+ await visuals.flash(frame, abort.signal); await tick();
+ await visuals.flash(frame); await tick();
+ abort.abort(); await tick();
+ expect(send).toHaveBeenCalledTimes(2);
+ settings.update({ preset: "off" }); await tick();
+ expect(send.mock.calls.at(-1)).toEqual(["set_agent_visuals", expect.objectContaining({ preset: "off" })]);
+ const count = send.mock.calls.length;
+ expect(await visuals.flash(frame)).toMatchObject({ status: "skipped" });
+ await tick(); expect(send).toHaveBeenCalledTimes(count); visuals.close();
+});
+
+test("selected-display regions retain their physical monitor origin", async () => {
+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
+ await visuals.flash({ display: "DISPLAY2", bounds: { x: -1920, y: 0, width: 1920, height: 1080 }, mark: {
+ kind: "detail", region: { x: -1900, y: 40, width: 200, height: 100 },
+ } }); await tick();
+ expect(send.mock.calls[0][1]).toMatchObject({ source: { display: "DISPLAY2", bounds: { x: -1920, y: 0 } }, region: { x: -1900, y: 40 } });
+ visuals.close();
+});
+
+test("restoring a branch resets unrelated preferences and rejects prototype keys", () => {
+ const settings = new VisualSettings();
+ settings.update({ keyboard: false, quality: "high" });
+ expect(settings.restore({ preset: "subtle" })).toMatchObject({ keyboard: true, quality: "balanced", previews: false });
+ const before = settings.current();
+ expect(() => settings.update(JSON.parse('{"constructor":false}'))).toThrow();
+ expect(settings.current()).toEqual(before);
+});
+test("settings validate before mutation and unavailable native presentation does not throw into operations", async () => {
+ const settings = new VisualSettings();
+ expect(() => settings.update({ opacity: NaN })).toThrow();
+ expect(settings.current().opacity).toBe(0.8);
+ const visuals = new NativeVisuals(async () => { throw new Error("not available"); }, settings);
+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" }); await tick();
+ expect(await visuals.flash(frame)).toMatchObject({ status: "unavailable" }); visuals.close();
+});
+
+
+async function actionFixture(outcome: "delivered" | "refused" | "unknown" = "delivered") {
+ const png = { type: "image" as const, mimeType: "image/png", data: (await sharp({ create: {
+ width: frame.bounds.width, height: frame.bounds.height, channels: 3, background: "white",
+ } }).png().toBuffer()).toString("base64") };
+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true);
+ const visuals = new NativeVisuals(send);
+ const driver: Driver = {
+ visuals: visuals.flash, heldKeys: true, heldPaths: true, captureBoundDrag: true, semanticActions: true,
+ identity: vi.fn(async () => frame.identity), close: vi.fn(async () => {}),
+ call: vi.fn<Driver["call"]>(async (name, args): Promise<DriverResult> => {
+ if (name === "list_windows") return { content: [], structuredContent: { windows: [{
+ ...frame.target, bounds: frame.bounds, app_name: "Fixture.exe", title: "Action fixture",
+ }] } };
+ if (name === "get_window_state") return { content: [png], structuredContent: {
+ ...frame.target, window_bounds: frame.bounds, capture_id: "11111111-1111-4111-8111-111111111111",
+ screenshot_width: frame.bounds.width, screenshot_height: frame.bounds.height,
+ elements: [{ element_token: "field", role: "TextBox", frame: { x: 0, y: 60, w: 80, h: 40 } }],
+ } };
+ if (outcome !== "delivered") return { content: [], structuredContent: { effect: outcome } };
+ return { content: [], structuredContent: name === "semantic_action"
+ ? { operation: args.operation, pattern: "value", route: "accessibility", effect: "unverifiable", fallback: false, delivery: { mode: "background" } }
+ : { effect: "unverifiable" } };
+ }),
+ };
+ const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({
+ url: "https://fixture.invalid", title: "Action fixture", snapshot: '- textbox "Field"', tabs: [], events: [],
+ })) };
+ const launch: WebLauncher = async () => ({ browser: browser as unknown as WebBrowser,
+ opened: { pid: frame.target.pid, version: "155", url: "https://fixture.invalid", title: "Action fixture", via: "cdp" } });
+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, launch);
+ return { session, driver, send, browser, close: async () => { visuals.close(); await session.close(); } };
+}
+
+const nativeFamilies: { label: string; command: InputCommand; family: string; native: string }[] = [
+ { label: "typing", command: { action: "type_text", text: "synthetic-private-text" }, family: "type_text", native: "type_text" },
+ { label: "key press", command: { action: "key", key: "f12" }, family: "press_key", native: "press_key" },
+ { label: "select all", command: { action: "select_all" }, family: "hotkey", native: "hotkey" },
+ { label: "semantic value", command: { action: "set_value", element_token: "field", text: "synthetic-private-value" }, family: "fill", native: "semantic_action" },
+ { label: "scroll", command: { action: "scroll", element_token: "field", direction: "down" }, family: "scroll", native: "scroll" },
+ { label: "drag", command: { action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }, family: "drag", native: "drag" },
+ { label: "keyboard hold", command: { action: "hold", keys: ["ctrl"], duration_ms: 10 }, family: "hotkey", native: "hold_keys" },
+ { label: "held pointer", command: { action: "hold", keys: ["ctrl"], duration_ms: 10,
+ actions: [{ action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }] }, family: "input_timeline", native: "hold_keys" },
+ { label: "keyboard timeline", command: { action: "timeline", events: [
+ { at_ms: 0, type: "key_down", key: "ctrl" }, { at_ms: 10, type: "key_up", key: "ctrl" },
+ ] }, family: "hotkey", native: "input_timeline" },
+ { label: "pointer timeline", command: { action: "timeline", events: [
+ { at_ms: 0, type: "move", x: 10, y: 10 }, { at_ms: 10, type: "wheel", dy: 1 },
+ ] }, family: "input_timeline", native: "input_timeline" },
+];
+
+test.each(nativeFamilies)("native $label retains its presentation family before and after one dispatch", async ({ command, family, native }) => {
+ const f = await actionFixture();
+ try {
+ const listed = await f.session.windows();
+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();
+ const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, ...command });
+ await tick();
+ expect(result.details.status).toBe("delivered");
+ expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual([native]);
+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);
+ expect(activities.map(args => ({ kind: args.kind, action: args.action, outcome: args.outcome }))).toEqual([
+ { kind: "action", action: family, outcome: undefined }, { kind: "action", action: family, outcome: "delivered" },
+ ]);
+ for (const args of activities) {
+ expect(args.source).toEqual({ ...frame.target, identity: frame.identity, bounds: frame.bounds });
+ expect(Object.keys(args).every(key => ["generation", "kind", "source", "region", "action", "outcome"].includes(key))).toBe(true);
+ if (!command.element_token) expect(args).not.toHaveProperty("region");
+ }
+ } finally { await f.close(); }
+});
+
+test.each(["refused", "unknown"] as const)("native keyboard %s keeps its family without replaying input", async outcome => {
+ const f = await actionFixture(outcome);
+ try {
+ const listed = await f.session.windows();
+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
+ const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();
+ const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "f12" });
+ await tick();
+ expect(result.details.status).toBe(outcome);
+ expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual(["press_key"]);
+ expect(f.send.mock.calls.map(([, args]) => ({ action: args.action, outcome: args.outcome }))).toEqual([
+ { action: "press_key", outcome: undefined }, { action: "press_key", outcome },
+ ]);
+ } finally { await f.close(); }
+});
+
+test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presentation without forwarding key or target", async route => {
+ for (const outcome of ["delivered", "refused", "unknown"] as const) {
+ const f = await actionFixture();
+ try {
+ const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;
+ const read = await f.session.observeDom(ref); await tick(); f.send.mockClear(); f.browser.page.mockClear();
+ if (outcome !== "delivered") f.browser.page.mockRejectedValueOnce(new WebError("Synthetic page failure", outcome === "unknown"));
+ const action: WebAction = { action: "press", target: { ref: "e1" }, key: "Control+Shift+F12" };
+ const result = route === "dom"
+ ? await f.session.actDom(ref, read.details.observation_id!, action)
+ : await f.session.actPage(ref, action);
+ await tick();
+ expect(result.details.status).toBe(outcome);
+ expect(f.browser.page).toHaveBeenCalledExactlyOnceWith("act", action, undefined);
+ expect(f.send.mock.calls).toEqual([["present_agent_activity", {
+ generation: expect.any(Number), kind: "action", source: { ...frame.target, identity: frame.identity, bounds: frame.bounds },
+ action: "press_key", outcome,
+ }]]);
+ } finally { await f.close(); }
+ }
+});
diff --git a/test/visual-commands.integration.test.ts b/test/visual-commands.integration.test.ts
new file mode 100644
index 0000000..447c471
--- /dev/null
+++ b/test/visual-commands.integration.test.ts
@@ -0,0 +1,45 @@
+import { RpcClient } from "@earendil-works/pi-coding-agent";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { expect, test } from "vitest";
+
+test("human visual commands persist preferences and restore the selected Pi branch without model work", async () => {
+ const cwd = await mkdtemp(path.join(tmpdir(), "computer-visual-commands-"));
+ const client = new RpcClient({ cwd,
+ cliPath: path.join(path.dirname(fileURLToPath(import.meta.resolve("@earendil-works/pi-coding-agent"))), "cli.js"),
+ env: { PI_CODING_AGENT_DIR: path.join(cwd, "agent"), PI_OFFLINE: "1" },
+ args: ["--no-session", "--no-extensions", "--no-mcp", "--no-skills", "--no-context-files", "--approve",
+ "--extension", path.resolve("test/fixtures/visual-commands-extension.ts")],
+ });
+ const probe = async () => {
+ expect(await client.prompt("/visual-probe")).toBe("handled");
+ return JSON.parse(await readFile(path.join(cwd, "preferences.json"), "utf8")) as {
+ visuals: { preset: string; previews: boolean; keyboard: boolean; quality: string };
+ images: { maxLongEdge: number; hints: boolean };
+ };
+ };
+ try {
+ await client.start();
+ const commands = await client.getCommands();
+ expect(commands.filter(command => command.name === "computer-visuals")).toHaveLength(1);
+ expect(commands.filter(command => command.name === "computer-images")).toHaveLength(1);
+ expect(await client.prompt("/computer-visuals subtle")).toBe("handled");
+ const saved = (await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals");
+ expect(saved).toHaveLength(1);
+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });
+ expect(await client.prompt('/computer-visuals {"preset":"full","keyboard":false,"quality":"high"}')).toBe("handled");
+ expect((await probe()).visuals).toMatchObject({ preset: "full", keyboard: false, quality: "high" });
+ expect(await client.prompt(`/visual-navigate ${saved[0].id}`)).toBe("handled");
+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false, keyboard: true, quality: "balanced" });
+ expect(await client.prompt("/computer-images cap 960")).toBe("handled");
+ expect(await client.prompt("/computer-images hints off")).toBe("handled");
+ expect((await probe()).images).toMatchObject({ maxLongEdge: 960, hints: false });
+ expect(await client.prompt('/computer-visuals {"constructor":false}')).toBe("handled");
+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });
+ expect((await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals")).toHaveLength(2);
+ expect((await client.getMessages()).filter(message => message.role === "assistant")).toHaveLength(0);
+ expect(client.getStderr()).toBe("");
+ } finally { await client.stop(); await rm(cwd, { recursive: true, force: true }); }
+});