Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
test(computer): cover native visuals and human-only settings
Aleksandr Shpuntenko45d0ec6eda44
diff --git a/test/fixtures/visual-commands-extension.ts b/test/fixtures/visual-commands-extension.tsnew file mode 100644index 0000000..5fcc654--- /dev/null+++ b/test/fixtures/visual-commands-extension.ts@@ -0,0 +1,24 @@+import { writeFile } from "node:fs/promises";+import path from "node:path";+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";+import { CuaImageSettings } from "../../src/cua-image-settings.ts";+import { registerImageSettingsCommand } from "../../src/image-settings-command.ts";+import { VisualSettings } from "../../src/native-visuals.ts";+import { registerVisualSettingsCommand } from "../../src/visual-settings-command.ts";++/** Real Pi owns command dispatch, session entries and tree navigation; this fixture only reads the outcome. */+export default function(pi: ExtensionAPI): void {+ const visuals = new VisualSettings(); const images = new CuaImageSettings();+ registerVisualSettingsCommand(pi, visuals); registerImageSettingsCommand(pi, images);+ pi.registerCommand("visual-probe", {+ description: "Save the current test preferences.",+ handler: async (_args, ctx) => {+ await writeFile(path.join(ctx.cwd, "preferences.json"), JSON.stringify({ visuals: visuals.current(), images: images.settings(),+ branch: ctx.sessionManager.getBranch() }));+ },+ });+ pi.registerCommand("visual-navigate", {+ description: "Navigate the test session without inference.",+ handler: async (target, ctx) => { await ctx.navigateTree(target.trim(), { summarize: false }); },+ });+}diff --git a/test/image-settings-command.test.ts b/test/image-settings-command.test.tsnew file mode 100644index 0000000..f74c7e5--- /dev/null+++ b/test/image-settings-command.test.ts@@ -0,0 +1,43 @@+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";+import { expect, test, vi } from "vitest";+import { CuaImageSettings } from "../src/cua-image-settings.ts";+import { registerImageSettingsCommand } from "../src/image-settings-command.ts";++type Command = { handler(args: string, ctx: ExtensionCommandContext): Promise<void> };+function fixture() {+ const commands = new Map<string, Command>();+ const images = new CuaImageSettings(() => "bilinear");+ const notify = vi.fn();+ registerImageSettingsCommand({ registerCommand: (name: string, command: Command) => commands.set(name, command) } as unknown as ExtensionAPI, images);+ const ctx = { hasUI: true, ui: { notify } } as unknown as ExtensionCommandContext;+ return { images, notify, run: (input: string) => commands.get("computer-images")!.handler(input, ctx), commands };+}++test("the human image command changes future native captures without a filter switch", async () => {+ const f = fixture(); const old = f.images.settings();+ await f.run("cap 800"); await f.run("scale 0.5"); await f.run("hints off"); await f.run("guides all");+ expect(f.images.settings()).toMatchObject({ kernel: "bilinear", maxLongEdge: 800, scale: 0.5, hints: false,+ guides: { grid: true, action: true }, version: 4 });+ expect(old).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });+ expect(old.hints).toBeUndefined(); expect(old.guides).toBeUndefined();+ await f.run("reset"); expect(f.images.settings()).toEqual(old);+});++test("bad image command arguments preserve the full settings tuple", async () => {+ const f = fixture(); await f.run("cap 800"); const saved = f.images.settings();+ for (const input of ["cap 63", "cap 800.5", "cap Infinity", "scale 0", "scale NaN", "hints false", "hints off extra",+ "guides on", "kernel lanczos3", "mode yolo", "reset extra"]) {+ await f.run(input); expect(f.images.settings()).toEqual(saved);+ expect(f.notify.mock.calls.at(-1)?.[1]).toBe("warning");+ }+});++test("showing image settings does not change captures and instances stay independent", async () => {+ const first = fixture(); const second = fixture();+ await first.run("cap 600"); await first.run("guides grid");+ const saved = first.images.settings(); await first.run(""); await first.run("show");+ expect(first.images.settings()).toEqual(saved);+ expect(second.images.settings()).toMatchObject({ maxLongEdge: 1280, scale: 1, version: 0 });+ expect(first.commands.size).toBe(1);+ expect(first.notify.mock.calls.at(-1)?.[0]).toContain("600");+});diff --git a/test/native-visuals.test.ts b/test/native-visuals.test.tsnew file mode 100644index 0000000..82cae1a--- /dev/null+++ b/test/native-visuals.test.ts@@ -0,0 +1,196 @@+import { expect, test, vi } from "vitest";+import { NativeVisuals, VisualSettings, notifyHighlight } from "../src/native-visuals.ts";+import sharp from "sharp";+import { ComputerSession, type Driver, type DriverResult } from "../src/session.ts";+import type { InputCommand } from "../src/complete-input.ts";+import { WebError, type WebAction, type WebBrowser, type WebLauncher } from "../src/web-browser.ts";++const frame = { target: { window_id: 101, pid: 42 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`,+ bounds: { x: -100, y: 20, width: 800, height: 600 } };+const tick = async () => { for (let n = 0; n < 8; n++) await Promise.resolve(); };++test("presentation returns immediately while native admission is pending, with only the newest pending image retained", async () => {+ const sent: { name: string; args: Record<string, unknown> }[] = [];+ let release!: () => void;+ const visuals = new NativeVisuals(async (name, args) => { sent.push({ name, args }); await new Promise<void>(r => { release = r; }); return true; });+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" });+ await tick();+ for (let n = 1; n <= 100; n++) await visuals.flash({ ...frame, mark: { kind: "action", point: { x: n, y: 50 } } });+ expect(sent).toHaveLength(1);+ release(); await tick();+ expect(sent).toHaveLength(2);+ expect(sent[1].args).toMatchObject({ generation: 101, point: { x: 100, y: 50 } });+ release(); await tick(); visuals.close();+});++test("previews contain the supplied output image and validated physical source, never a new capture", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ const image = { type: "image" as const, mimeType: "image/png", data: "aGVsbG8=" };+ await notifyHighlight(visuals.flash, { ...frame, image }); await tick();+ expect(send).toHaveBeenCalledExactlyOnceWith("present_agent_activity", {+ generation: 1, kind: "overview", source: { window_id: 101, pid: 42, identity: frame.identity, bounds: frame.bounds },+ image: { data: image.data, mime_type: image.mimeType },+ });+ visuals.close();+});++test("invalid or out-of-source geometry is cleared without sending image content", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ const result = await notifyHighlight(visuals.flash, { ...frame, identity: frame.identity.replace(":42:", ":99:") });+ await tick();+ expect(result?.status).toBe("unavailable");+ expect(send.mock.calls).toEqual([["present_agent_activity", { kind: "clear", generation: 1 }]]);+ visuals.close();+});++test("off discards queued images and old cancellation cannot clear a newer event", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const settings = new VisualSettings(); const visuals = new NativeVisuals(send, settings);+ const abort = new AbortController();+ await visuals.flash(frame, abort.signal); await tick();+ await visuals.flash(frame); await tick();+ abort.abort(); await tick();+ expect(send).toHaveBeenCalledTimes(2);+ settings.update({ preset: "off" }); await tick();+ expect(send.mock.calls.at(-1)).toEqual(["set_agent_visuals", expect.objectContaining({ preset: "off" })]);+ const count = send.mock.calls.length;+ expect(await visuals.flash(frame)).toMatchObject({ status: "skipped" });+ await tick(); expect(send).toHaveBeenCalledTimes(count); visuals.close();+});++test("selected-display regions retain their physical monitor origin", async () => {+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);+ await visuals.flash({ display: "DISPLAY2", bounds: { x: -1920, y: 0, width: 1920, height: 1080 }, mark: {+ kind: "detail", region: { x: -1900, y: 40, width: 200, height: 100 },+ } }); await tick();+ expect(send.mock.calls[0][1]).toMatchObject({ source: { display: "DISPLAY2", bounds: { x: -1920, y: 0 } }, region: { x: -1900, y: 40 } });+ visuals.close();+});++test("restoring a branch resets unrelated preferences and rejects prototype keys", () => {+ const settings = new VisualSettings();+ settings.update({ keyboard: false, quality: "high" });+ expect(settings.restore({ preset: "subtle" })).toMatchObject({ keyboard: true, quality: "balanced", previews: false });+ const before = settings.current();+ expect(() => settings.update(JSON.parse('{"constructor":false}'))).toThrow();+ expect(settings.current()).toEqual(before);+});+test("settings validate before mutation and unavailable native presentation does not throw into operations", async () => {+ const settings = new VisualSettings();+ expect(() => settings.update({ opacity: NaN })).toThrow();+ expect(settings.current().opacity).toBe(0.8);+ const visuals = new NativeVisuals(async () => { throw new Error("not available"); }, settings);+ expect(await visuals.flash(frame)).toMatchObject({ status: "queued" }); await tick();+ expect(await visuals.flash(frame)).toMatchObject({ status: "unavailable" }); visuals.close();+});+++async function actionFixture(outcome: "delivered" | "refused" | "unknown" = "delivered") {+ const png = { type: "image" as const, mimeType: "image/png", data: (await sharp({ create: {+ width: frame.bounds.width, height: frame.bounds.height, channels: 3, background: "white",+ } }).png().toBuffer()).toString("base64") };+ const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true);+ const visuals = new NativeVisuals(send);+ const driver: Driver = {+ visuals: visuals.flash, heldKeys: true, heldPaths: true, captureBoundDrag: true, semanticActions: true,+ identity: vi.fn(async () => frame.identity), close: vi.fn(async () => {}),+ call: vi.fn<Driver["call"]>(async (name, args): Promise<DriverResult> => {+ if (name === "list_windows") return { content: [], structuredContent: { windows: [{+ ...frame.target, bounds: frame.bounds, app_name: "Fixture.exe", title: "Action fixture",+ }] } };+ if (name === "get_window_state") return { content: [png], structuredContent: {+ ...frame.target, window_bounds: frame.bounds, capture_id: "11111111-1111-4111-8111-111111111111",+ screenshot_width: frame.bounds.width, screenshot_height: frame.bounds.height,+ elements: [{ element_token: "field", role: "TextBox", frame: { x: 0, y: 60, w: 80, h: 40 } }],+ } };+ if (outcome !== "delivered") return { content: [], structuredContent: { effect: outcome } };+ return { content: [], structuredContent: name === "semantic_action"+ ? { operation: args.operation, pattern: "value", route: "accessibility", effect: "unverifiable", fallback: false, delivery: { mode: "background" } }+ : { effect: "unverifiable" } };+ }),+ };+ const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({+ url: "https://fixture.invalid", title: "Action fixture", snapshot: '- textbox "Field"', tabs: [], events: [],+ })) };+ const launch: WebLauncher = async () => ({ browser: browser as unknown as WebBrowser,+ opened: { pid: frame.target.pid, version: "155", url: "https://fixture.invalid", title: "Action fixture", via: "cdp" } });+ const session = new ComputerSession(driver, () => "yolo", undefined, undefined, launch);+ return { session, driver, send, browser, close: async () => { visuals.close(); await session.close(); } };+}++const nativeFamilies: { label: string; command: InputCommand; family: string; native: string }[] = [+ { label: "typing", command: { action: "type_text", text: "synthetic-private-text" }, family: "type_text", native: "type_text" },+ { label: "key press", command: { action: "key", key: "f12" }, family: "press_key", native: "press_key" },+ { label: "select all", command: { action: "select_all" }, family: "hotkey", native: "hotkey" },+ { label: "semantic value", command: { action: "set_value", element_token: "field", text: "synthetic-private-value" }, family: "fill", native: "semantic_action" },+ { label: "scroll", command: { action: "scroll", element_token: "field", direction: "down" }, family: "scroll", native: "scroll" },+ { label: "drag", command: { action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }, family: "drag", native: "drag" },+ { label: "keyboard hold", command: { action: "hold", keys: ["ctrl"], duration_ms: 10 }, family: "hotkey", native: "hold_keys" },+ { label: "held pointer", command: { action: "hold", keys: ["ctrl"], duration_ms: 10,+ actions: [{ action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }] }, family: "input_timeline", native: "hold_keys" },+ { label: "keyboard timeline", command: { action: "timeline", events: [+ { at_ms: 0, type: "key_down", key: "ctrl" }, { at_ms: 10, type: "key_up", key: "ctrl" },+ ] }, family: "hotkey", native: "input_timeline" },+ { label: "pointer timeline", command: { action: "timeline", events: [+ { at_ms: 0, type: "move", x: 10, y: 10 }, { at_ms: 10, type: "wheel", dy: 1 },+ ] }, family: "input_timeline", native: "input_timeline" },+];++test.each(nativeFamilies)("native $label retains its presentation family before and after one dispatch", async ({ command, family, native }) => {+ const f = await actionFixture();+ try {+ const listed = await f.session.windows();+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;+ const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();+ const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, ...command });+ await tick();+ expect(result.details.status).toBe("delivered");+ expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual([native]);+ const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);+ expect(activities.map(args => ({ kind: args.kind, action: args.action, outcome: args.outcome }))).toEqual([+ { kind: "action", action: family, outcome: undefined }, { kind: "action", action: family, outcome: "delivered" },+ ]);+ for (const args of activities) {+ expect(args.source).toEqual({ ...frame.target, identity: frame.identity, bounds: frame.bounds });+ expect(Object.keys(args).every(key => ["generation", "kind", "source", "region", "action", "outcome"].includes(key))).toBe(true);+ if (!command.element_token) expect(args).not.toHaveProperty("region");+ }+ } finally { await f.close(); }+});++test.each(["refused", "unknown"] as const)("native keyboard %s keeps its family without replaying input", async outcome => {+ const f = await actionFixture(outcome);+ try {+ const listed = await f.session.windows();+ const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;+ const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();+ const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "f12" });+ await tick();+ expect(result.details.status).toBe(outcome);+ expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual(["press_key"]);+ expect(f.send.mock.calls.map(([, args]) => ({ action: args.action, outcome: args.outcome }))).toEqual([+ { action: "press_key", outcome: undefined }, { action: "press_key", outcome },+ ]);+ } finally { await f.close(); }+});++test.each(["dom", "page"] as const)("browser %s key outcomes use keyboard presentation without forwarding key or target", async route => {+ for (const outcome of ["delivered", "refused", "unknown"] as const) {+ const f = await actionFixture();+ try {+ const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;+ const read = await f.session.observeDom(ref); await tick(); f.send.mockClear(); f.browser.page.mockClear();+ if (outcome !== "delivered") f.browser.page.mockRejectedValueOnce(new WebError("Synthetic page failure", outcome === "unknown"));+ const action: WebAction = { action: "press", target: { ref: "e1" }, key: "Control+Shift+F12" };+ const result = route === "dom"+ ? await f.session.actDom(ref, read.details.observation_id!, action)+ : await f.session.actPage(ref, action);+ await tick();+ expect(result.details.status).toBe(outcome);+ expect(f.browser.page).toHaveBeenCalledExactlyOnceWith("act", action, undefined);+ expect(f.send.mock.calls).toEqual([["present_agent_activity", {+ generation: expect.any(Number), kind: "action", source: { ...frame.target, identity: frame.identity, bounds: frame.bounds },+ action: "press_key", outcome,+ }]]);+ } finally { await f.close(); }+ }+});diff --git a/test/visual-commands.integration.test.ts b/test/visual-commands.integration.test.tsnew file mode 100644index 0000000..447c471--- /dev/null+++ b/test/visual-commands.integration.test.ts@@ -0,0 +1,45 @@+import { RpcClient } from "@earendil-works/pi-coding-agent";+import { mkdtemp, readFile, rm } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { fileURLToPath } from "node:url";+import { expect, test } from "vitest";++test("human visual commands persist preferences and restore the selected Pi branch without model work", async () => {+ const cwd = await mkdtemp(path.join(tmpdir(), "computer-visual-commands-"));+ const client = new RpcClient({ cwd,+ cliPath: path.join(path.dirname(fileURLToPath(import.meta.resolve("@earendil-works/pi-coding-agent"))), "cli.js"),+ env: { PI_CODING_AGENT_DIR: path.join(cwd, "agent"), PI_OFFLINE: "1" },+ args: ["--no-session", "--no-extensions", "--no-mcp", "--no-skills", "--no-context-files", "--approve",+ "--extension", path.resolve("test/fixtures/visual-commands-extension.ts")],+ });+ const probe = async () => {+ expect(await client.prompt("/visual-probe")).toBe("handled");+ return JSON.parse(await readFile(path.join(cwd, "preferences.json"), "utf8")) as {+ visuals: { preset: string; previews: boolean; keyboard: boolean; quality: string };+ images: { maxLongEdge: number; hints: boolean };+ };+ };+ try {+ await client.start();+ const commands = await client.getCommands();+ expect(commands.filter(command => command.name === "computer-visuals")).toHaveLength(1);+ expect(commands.filter(command => command.name === "computer-images")).toHaveLength(1);+ expect(await client.prompt("/computer-visuals subtle")).toBe("handled");+ const saved = (await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals");+ expect(saved).toHaveLength(1);+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });+ expect(await client.prompt('/computer-visuals {"preset":"full","keyboard":false,"quality":"high"}')).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "full", keyboard: false, quality: "high" });+ expect(await client.prompt(`/visual-navigate ${saved[0].id}`)).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false, keyboard: true, quality: "balanced" });+ expect(await client.prompt("/computer-images cap 960")).toBe("handled");+ expect(await client.prompt("/computer-images hints off")).toBe("handled");+ expect((await probe()).images).toMatchObject({ maxLongEdge: 960, hints: false });+ expect(await client.prompt('/computer-visuals {"constructor":false}')).toBe("handled");+ expect((await probe()).visuals).toMatchObject({ preset: "subtle", previews: false });+ expect((await client.getEntries()).entries.filter(entry => entry.type === "custom" && entry.customType === "computer-use-visuals")).toHaveLength(2);+ expect((await client.getMessages()).filter(message => message.role === "assistant")).toHaveLength(0);+ expect(client.getStderr()).toBe("");+ } finally { await client.stop(); await rm(cwd, { recursive: true, force: true }); }+});