Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { expect, test, vi } from "vitest";
import { NativeVisuals, VisualSettings, nativeVisualOptions, notifyHighlight } from "../src/native-visuals.ts";
import sharp from "sharp";
import { ComputerSession, type Driver, type DriverResult } from "../src/session.ts";
import type { InputCommand } from "../src/complete-input.ts";
import { WebError, type WebAction, type WebBrowser, type WebLauncher } from "../src/web-browser.ts";
const frame = { target: { window_id: 101, pid: 42 }, identity: `PiComputerUse.${"a".repeat(32)}:42:1`,
bounds: { x: -100, y: 20, width: 800, height: 600 } };
const tick = async () => { for (let n = 0; n < 8; n++) await Promise.resolve(); };
test("presentation returns immediately while native admission is pending, with only the newest pending image retained", async () => {
const sent: { name: string; args: Record<string, unknown> }[] = [];
let release!: () => void;
const visuals = new NativeVisuals(async (name, args) => { sent.push({ name, args }); await new Promise<void>(r => { release = r; }); return true; });
expect(await visuals.flash(frame)).toMatchObject({ status: "queued" });
await tick();
for (let n = 1; n <= 100; n++) await visuals.flash({ ...frame, mark: { kind: "action", point: { x: n, y: 50 } } });
expect(sent).toHaveLength(1);
release(); await tick();
expect(sent).toHaveLength(2);
expect(sent[1].args).toMatchObject({ generation: 101, point: { x: 100, y: 50 } });
release(); await tick(); visuals.close();
});
test("previews contain the supplied output image and validated physical source, never a new capture", async () => {
const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
const image = { type: "image" as const, mimeType: "image/png", data: "aGVsbG8=" };
await notifyHighlight(visuals.flash, { ...frame, image }); await tick();
expect(send).toHaveBeenCalledExactlyOnceWith("present_agent_activity", {
generation: 1, kind: "overview", source: { window_id: 101, pid: 42, identity: frame.identity, bounds: frame.bounds },
image: { data: image.data, mime_type: image.mimeType },
});
visuals.close();
});
test("invalid or out-of-source geometry is cleared without sending image content", async () => {
const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
const result = await notifyHighlight(visuals.flash, { ...frame, identity: frame.identity.replace(":42:", ":99:") });
await tick();
expect(result?.status).toBe("unavailable");
expect(send.mock.calls).toEqual([["present_agent_activity", { kind: "clear", generation: 1 }]]);
visuals.close();
});
test("off discards queued images and old cancellation cannot clear a newer event", async () => {
const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const settings = new VisualSettings(); const visuals = new NativeVisuals(send, settings);
const abort = new AbortController();
await visuals.flash(frame, abort.signal); await tick();
await visuals.flash(frame); await tick();
abort.abort(); await tick();
expect(send).toHaveBeenCalledTimes(2);
settings.update({ preset: "off" }); await tick();
expect(send.mock.calls.at(-1)).toEqual(["set_agent_visuals", expect.objectContaining({ preset: "off" })]);
const count = send.mock.calls.length;
expect(await visuals.flash(frame)).toMatchObject({ status: "skipped" });
await tick(); expect(send).toHaveBeenCalledTimes(count); visuals.close();
});
test("selected-display regions retain their physical monitor origin", async () => {
const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true); const visuals = new NativeVisuals(send);
await visuals.flash({ display: "DISPLAY2", bounds: { x: -1920, y: 0, width: 1920, height: 1080 }, mark: {
kind: "detail", region: { x: -1900, y: 40, width: 200, height: 100 },
} }); await tick();
expect(send.mock.calls[0][1]).toMatchObject({ source: { display: "DISPLAY2", bounds: { x: -1920, y: 0 } }, region: { x: -1900, y: 40 } });
visuals.close();
});
test("restoring a branch resets unrelated preferences and rejects prototype keys", () => {
const settings = new VisualSettings();
settings.update({ keyboard: false, quality: "high" });
expect(settings.restore({ preset: "subtle" })).toMatchObject({ keyboard: true, quality: "balanced", previews: false });
const before = settings.current();
expect(() => settings.update(JSON.parse('{"constructor":false}'))).toThrow();
expect(settings.current()).toEqual(before);
});
test("settings validate before mutation and unavailable native presentation does not throw into operations", async () => {
const settings = new VisualSettings();
expect(() => settings.update({ opacity: NaN })).toThrow();
expect(settings.current().opacity).toBe(0.8);
const visuals = new NativeVisuals(async () => { throw new Error("not available"); }, settings);
expect(await visuals.flash(frame)).toMatchObject({ status: "queued" }); await tick();
expect(await visuals.flash(frame)).toMatchObject({ status: "unavailable" }); visuals.close();
});
async function actionFixture(outcome: "delivered" | "refused" | "unknown" = "delivered", options: { password?: boolean; settings?: VisualSettings } = {}) {
const png = { type: "image" as const, mimeType: "image/png", data: (await sharp({ create: {
width: frame.bounds.width, height: frame.bounds.height, channels: 3, background: "white",
} }).png().toBuffer()).toString("base64") };
const send = vi.fn(async (_name: string, _args: Record<string, unknown>) => true);
const visuals = new NativeVisuals(send, options.settings);
const driver: Driver = {
visuals: visuals.flash, heldKeys: true, heldPaths: true, captureBoundDrag: true, semanticActions: true,
identity: vi.fn(async () => frame.identity), close: vi.fn(async () => {}),
call: vi.fn<Driver["call"]>(async (name, args): Promise<DriverResult> => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{
...frame.target, bounds: frame.bounds, app_name: "Fixture.exe", title: "Action fixture - Google Chrome",
}] } };
if (name === "get_window_state") return { content: [png], structuredContent: {
...frame.target, window_bounds: frame.bounds, capture_id: "11111111-1111-4111-8111-111111111111",
screenshot_width: frame.bounds.width, screenshot_height: frame.bounds.height,
elements: [{ element_token: "field", role: "TextBox", focused: true, frame: { x: 0, y: 60, w: 80, h: 40 }, ...(options.password ? { password: true } : {}) }],
} };
if (outcome !== "delivered") return { content: [], structuredContent: { effect: outcome } };
return { content: [], structuredContent: name === "semantic_action"
? { operation: args.operation, pattern: "value", route: "accessibility", effect: "unverifiable", fallback: false, delivery: { mode: "background" } }
: { effect: "unverifiable" } };
}),
};
const browser = { close: vi.fn(async () => {}), page: vi.fn(async () => ({
url: "https://fixture.invalid", title: "Action fixture", snapshot: '- textbox "Field"', tabs: [], events: [],
})) };
const launch: WebLauncher = async () => ({ browser: browser as unknown as WebBrowser,
opened: { pid: frame.target.pid, version: "155", url: "https://fixture.invalid", title: "Action fixture", via: "cdp" } });
const session = new ComputerSession(driver, () => "yolo", undefined, undefined, launch);
return { session, driver, send, browser, close: async () => { visuals.close(); await session.close(); } };
}
const nativeFamilies: { label: string; command: InputCommand; family: string; native: string }[] = [
{ label: "typing", command: { action: "type_text", text: "synthetic-private-text" }, family: "type_text", native: "type_text" },
{ label: "key press", command: { action: "key", key: "f12" }, family: "press_key", native: "press_key" },
{ label: "select all", command: { action: "select_all" }, family: "hotkey", native: "hotkey" },
{ label: "semantic value", command: { action: "set_value", element_token: "field", text: "synthetic-private-value" }, family: "fill", native: "semantic_action" },
{ label: "scroll", command: { action: "scroll", element_token: "field", direction: "down" }, family: "scroll", native: "scroll" },
{ label: "drag", command: { action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }, family: "drag", native: "drag" },
{ label: "keyboard hold", command: { action: "hold", keys: ["ctrl"], duration_ms: 10 }, family: "hotkey", native: "hold_keys" },
{ label: "held pointer", command: { action: "hold", keys: ["ctrl"], duration_ms: 10,
actions: [{ action: "drag", from_x: 10, from_y: 10, to_x: 100, to_y: 100, duration_ms: 10 }] }, family: "input_timeline", native: "hold_keys" },
{ label: "keyboard timeline", command: { action: "timeline", events: [
{ at_ms: 0, type: "key_down", key: "ctrl" }, { at_ms: 10, type: "key_up", key: "ctrl" },
] }, family: "hotkey", native: "input_timeline" },
{ label: "pointer timeline", command: { action: "timeline", events: [
{ at_ms: 0, type: "move", x: 10, y: 10 }, { at_ms: 10, type: "wheel", dy: 1 },
] }, family: "input_timeline", native: "input_timeline" },
];
test("keyboard previews include the requested shortcut", async () => {
const f = await actionFixture();
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear();
await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "hotkey", keys: ["ctrl", "shift", "f12"] });
await tick();
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);
expect(activities).toHaveLength(2);
for (const args of activities) {
expect(String(args.typed_text).toLowerCase()).toContain("ctrl");
expect(String(args.typed_text).toLowerCase()).toContain("shift");
expect(String(args.typed_text).toLowerCase()).toContain("f12");
}
} finally { await f.close(); }
});
test.each([
{ options: { password: true } },
{ options: { settings: (() => { const s = new VisualSettings(); s.update({ typed_text: false }); return s; })() } },
])("printable keys are hidden in a protected field or when typed text is disabled", async ({ options }) => {
const f = await actionFixture("delivered", options);
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear();
await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "a" });
await tick();
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
expect(activities).toHaveLength(2);
for (const [, args] of activities) expect(args).not.toHaveProperty("typed_text");
} finally { await f.close(); }
});
test("timeline previews include non-text keys and pointer event types", async () => {
const f = await actionFixture();
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear();
await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "timeline", events: [
{ at_ms: 0, type: "key_down", key: "f12" }, { at_ms: 1, type: "key_up", key: "f12" },
{ at_ms: 2, type: "wheel", dy: 1 },
] });
await tick();
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
expect(activities).toHaveLength(2);
for (const [, args] of activities) {
expect(String(args.typed_text)).toContain("f12");
expect(String(args.typed_text)).toContain("wheel");
}
} finally { await f.close(); }
});
test.each(nativeFamilies)("native $label retains its presentation family before and after one dispatch", async ({ command, family, native }) => {
const f = await actionFixture();
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();
const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, ...command });
await tick();
expect(result.details.status).toBe("delivered");
expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual([native]);
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);
expect(activities.map(args => ({ kind: args.kind, action: args.action, outcome: args.outcome }))).toEqual([
{ kind: "action", action: family, outcome: undefined }, { kind: "action", action: family, outcome: "delivered" },
]);
for (const args of activities) {
expect(args.source).toEqual({ ...frame.target, identity: frame.identity, bounds: frame.bounds });
expect(Object.keys(args).every(key => ["generation", "kind", "source", "region", "action", "outcome", "typed_text"].includes(key))).toBe(true);
if (!command.element_token) expect(args).not.toHaveProperty("region");
// The human sees what the agent types, before and after the dispatch.
if ("text" in command) expect(args.typed_text).toBe(command.text);
}
} finally { await f.close(); }
});
test.each([
{ label: "a password field", options: { password: true } },
{ label: "typed text turned off", options: { settings: (() => { const settings = new VisualSettings(); settings.update({ typed_text: false }); return settings; })() } },
])("typed text is not shown for $label", async ({ options }) => {
const f = await actionFixture("delivered", options);
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear();
const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "set_value", element_token: "field", text: "secret" });
await tick();
expect(result.details.status).toBe("delivered");
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity").map(([, args]) => args);
expect(activities).toHaveLength(2);
for (const args of activities) expect(args).not.toHaveProperty("typed_text");
} finally { await f.close(); }
});
test("the native renderer gets its own settings without the typed text toggle", () => {
const options = new VisualSettings().current();
expect(options.typed_text).toBe(true);
expect(nativeVisualOptions(options)).toEqual(Object.fromEntries(Object.entries(options).filter(([key]) => key !== "typed_text")));
});
test.each(["refused", "unknown"] as const)("native keyboard %s keeps its family without replaying input", async outcome => {
const f = await actionFixture(outcome);
try {
const listed = await f.session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const read = await f.session.observe(ref); await tick(); f.send.mockClear(); vi.mocked(f.driver.call).mockClear();
const result = await f.session.act({ window_ref: ref, observation_id: read.details.observation_id!, action: "key", key: "f12" });
await tick();
expect(result.details.status).toBe(outcome);
expect(vi.mocked(f.driver.call).mock.calls.map(([name]) => name)).toEqual(["press_key"]);
expect(f.send.mock.calls.map(([, args]) => ({ action: args.action, outcome: args.outcome }))).toEqual([
{ action: "press_key", outcome: undefined }, { action: "press_key", outcome },
]);
} finally { await f.close(); }
});
test.each(["a", "Shift+a", "Control+Alt+a"]) ("browser printable key %s is not exposed when focus is unknown", async key => {
const f = await actionFixture();
try {
const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;
await f.session.observeDom(ref); await tick(); f.send.mockClear();
await f.session.actPage(ref, { action: "press", key }); await tick();
const activities = f.send.mock.calls.filter(([name]) => name === "present_agent_activity");
expect(activities).toHaveLength(1);
expect(activities[0][1]).not.toHaveProperty("typed_text");
} finally { await f.close(); }
});
test.each(["dom", "page"] as const)("browser %s key outcomes include the shortcut without forwarding the target", async route => {
for (const outcome of ["delivered", "refused", "unknown"] as const) {
const f = await actionFixture();
try {
const opened = await f.session.openBrowser(); const ref = opened.details.window_ref!;
const read = await f.session.observeDom(ref); await tick(); f.send.mockClear(); f.browser.page.mockClear();
if (outcome !== "delivered") f.browser.page.mockRejectedValueOnce(new WebError("Synthetic page failure", outcome === "unknown"));
const action: WebAction = { action: "press", target: { ref: "e1" }, key: "Control+Shift+F12" };
const result = route === "dom"
? await f.session.actDom(ref, read.details.observation_id!, action)
: await f.session.actPage(ref, action);
await tick();
expect(result.details.status).toBe(outcome);
expect(f.browser.page).toHaveBeenCalledExactlyOnceWith("act", action, undefined);
expect(f.send.mock.calls).toEqual([["present_agent_activity", {
generation: expect.any(Number), kind: "action", source: { ...frame.target, identity: frame.identity, bounds: frame.bounds },
action: "press_key", outcome, typed_text: action.key,
}]]);
} finally { await f.close(); }
}
});