Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { expect, test, vi } from "vitest";
import { ComputerSession, SEQUENCE_ELAPSED_MS, type Action, type ComputerMode, type Driver } from "../src/session.ts";
async function fixture() {
let mode: ComputerMode = "yolo", lifetime = "alive";
const driver: Driver = {
heldPaths: true, identity: vi.fn(async () => lifetime), close: vi.fn(async () => {}),
call: vi.fn<Driver["call"]>(async (name) => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 100 }] } };
if (name === "get_window_state") return { content: [{ type: "image", mimeType: "image/png", data: "cG5n" }], structuredContent: {
elements: [{ element_token: "field" }], screenshot_width: 500, screenshot_height: 350, capture_id: "capture-one",
} };
return { content: [], structuredContent: { effect: "unverifiable" } };
}),
};
const session = new ComputerSession(driver, () => mode);
const listed = await session.windows();
const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
const overview = await session.observe(ref);
const base = { window_ref: ref, observation_id: overview.details.observation_id! };
const send = (command: Record<string, unknown>, signal?: AbortSignal) => session.act({ ...base, ...command } as Action, signal);
const input = () => vi.mocked(driver.call).mock.calls.filter(([name]) => !["list_windows", "get_window_state"].includes(name));
return { session, driver, send, base, input, mode: (value: ComputerMode) => { mode = value; }, recycle: () => { lifetime = "reused"; } };
}
test.each([
[{ action: "key", key: "up", modifiers: ["shift"] }, "press_key", { key: "up", modifiers: ["shift"] }],
[{ action: "hotkey", keys: ["ctrl", "a"] }, "hotkey", { keys: ["ctrl", "a"] }],
[{ action: "type_text", text: "abc", delay_ms: 20 }, "type_text", { text: "abc", delay_ms: 20 }],
[{ action: "right_click", x: 100, y: 80, modifiers: ["shift"] }, "right_click", { x: 100, y: 80, modifier: ["shift"] }],
[{ action: "double_click", element_token: "field" }, "double_click", { element_token: "field" }],
[{ action: "click", x: 100, y: 80, button: "middle" }, "click", { x: 100, y: 80, button: "middle", capture_id: "capture-one" }],
[{ action: "click", x: 100, y: 80, count: 3 }, "click", { x: 100, y: 80, count: 3, capture_id: "capture-one" }],
[{ action: "click", element_token: "field", button: "right", count: 2 }, "click", { element_token: "field", button: "right", count: 2 }],
[{ action: "scroll", element_token: "field", direction: "down", amount: 2, by: "page" }, "scroll", { element_token: "field", direction: "down", amount: 2, by: "page" }],
[{ action: "drag", from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifiers: ["ctrl"] }, "drag",
{ from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12, button: "left", modifier: ["ctrl"] }],
])("maps complete native input without a desktop or browser route: %j", async (command, nativeName, expected) => {
const f = await fixture();
expect((await f.send(command)).details.status).toBe("delivered");
const target = ["right_click", "double_click"].includes(nativeName) ? { pid: 42, window_id: 100 } : { target: { kind: "window", pid: 42, window_id: 100 } };
expect(f.input()).toEqual([[nativeName, { ...target, delivery_mode: "background", ...expected }, undefined]]);
expect((await f.send(command)).details.status).toBe("refused");
expect(f.input()).toHaveLength(1);
});
test.each([
{ action: "key", key: "key_down" }, { action: "key", key: "a", modifiers: ["hyper"] },
{ action: "hotkey", keys: ["a", "b"] }, { action: "hotkey", keys: ["ctrl", "ctrl", "a"] },
{ action: "type_text", text: "abc", delay_ms: 201 }, { action: "type_text", text: "x".repeat(2001) },
{ action: "drag", from_x: 10, from_y: 10, to_x: 500, to_y: 20 },
{ action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20, steps: 201 },
{ action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20, duration_ms: 10001 },
{ action: "scroll", direction: "down", amount: 51 }, { action: "scroll", direction: "sideways" },
{ action: "scroll", x: 100, y: 80, direction: "down" },
{ action: "scroll", direction: "down", amount: 2 },
{ action: "key", key: "a", duration_ms: 1000 }, { action: "click", x: 10, y: 10, keys: ["ctrl", "a"] },
{ action: "click", x: 10, y: 10, count: 4 }, { action: "click", x: 10, y: 10, count: 0 }, { action: "double_click", x: 10, y: 10, count: 3 },
{ action: "sequence", sequence: [] },
{ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "drag", from_x: 1, from_y: 1, to_x: NaN, to_y: 2 }] },
{ action: "sequence", sequence: [{ action: "hover", x: 10, y: 10 }] },
{ action: "sequence", sequence: Array.from({ length: 2001 }, () => ({ action: "key", key: "a" })) },
{ action: "sequence", sequence: Array.from({ length: 181 }, () => ({ action: "key", key: "a", wait_ms: 5000 })) },
])("rejects the entire unsupported or ungrounded command before input: %j", async command => {
const f = await fixture();
expect((await f.send(command)).details.status).toBe("refused"); expect(f.input()).toHaveLength(0);
expect((await f.send({ action: "key", key: "return" })).details.status).toBe("delivered");
});
test("a qualified straight drag keeps both endpoints in the captured window image", async () => {
const f = await fixture(); Object.assign(f.driver, { captureBoundDrag: true });
const result = await f.send({ action: "drag", from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12 });
expect(result.details.status).toBe("delivered");
expect(f.input()).toEqual([["drag", { target: { kind: "window", pid: 42, window_id: 100 }, delivery_mode: "background",
capture_id: "capture-one", from_x: 30, from_y: 40, to_x: 200, to_y: 80, duration_ms: 250, steps: 12 }, undefined]]);
});
test("a qualified drag with no native capture refuses the entire batch before its first key", async () => {
const f = await fixture(); Object.assign(f.driver, { captureBoundDrag: true }); const call = f.driver.call;
f.driver.call = vi.fn(async (name, args, signal) => {
const result = await call(name, args, signal);
if (name === "get_window_state" && result.structuredContent) delete result.structuredContent.capture_id;
return result;
});
const view = await f.session.observe(f.base.window_ref);
const result = await f.session.act({ window_ref: f.base.window_ref, observation_id: view.details.observation_id!, action: "sequence",
sequence: [{ action: "key", key: "a" }, { action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20 }] });
expect(result.details.status).toBe("refused"); expect(f.input()).toHaveLength(0);
});
test("native resize refusal stops a capture-bound drag batch without replay or an unbound fallback", async () => {
const f = await fixture(); Object.assign(f.driver, { captureBoundDrag: true }); const call = f.driver.call;
f.driver.call = vi.fn(async (name, args, signal) => name === "drag" && args.capture_id === "capture-one"
? { content: [], isError: true, structuredContent: { code: "capture_frame_mismatch", effect: "refused" } }
: call(name, args, signal));
const command = { action: "sequence", sequence: [{ action: "key", key: "a" },
{ action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20 }, { action: "key", key: "b" }] };
const result = await f.send(command);
expect(result.details.status).toBe("refused");
expect(result.details.state!.sequence).toMatchObject({ total: 3, attempted: 2, completed: 1 });
expect(f.input().map(([name]) => name)).toEqual(["press_key", "drag"]);
expect((await f.send(command)).details.status).toBe("refused"); expect(f.input()).toHaveLength(2);
});
test("an inactive held-path backend refuses the whole sequence before its first key", async () => {
const f = await fixture(); f.driver.heldPaths = false;
const result = await f.send({ action: "sequence", sequence: [
{ action: "key", key: "a" },
{ action: "drag", from_x: 1, from_y: 1, via: [{ x: 10, y: 20 }], to_x: 30, to_y: 40 },
] });
expect(result.details.status).toBe("refused"); expect(f.input()).toHaveLength(0);
});
test("a curved drag forwards all local points once and preserves a partial native receipt", async () => {
const f = await fixture(); Object.assign(f.driver, { captureBoundDrag: true });
f.driver.call = vi.fn(async () => ({ content: [], structuredContent: {
effect: "partial", delivery: { delivered_count: 2 }, summary: "2/4 points; pressed=true released=true",
} }));
const result = await f.send({ action: "sequence", delivery_mode: "foreground", sequence: [
{ action: "drag", from_x: 20, from_y: 30, via: [{ x: 40, y: 80 }, { x: 90, y: 70 }], to_x: 100, to_y: 30, duration_ms: 800 },
{ action: "key", key: "a" },
] });
expect(f.driver.call).toHaveBeenCalledExactlyOnceWith("drag", {
pid: 42, window_id: 100, delivery_mode: "foreground", from_x: 20, from_y: 30,
via: [{ x: 40, y: 80 }, { x: 90, y: 70 }], to_x: 100, to_y: 30, duration_ms: 800,
}, undefined);
expect(result.details.status).toBe("unknown");
expect(result.details.state).toMatchObject({ delivery: { delivered_count: 2 }, summary: "2/4 points; pressed=true released=true",
sequence: { attempted: 1, completed: 0 } });
expect((await f.send({ action: "key", key: "a" })).details.status).toBe("refused");
expect(f.driver.call).toHaveBeenCalledTimes(1);
});
test.each([
{ via: [] }, { via: [{ x: 500, y: 1 }] }, { via: [{ x: NaN, y: 2 }] },
{ via: [{ x: 2, y: Infinity }] }, { via: [{ x: 2, y: 3, desktop: true }] },
{ via: Array.from({ length: 255 }, () => ({ x: 2, y: 3 })) },
{ via: [{ x: 2, y: 3 }], steps: 10 }, { via: [{ x: 2, y: 3 }], modifiers: [] },
{ via: [{ x: 2, y: 3 }], button: "middle" },
])("an invalid curved tail prevents every batch input: %j", async tail => {
const f = await fixture();
const result = await f.send({ action: "sequence", sequence: [{ action: "key", key: "a" },
{ action: "drag", from_x: 1, from_y: 1, to_x: 10, to_y: 10, ...tail }] });
expect(result.details.status).toBe("refused"); expect(f.input()).toHaveLength(0);
});
test("admission snapshots the entire path before asynchronous native targeting", async () => {
const f = await fixture(); const via = [{ x: 40, y: 80 }];
f.driver.identity = vi.fn(async () => { via[0].x = 499; via.push({ x: 100, y: 110 }); return "alive"; });
const result = await f.send({ action: "drag", from_x: 20, from_y: 30, via, to_x: 100, to_y: 30 });
expect(result.details.status).toBe("delivered");
expect(f.input()[0][1].via).toEqual([{ x: 40, y: 80 }]);
});
test("a bounded sequence dispatches complete steps once, without new captures, and measures waits", async () => {
const f = await fixture();
const result = await f.send({ action: "sequence", sequence: [
{ action: "key", key: "a" }, { action: "hotkey", keys: ["shift", "b"], wait_ms: 20 },
{ action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20, duration_ms: 0, steps: 1 },
] });
expect(result.details.status).toBe("delivered");
expect(f.input().map(([name]) => name)).toEqual(["press_key", "hotkey", "drag"]);
const trace = result.details.state!.sequence as { total: number; completed: number; attempted: number; steps: { status: string; requested_wait_ms: number; dispatch_ms: number; completion_ms: number }[] };
expect(trace).toMatchObject({ total: 3, completed: 3, attempted: 3 });
expect(trace.steps.map(step => step.status)).toEqual(["delivered", "delivered", "delivered"]);
expect(trace.steps[1].requested_wait_ms).toBe(20);
expect(trace.steps[1].dispatch_ms - trace.steps[0].completion_ms).toBeGreaterThanOrEqual(19);
expect(vi.mocked(f.driver.call).mock.calls.filter(([name]) => name === "get_window_state")).toHaveLength(1);
expect((await f.send({ action: "key", key: "a" })).details.status).toBe("refused");
});
// Models write Enter, ESC or Control+S; the benchmark lost turns to "Unsupported key" for them (LPT-838).
test("common key spellings reach the driver under its own names, and an unknown key names the valid ones", async () => {
const f = await fixture();
const result = await f.send({ action: "sequence", sequence: [
{ action: "key", key: "Enter" }, { action: "key", key: "ESC", modifiers: ["Shift"] }, { action: "hotkey", keys: ["Control", "S"] },
{ action: "key", key: "minus" }, { action: "key", key: "/" }] });
expect(result.details.status).toBe("delivered");
expect(f.input().map(([, args]) => args.key ?? args.keys)).toEqual(["return", "escape", ["ctrl", "s"], "-", "/"]);
expect(f.input()[1][1].modifiers).toEqual(["shift"]);
const unknown = await (await fixture()).send({ action: "key", key: "Enterr" });
expect(unknown.details.status).toBe("refused");
expect(JSON.stringify(unknown.content)).toMatch(/Unsupported key \\"enterr\\". Use a-z, 0-9, .*return \(Enter\)/);
});
test.each(["refused", "partial", "throw"])("sequence stops after %s with a truthful completed prefix and no replay", async failure => {
const f = await fixture(); const native = f.driver.call; let n = 0;
f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
if (++n === 2) {
if (failure === "throw") throw new Error("transport lost during input");
return { isError: true, content: [{ type: "text", text: "native diagnostic" }], structuredContent: { effect: failure } };
}
return native(...args);
});
const result = await f.send({ action: "sequence", sequence: ["a", "b", "c"].map(key => ({ action: "key", key })) });
expect(result.details.status).toBe(failure === "refused" ? "refused" : "unknown");
expect(f.input()).toHaveLength(2);
expect(result.details.state!.sequence).toMatchObject({ total: 3, attempted: 2, completed: 1 });
expect((await f.send({ action: "key", key: "c" })).details.status).toBe("refused");
expect(f.input()).toHaveLength(2);
});
test("CUA's occlusion refusal stops a sequence without calling it unknown or retrying it", async () => {
const f = await fixture();
f.driver.call = vi.fn(async () => ({ isError: true, content: [], structuredContent: {
code: "background_occluded", cause: "Target point is occluded; no coordinate injection was sent.",
} }));
const result = await f.send({ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "key", key: "b" }] });
expect(result.details.status).toBe("refused");
expect(result.details.state!.sequence).toMatchObject({ attempted: 1, completed: 0 });
expect(f.driver.call).toHaveBeenCalledTimes(1);
expect((await f.send({ action: "key", key: "a" })).details.status).toBe("refused");
expect(f.driver.call).toHaveBeenCalledTimes(1);
});
test("maximum held paths return readable counts rather than a truncated point dump", async () => {
const f = await fixture();
const sequence = Array.from({ length: 32 }, () => ({ action: "drag", from_x: 20, from_y: 30,
via: Array.from({ length: 254 }, (_, i) => ({ x: 20 + i / 2, y: 30 + i / 2 })), to_x: 200, to_y: 200, duration_ms: 0 }));
const result = await f.send({ action: "sequence", delivery_mode: "foreground", sequence });
expect(result.details.status).toBe("delivered");
const content = result.content.find(block => block.type === "text")!;
if (content.type !== "text") throw Error("Missing receipt");
const readable = JSON.parse(content.text.slice(content.text.indexOf("{")));
expect(readable.state.sequence.steps).toHaveLength(32);
expect(readable.state.sequence.steps.every((step: { requested: { via_points: number; via?: unknown } }) =>
step.requested.via_points === 254 && step.requested.via === undefined)).toBe(true);
});
test("the maximum stroke sequence returns every measured step as readable JSON", async () => {
const f = await fixture();
const sequence = Array.from({length:32}, () => ({action:"drag",from_x:11.123456789123456,from_y:22.98765432198765,
to_x:100.12345678912345,to_y:200.98765432198765,duration_ms:0,steps:200,button:"middle",modifiers:["ctrl","shift","alt","win"]}));
const result = await f.send({action:"sequence",sequence});
const content = result.content.find(block => block.type === "text")!;
if (content.type !== "text") throw Error("Missing receipt");
const readable = JSON.parse(content.text.slice(content.text.indexOf("{")));
expect(readable.state.sequence.steps).toHaveLength(32);
});
test("sequence stops if admission uses the remaining elapsed budget", async () => {
const f = await fixture(); let now = 0, checks = 0;
const clock = vi.spyOn(performance, "now").mockImplementation(() => now);
const identity = f.driver.identity;
f.driver.identity = vi.fn<Driver["identity"]>(async (...args) => { if (++checks === 2) now = SEQUENCE_ELAPSED_MS; return identity(...args); });
try {
const result = await f.send({ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "key", key: "b" }] });
expect(result.details.status).toBe("refused");
expect(f.input()).toHaveLength(1);
expect(result.details.state!.sequence).toMatchObject({ attempted: 1, completed: 1 });
} finally { clock.mockRestore(); }
});
test.each(["cancel", "safe", "recycle"])("sequence stops before another dispatch on %s", async interruption => {
const f = await fixture(), abort = new AbortController(); Object.assign(f.driver, { captureBoundDrag: true }); const native = f.driver.call;
f.driver.call = vi.fn<Driver["call"]>(async (...args) => {
const result = await native(...args);
if (interruption === "cancel") abort.abort();
if (interruption === "safe") f.mode("safe");
if (interruption === "recycle") f.recycle();
return result;
});
const result = await f.send({ action: "sequence", sequence: [{ action: "key", key: "a" }, { action: "drag", from_x: 10, from_y: 10, to_x: 20, to_y: 20, wait_ms: 200 }] }, abort.signal);
expect(result.details.status).not.toBe("delivered"); expect(f.input()).toHaveLength(1);
expect(result.details.state!.sequence).toMatchObject({ total: 2, attempted: 1 });
});