Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import sharp from "sharp";
import { expect, test, vi } from "vitest";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
import { presentComputerResult } from "../src/provider.ts";
import { ComputerSession, type ComputerResult, type Content, type Driver } from "../src/session.ts";
const cwd = "/synthetic-action-changes";
const address = requireComputerAddress;
// LPT-914: every action reports what changed in the window, like Playwright MCP's incremental snapshots.
// LPT-915: and how the window's pixels changed, with before/after crops of the changed area.
async function fixture() {
let count = 0, serial = 0;
const dots: { x: number; y: number }[] = [];
// The window paints one bar per count under the button and a dot wherever a pixel click lands on the canvas.
const frame = async () => {
const shapes = [...Array.from({ length: count }, (_, i) => `<rect x="${20 + i * 12}" y="40" width="10" height="10" fill="#000"/>`),
...dots.map(dot => `<rect x="${dot.x - 3}" y="${dot.y - 3}" width="6" height="6" fill="#c00"/>`)];
const png = await sharp({ create: { width: 500, height: 320, channels: 3, background: "#fff" } })
.composite([{ input: Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="500" height="320">${shapes.join("")}</svg>`) }]).png().toBuffer();
return { type: "image" as const, mimeType: "image/png", data: png.toString("base64") };
};
const element = (index: number, role: string, label: string) => ({ element_token: `s${serial}:${index}`, role, label, enabled: true, actions: role === "Button" ? ["invoke"] : [],
frame: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });
const driver: Driver = {
identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),
call: vi.fn<Driver["call"]>(async (name, args) => {
const input = args as { element_token?: string; x?: number; y?: number; include_screenshot?: boolean };
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };
if (name === "get_window_state") {
serial++;
const pixels = input.include_screenshot !== false;
return { content: pixels ? [await frame()] : [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },
...(pixels ? { capture_id: `c${serial}`, screenshot_width: 500, screenshot_height: 320 } : {}),
elements_complete: true, elements: [element(0, "Button", `Count ${count}`), element(1, "Text", "Status")] } };
}
// Clicking the button counts; clicking the status label does nothing; a pixel click paints a dot there.
if (name === "click") {
if (input.element_token?.endsWith(":0")) count++;
if (input.x !== undefined && input.y !== undefined && input.y > 100) dots.push({ x: input.x, y: input.y });
return { content: [], structuredContent: { effect: "confirmed" } };
}
throw Error(`Unexpected native call ${name}`);
}),
};
const session = new ComputerSession(driver, () => "yolo");
const resources = new ComputerResources(async () => session, async () => session.close());
const ref = ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
const window = computerSource("window", ref);
const read = async (views = ["ui"]) => {
const view = (await resources.read(address(window), cwd, views))!;
const observation = computerSource("observation", view.details.observation_id!);
return { view, observation: address(observation), at: (index: number) => ({ ...address(observation), hint: view.details.ui!.elements[index].hint }) };
};
const walks = () => vi.mocked(driver.call).mock.calls.filter(([name]) => name === "get_window_state").length;
// The app paints by itself, as an animation or another person would.
const paint = (x: number, y: number) => { dots.push({ x, y }); };
return { resources, window, read, walks, paint };
}
const images = (result: ComputerResult) => result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
test("an action on an observation with pixels reports the changed area with before and after crops", async () => {
const f = await fixture(), { at } = await f.read(["full"]);
const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
expect(result.details.successor?.diff?.pixels).toMatchObject({ box: { x: 20, y: 40, width: 10, height: 10 }, regions: 1, overview: true, crops: [{ x: 8, y: 28, width: 34, height: 34, scale: 4 }] });
expect(presentComputerResult(result)).toMatch(/~ \d+:a Button "Count 0" → "Count 1"\nPixels changed in 1 area \(overview pixels; 0\.06% of the window\); image 1 outlines and numbers them on the window:\n 1\. x=20 y=40 w=10 h=10( \(at the input point\))?; images 2 and 3 show x=8 y=28 w=34 h=34 before and after, enlarged 4x\n/);
// Image 1 is the window with the area outlined; the crops follow it.
const [overview, before, after] = images(result);
expect(await sharp(Buffer.from(overview.data, "base64")).metadata()).toMatchObject({ width: expect.any(Number) });
expect((await sharp(Buffer.from(before.data, "base64")).stats()).channels[0].min).toBe(255);
expect((await sharp(Buffer.from(after.data, "base64")).stats()).channels[0].min).toBe(0);
});
test("pixel input on a canvas reports only its pixel change, and empty space reports nothing", async () => {
const f = await fixture(), { observation } = await f.read(["full"]);
const stroke = await f.resources.perform(observation, cwd, JSON.stringify({ action: "click", x: 300, y: 200 }));
expect(stroke.details.successor?.diff).toMatchObject({ counts: { changed: 0, added: 0, removed: 0 }, pixels: { box: { x: 297, y: 197, width: 6, height: 6 } } });
// The overview and one before/after pair; the stroke is at the input point, so it is area 1.
expect(images(stroke)).toHaveLength(3);
expect(stroke.details.successor?.diff?.pixels?.areas[0]).toMatchObject({ at_input: true });
const next = address(computerSource("observation", stroke.details.successor!.observation_id!));
const empty = await f.resources.perform(next, cwd, JSON.stringify({ action: "click", x: 300, y: 60 }));
expect(presentComputerResult(empty)).toMatch(/Nothing changed in the UI or the window's pixels within \d+ ms after the action\./);
expect(images(empty)).toHaveLength(0);
});
test("an action on a UI-only observation says that pixels were not compared, and its successor has them", async () => {
const f = await fixture(), { at } = await f.read();
const first = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
expect(presentComputerResult(first)).toMatch(/Pixels were not compared: the input observation had no pixels\./);
const view = (await f.resources.read(address(computerSource("observation", first.details.successor!.observation_id!)), cwd, []))!;
const hint = view.details.ui!.elements[0].hint;
const second = await f.resources.perform({ ...address(computerSource("observation", first.details.successor!.observation_id!)), hint } as ReturnType<typeof address>, cwd, JSON.stringify({ action: "click" }));
expect(second.details.successor?.diff?.pixels?.box).toEqual({ x: 32, y: 40, width: 10, height: 10 });
});
test("a changes Read compares the window's pixels with its previous observation", async () => {
const f = await fixture();
await f.read(["full"]);
f.paint(200, 150);
const changes = (await f.resources.read(address(f.window), cwd, ["changes"]))!;
expect(changes.details.changes?.diff?.pixels?.box).toEqual({ x: 197, y: 147, width: 6, height: 6 });
expect(presentComputerResult(changes)).toMatch(/Pixels changed in 1 area .*\n 1\. x=197 y=147 w=6 h=6;/);
// The outlined overview and the two crops, not the full-size window.
expect(images(changes)).toHaveLength(3);
});
test("an action reports what it changed by default, and its successor is the current observation", async () => {
const f = await fixture(), { at } = await f.read();
const result = await f.resources.perform(at(0), cwd, JSON.stringify({ action: "click" }));
const shown = presentComputerResult(result);
expect(shown).toMatch(/Changes: 1 changed, 0 added, 0 removed\n~ \d+:a Button "Count 0" → "Count 1"/);
expect(f.resources.grantsAuthority(address(computerSource("observation", result.details.successor!.observation_id!)))).toBe(true);
});
test("an action that changes nothing says so after looking again for a moment", async () => {
const f = await fixture(), { at } = await f.read();
const result = await f.resources.perform(at(1), cwd, JSON.stringify({ action: "click" }));
expect(result.details.successor?.diff?.waited_ms).toBeGreaterThanOrEqual(300);
expect(presentComputerResult(result)).toMatch(/Nothing changed in the UI within \d+ ms after the action\. Check that the target was right/);
});
test("a locator action on the window address compares with the walk that resolved its target", async () => {
const f = await fixture();
const result = await f.resources.perform(address(f.window), cwd, JSON.stringify({ action: "click", target: "getByRole('button', { name: /Count/ })" }));
expect(result.details.successor?.diff?.changed).toEqual([expect.stringMatching(/Button "Count 0" → "Count 1"$/)]);
});