Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { expect, test, vi } from "vitest";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress } from "../src/computer-resources.ts";
import { ComputerSession, type Driver } from "../src/session.ts";
const cwd = "/synthetic-changes-fixture";
// LPT-837: a changes Read lists only what changed since the window's previous observation, with fresh hints.
test("a changes Read compares the window's UI with its previous observation", async () => {
let text = "", serial = 0;
const element = (index: number, role: string, label: string, value?: string) => ({ element_token: `s${serial}:${index}`, role, label, value, enabled: true,
frame: { x: 100 + index * 50, y: 100, w: 40, h: 20 }, screenshot_frame: { x: 20 + index * 50, y: 10, w: 40, h: 20 } });
const driver: Driver = {
identity: vi.fn(async () => "life"), close: vi.fn(async () => {}),
call: vi.fn<Driver["call"]>(async (name, args) => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 42, window_id: 91 }] } };
if (name === "get_window_state") {
serial++;
// This window cannot be captured, like a minimized one: a capture with pixels reports why.
const pixels = (args as { include_screenshot?: boolean }).include_screenshot !== false;
return { content: [], structuredContent: { pid: 42, window_id: 91, snapshot_id: `s${serial}`, window_bounds: { x: 80, y: 90, width: 500, height: 320 },
...(pixels ? { screenshot_error: "The window has no visible surface." } : {}),
elements_complete: true, elements: [element(0, "Button", "Save"), element(1, "Edit", "Text editor", text), element(2, "Button", "Close")] } };
}
throw Error(`Unexpected native call ${name}`);
}),
};
const session = new ComputerSession(driver, () => "yolo");
const resources = new ComputerResources(async () => session, async () => session.close());
const ref = ((await session.windows()).details.state!.windows as { window_ref: string }[])[0].window_ref;
const window = requireComputerAddress(computerSource("window", ref));
const first = await resources.read(window, cwd, ["changes"]);
expect(first!.details.changes).toEqual({ reason: expect.stringMatching(/full observation/) });
// This window gives no screenshot, so changes falls back to its UI alone.
expect(first!.details.captured).toEqual({ image: false, ui: true });
const same = await resources.read(window, cwd, ["changes"]);
expect(same!.details.changes).toMatchObject({ since: first!.details.observation_id, diff: { counts: { changed: 0, added: 0, removed: 0 } } });
text = "hello";
const typed = await resources.read(window, cwd, ["changes"]);
const diff = typed!.details.changes!.diff!;
expect(diff.counts).toEqual({ changed: 1, added: 0, removed: 0 });
const editor = typed!.details.ui!.elements.find(e => e.name === "Text editor")!;
// The listed hint is the new observation's and resolves to its element.
expect(diff.changed[0]).toBe(`${editor.hint} Edit "Text editor": value="" → "hello"`);
expect(resources.grantsAuthority(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)))).toBe(true);
await expect(resources.read(window, cwd, ["changes", "ocr"])).rejects.toThrow();
// Pixels asked for explicitly are not replaced by a UI-only walk.
expect((await resources.read(window, cwd, ["changes", "full"]))!.details.status).toBe("refused");
await expect(resources.read(requireComputerAddress(computerSource("observation", typed!.details.observation_id!)), cwd, ["changes"]))
.rejects.toThrow(/window address/);
});