Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { expect, test, vi } from "vitest";
import { parseComputerViews } from "../src/computer-views.ts";
import { ComputerSession, textMatcher, type Driver, type DriverResult } from "../src/session.ts";
test("discovery queries match substrings or explicit regular expressions", () => {
expect(textMatcher("paint")("Untitled - Paint")).toBe(true);
expect(textMatcher("regex:.")("anything")).toBe(true);
expect(textMatcher("regex:^untitled.*paint$")("Untitled - Paint")).toBe(true);
expect(textMatcher("regex:^paint")("Untitled - Paint")).toBe(false);
expect(textMatcher("regex:.")("")).toBe(false);
expect(() => textMatcher("regex:(")).toThrow(/Invalid regular expression/);
});
test("Read views accept granularity in either form and name conflicts", () => {
expect(parseComputerViews(['capture:{"granularity":"image","maxLongEdge":320}']))
.toEqual({ kind: "overview", capture: { granularity: "image", maxLongEdge: 320 } });
expect(parseComputerViews(["image", 'capture:{"maxLongEdge":320}'])).toEqual({ kind: "overview", capture: { granularity: "image", maxLongEdge: 320 } });
expect(() => parseComputerViews(["backend:default"])).toThrow(/Unknown view backend:default/);
expect(() => parseComputerViews(["image", 'capture:{"granularity":"ui"}'])).toThrow(/granularity is given twice/);
expect(() => parseComputerViews(["image", "full"])).toThrow(/got image and full/);
expect(() => parseComputerViews(['capture:{"size":1}'])).toThrow(/unsupported options: size/);
});
test("an anchored window regex matches the title alone", async () => {
const call = vi.fn<Driver["call"]>(async (name): Promise<DriverResult> => {
if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid: 4, window_id: 9, app_name: "explorer.exe", title: "Program Manager" }] } };
throw new Error(`Unexpected native call ${name}`);
});
const session = new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }, () => "yolo");
const titles = async (query: string) => ((await session.windows(undefined, query)).details.state!.windows as { title: string }[]).map(window => window.title);
expect(await titles("regex:^Program Manager$")).toEqual(["Program Manager"]);
expect(await titles("regex:^explorer\\.exe$")).toEqual(["Program Manager"]);
expect(await titles("regex:^Manager")).toEqual([]);
});
function appSession(apps: { name: string; launch_path: string }[]) {
const call = vi.fn<Driver["call"]>(async (name): Promise<DriverResult> => {
if (name === "list_apps") return { content: [], structuredContent: { apps } };
if (name === "list_windows") return { content: [], structuredContent: { windows: [] } };
if (name === "launch_app") throw new Error("launch stopped by the fixture");
throw new Error(`Unexpected native call ${name}`);
});
return { call, session: new ComputerSession({ call, identity: vi.fn(async () => "life"), close: vi.fn(async () => {}) }) };
}
test("apps open by executable name only when no installed name matches, and misses suggest close names", async () => {
const { call, session } = appSession([
{ name: "Notepad", launch_path: "shell:appsFolder\\Microsoft.WindowsNotepad_8wekyb3d8bbwe!App" },
{ name: "Paint", launch_path: "shell:appsFolder\\Microsoft.Paint_8wekyb3d8bbwe!App" },
{ name: "Calculator Pro", launch_path: "C:\\Tools\\calc2.exe" },
]);
await session.open("calc2");
expect(call).toHaveBeenCalledWith("launch_app", { launch_path: "C:\\Tools\\calc2.exe" }, undefined);
const missed = await session.open("mspaint");
expect(missed.details.status).toBe("refused");
expect(missed.content[0]).toMatchObject({ text: expect.stringContaining("Did you mean: Paint?") });
expect(call.mock.calls.filter(([name]) => name === "launch_app")).toHaveLength(1);
});