Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { afterAll, expect, test } from "vitest";
import { requireComputerAddress, type InputPlan } from "../../src/computer-resources.ts";
import { clickPlan, dragPlan, hoverPlan, pressPlan, scrollPlan, selectPlan, timelinePlan, typePlan, waitCommand } from "../../src/computer-tools.ts";
import { presentComputerResult } from "../../src/provider.ts";
import { deleteComputer } from "../../src/provider.ts";
import { connectLive, nativeForm, openWindow, serveFixture, WINDOWS_POWERSHELL_PATH } from "./remote.ts";
// LPT-927: the typed input tools on a native WinForms window and on a Playwright browser page of the configured machine.
const live = await connectLive();
const fixture = await serveFixture(live.runner);
afterAll(async () => { await live.close(); fixture.close(); });
const timings: string[] = [];
const act = async (window: string, plan: InputPlan) => {
const started = Date.now();
const result = await live.resources.input(requireComputerAddress(window), live.cwd, plan);
const body = presentComputerResult(result);
timings.push(`${plan.label}${plan.target ? ` ${plan.target}` : plan.point ? ` ${plan.point.x},${plan.point.y}` : ""}: ${result.details.status} in ${Date.now() - started} ms`);
console.log(timings.at(-1));
return { result, body };
};
const holds = async (window: string, conditions: Parameters<typeof waitCommand>[0]["conditions"]) => {
const result = await live.insert(window, waitCommand({ window, timeout_ms: 3000, conditions }));
return (result.details.verification as { verdict?: string } | undefined)?.verdict;
};
afterAll(() => console.log(timings.join("\n")));
test("one action vocabulary drives a native window by locator, ref and point", async () => {
const title = `pi-input-${Date.now()}`;
const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);
try {
const button = "getByRole('button', { name: /^Count/ })";
expect((await act(window, clickPlan({ window, target: button }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 1' })", is: "exists" }])).toBe("satisfied");
// A full Read gives the button's place in the image; a later UI-only Read keeps that image as the space of points.
const full = await live.read(window, ["full"]);
const element = (full.details.ui as { elements: { hint: string; role: string; name?: string; bounds: { window_image?: { x: number; y: number; width: number; height: number } } }[] })
.elements.find(item => /Count 1/.test(item.name ?? ""))!;
const box = element.bounds.window_image!;
const centre = { x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) };
const aria = await live.read(window, ["aria"]);
const ref = /button "Count 1" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(aria))![1];
expect((await act(window, clickPlan({ window, target: ref }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 2' })", is: "exists" }])).toBe("satisfied");
await live.read(window, ["aria"]);
// The UI-only Read has no pixels, so the point click captures the window first and keeps the full Read's scale.
expect((await act(window, clickPlan({ window, ...centre }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 3' })", is: "exists" }])).toBe("satisfied");
const field = "getByRole('textbox', { name: 'Field' })";
expect((await act(window, typePlan({ window, target: field, text: "hello" }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: field, value: "hello" }])).toBe("satisfied");
expect((await act(window, typePlan({ window, target: field, text: "world", replace: true, submit: true }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: field, value: "world" }])).toBe("satisfied");
// Keys are real key presses, so the app behaves as for a person: a WinForms TextBox ignores ctrl+a. End clears the
// selection replace left, and shift+Home selects the text again only if the chord arrives as a chord: a bare Home
// would leave the caret at the start and Backspace would delete nothing.
for (const key of ["End", "shift+Home", "Backspace"]) expect((await act(window, pressPlan({ window, key }))).result.details.status).toBe("delivered");
// An empty field's value falls back to its name in conditions, so its aria line shows the emptiness: no ": value".
const field_line = /textbox "Field".*$/m.exec(presentComputerResult(await live.read(window, ["aria"])))![0];
expect(field_line).toMatch(/^textbox "Field" \[ref=[0-9a-z]+:[a-z]+\]$/);
expect((await act(window, selectPlan({ window, target: "getByRole('list', { name: 'Pick' })", values: ["beta"] }))).result.details.status).toBe("delivered");
expect(await holds(window, [{ target: "getByRole('option', { name: 'beta' })", is: "selected" }])).toBe("satisfied");
expect((await act(window, scrollPlan({ window, target: "getByRole('list', { name: 'Pick' })", delta_y: 1 }))).result.details.status).not.toBe("unknown");
const wheel = await act(window, scrollPlan({ window, x: centre.x, y: centre.y + 60, delta_y: 1 }));
expect(wheel.result.details.status, wheel.body).toBe("delivered");
expect((await act(window, hoverPlan({ window, target: button }))).result.details.status).toBe("delivered");
expect((await act(window, dragPlan({ window, from: { x: centre.x, y: centre.y }, to: { x: centre.x + 40, y: centre.y } }))).result.details.status).toBe("delivered");
const timeline = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: centre.x, y: centre.y }, { at_ms: 50, type: "button_down" }, { at_ms: 100, type: "button_up" }] }));
expect(timeline.result.details.status, timeline.body).toBe("delivered");
expect(await holds(window, [{ target: "getByRole('button', { name: /^Count [45]$/ })", is: "exists" }])).toBe("satisfied");
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
// LPT-942: what the agent sees structurally is pressed structurally; without a pattern the same call uses the real pointer.
test("a ref is pressed through UI Automation without taking the foreground, and a ref without a pattern gets the real pointer", async () => {
const title = `pi-route-${Date.now()}`;
const window = await openWindow(live, live.machine, WINDOWS_POWERSHELL_PATH, nativeForm(title), title);
try {
const foreground = async () => ((await live.read(window, ["state"])).details.state?.state as { foreground_at_target: boolean }).foreground_at_target;
const refOf = async (pattern: RegExp) => pattern.exec(presentComputerResult(await live.read(window, ["ui"])))![1];
const button = await refOf(/^([0-9a-z]+:[a-z]+) \| Button \| Count 0/m);
const structural = await act(window, clickPlan({ window, target: button }));
expect(structural.result.details.status, structural.body).toBe("delivered");
expect(structural.body).not.toContain("real pointer");
expect(await holds(window, [{ target: "getByRole('button', { name: 'Count 1' })", is: "exists" }])).toBe("satisfied");
// UI Automation pressed the button in the background: the window never came to the front.
expect(await foreground()).toBe(false);
const field = await refOf(/^([0-9a-z]+:[a-z]+) \| Edit \| Field/m);
const real = await act(window, clickPlan({ window, target: field }));
expect(real.result.details.status, real.body).not.toBe("refused");
expect(real.body).toContain("No UI Automation pattern of the target carried this click, so this call used the real pointer.");
} finally {
await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);
}
});
test("points of a native canvas draw strokes that the receipt shows as changed pixels", async () => {
const opened = await live.insert(live.machine, { action: "open", app: "Paint" });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
const window = `computer:window/${opened.details.window_ref}`;
try {
const image = await live.read(window, ["image"]);
const size = image.details.state as { screenshot_width: number; screenshot_height: number };
const middle = { x: Math.round(size.screenshot_width / 2), y: Math.round(size.screenshot_height * 0.6) };
const stroke = await act(window, dragPlan({ window, from: { x: middle.x - 120, y: middle.y }, to: { x: middle.x + 120, y: middle.y + 40 }, via: [{ x: middle.x, y: middle.y - 60 }] }));
expect(stroke.result.details.status, stroke.body).toBe("delivered");
expect(stroke.body).toMatch(/Pixels changed in \d+ area/);
const line = await act(window, timelinePlan({ window, events: [{ at_ms: 0, type: "move", x: middle.x - 100, y: middle.y + 80 }, { at_ms: 20, type: "button_down" },
{ at_ms: 60, type: "move", x: middle.x, y: middle.y + 100 }, { at_ms: 100, type: "move", x: middle.x + 100, y: middle.y + 80 }, { at_ms: 140, type: "button_up" }] }));
expect(line.result.details.status, line.body).toBe("delivered");
expect(line.body).toMatch(/Pixels changed in \d+ area/);
} finally {
// Paint asks about the unsaved drawing, so the window the session opened is ended without saving.
await deleteComputer(live.resources, `${window}?force=true`, { path: `${window}?force=true` }, live.cwd);
}
});
test("points of a browser page take clicks, wheel, drags, held keys and a pressure pen through Playwright", async () => {
const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/input` });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
const window = `computer:window/${opened.details.window_ref}`;
const since = (await fixture.journal()).length;
try {
const delivered = async (plan: InputPlan) => { const { result, body } = await act(window, plan); expect(result.details.status, body).toBe("delivered"); };
await delivered(clickPlan({ window, x: 200, y: 200, modifiers: ["shift"] }));
await delivered(clickPlan({ window, x: 220, y: 200, count: 2 }));
// The double click asked the pad for pointer lock; leave it so later moves are ordinary.
await delivered(pressPlan({ window, key: "Escape" }));
await delivered(scrollPlan({ window, x: 300, y: 300, delta_y: 2 }));
await delivered(dragPlan({ window, from: { x: 100, y: 150 }, to: { x: 400, y: 260 }, via: [{ x: 250, y: 150 }], duration_ms: 300 }));
await delivered(pressPlan({ window, key: "w", hold_ms: 400 }));
await delivered(timelinePlan({ window, events: [{ at_ms: 0, type: "pen_down", x: 150, y: 400, pressure: 0.8, tilt_x: 20 }, { at_ms: 40, type: "pen_move", x: 200, y: 410, pressure: 0.8 },
{ at_ms: 80, type: "pen_move", x: 260, y: 420, pressure: 0.8 }, { at_ms: 120, type: "pen_up", x: 260, y: 420 }] }));
const journal = (await fixture.journal()).slice(since);
console.log(journal.map(entry => JSON.stringify(entry)).join("\n"));
expect(journal).toContainEqual(expect.objectContaining({ type: "pointerdown", pointerType: "mouse", shift: true }));
expect(journal.filter(entry => entry.type === "pointerdown" && entry.pointerType === "mouse").length).toBeGreaterThanOrEqual(4);
expect(journal).toContainEqual(expect.objectContaining({ type: "wheel", deltaY: 200 }));
expect(journal).toContainEqual(expect.objectContaining({ type: "pointermove", pointerType: "mouse", buttons: 1 }));
const held = journal.filter(entry => entry.key === "w");
expect(held.map(entry => entry.type)).toEqual(["keydown", "keyup"]);
expect(journal).toContainEqual(expect.objectContaining({ type: "pointerdown", pointerType: "pen", pressure: 0.8, tiltX: 20 }));
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
// LPT-928: the change at the input comes first, with an outlined overview, and a navigation shows the whole page.
test("a ticked tile is the first change of a page click and of a native click, and a navigation returns the whole page", async () => {
const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/checks` });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
const window = `computer:window/${opened.details.window_ref}`;
try {
const measured = await live.resources.browserAction(requireComputerAddress(window), live.cwd,
{ action: "evaluate", expression: "JSON.stringify(document.querySelectorAll('input')[5].getBoundingClientRect())" });
const box = JSON.parse(JSON.parse(/Evaluation result: (.*)$/m.exec(presentComputerResult(measured))![1])) as { x: number; y: number; width: number; height: number };
const page = await act(window, clickPlan({ window, x: Math.round(box.x + box.width / 2), y: Math.round(box.y + box.height / 2) }));
const pixels = page.result.details.successor?.diff?.pixels;
console.log(page.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
expect(pixels, page.body).toMatchObject({ space: "page", overview: true });
// The banner changed too and is larger; the ticked tile comes first because the click landed on it.
expect(pixels!.areas[0]).toMatchObject({ at_input: true });
expect(pixels!.areas[0].width).toBeLessThan(80);
expect(pixels!.regions).toBeGreaterThan(1);
expect(page.result.content.filter(block => block.type === "image").length).toBeGreaterThanOrEqual(3);
// A native click on a tile by ref of a window Read compares the window's pixels. Edge reports page elements to
// UI Automation (this Chrome does not), so its refs reach the tiles and its page start is known.
const edge = await live.insert(live.machine, { action: "open_browser", browser: "msedge", url: `${fixture.base}/checks` });
const edgeWindow = `computer:window/${edge.details.window_ref}`;
try {
// Edge builds the page's accessibility tree a moment after the page loads.
let full = await live.read(edgeWindow, ["full"]);
for (let i = 0; i < 20 && !full.details.ui?.elements.some(element => element.in_web_content); i++) {
await new Promise(resolve => setTimeout(resolve, 500));
full = await live.read(edgeWindow, ["full"]);
}
const tile = full.details.ui?.elements.find(element => element.role === "CheckBox" && element.name === "Tile 7")?.hint;
expect(tile, JSON.stringify(full.details.ui?.elements.filter(element => element.in_web_content).map(element => [element.role, element.name]).slice(0, 20))).toBeDefined();
const native = await act(edgeWindow, clickPlan({ window: edgeWindow, target: tile, space: "window" }));
console.log(native.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
expect(native.result.details.successor?.diff?.pixels?.areas[0], native.body).toMatchObject({ at_input: true });
} finally { await deleteComputer(live.resources, edgeWindow, { path: edgeWindow }, live.cwd); }
await live.read(window, ["dom"]);
const left = await act(window, clickPlan({ window, target: "getByRole('link', { name: 'Leave' })" }));
console.log(left.body.split("\n").filter(line => /Pixels|^ \d|Browser chrome|Most of/.test(line)).join("\n"));
expect(left.result.details.successor?.diff?.pixels, left.body).toMatchObject({ full: true });
expect(left.result.content.filter(block => block.type === "image")).toHaveLength(1);
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});
test("the same vocabulary drives a browser page through Playwright by locator and page ref", async () => {
const opened = await live.insert(live.machine, { action: "open_browser", url: `${fixture.base}/wait` });
expect(opened.details.status, JSON.stringify(opened.content)).toBe("observed");
const window = `computer:window/${opened.details.window_ref}`;
try {
expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada" }))).result.details.status).toBe("delivered");
// A native Read and a DOM Read of one window keep each other: the browser's own Reload button stays a usable ref.
const reload = /button "Reload" \[ref=([0-9a-z]+:[a-z]+)\]/.exec(presentComputerResult(await live.read(window, ["aria"])))![1];
expect((await live.read(window, ["dom"])).details.status).toBe("observed");
const reloaded = await act(window, clickPlan({ window, target: reload }));
expect(reloaded.result.details.status, reloaded.body).toBe("delivered");
expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Ada", replace: true }))).result.details.status).toBe("delivered");
const dom = presentComputerResult(await live.read(window, ["dom"]));
const save = /button "Save" \[ref=((?:f\d+)?e\d+)\]/.exec(dom)![1];
const clicked = await act(window, clickPlan({ window, target: save }));
expect(clicked.result.details.status, clicked.body).toBe("delivered");
expect(await holds(window, [{ target: { text: "saved Ada" }, is: "visible" }])).toBe("satisfied");
expect((await act(window, typePlan({ window, target: "getByLabel('Name')", text: "Grace", replace: true, submit: true }))).result.details.status).toBe("delivered");
expect((await act(window, pressPlan({ window, key: "ctrl+a" }))).result.details.status).toBe("delivered");
expect((await act(window, hoverPlan({ window, target: "getByRole('button', { name: 'Unlock later' })" }))).result.details.status).toBe("delivered");
expect((await act(window, scrollPlan({ window, delta_y: 2 }))).result.details.status).toBe("delivered");
const refused = await live.resources.input(requireComputerAddress(window), live.cwd, timelinePlan({ window, events: [{ at_ms: 0, type: "move_relative", dx: 5, dy: 0 }] })).catch((error: Error) => error.message);
expect(String(refused)).toContain("move_relative needs native input");
} finally { await deleteComputer(live.resources, window, { path: window }, live.cwd); }
});