Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import path from "node:path";
import { assistantMessage, getToolExecution, getToolResultText, PiIntegrationTest, testArtifactsDir, text, toolCall } from "pi-coding-agent-test";
import { expect, test } from "vitest";
import { IDE_EXTENSION, PI_COMMAND } from "./fixtures/pi-agent-ide.ts";
const root = path.resolve(import.meta.dirname, "..");
const fixture = (name: string) => path.join(root, "test/fixtures", name);
// Pattern: real-Pi extensions example and the ordinary IDE computer-provider scenarios.
// Contract: computer_scroll at a point is real input that preserves capture authority and structured uncertain prefixes; IDE Insert refuses.
async function scenario(testName: string, scenario: "receipts" | "authority", providerFirst: boolean) {
const consumer = IDE_EXTENSION, provider = fixture("coordinate-scroll-extension.ts");
await mkdir(path.join(root, ".tmp/lpt-730"), { recursive: true });
const cwd = await mkdtemp(path.join(root, ".tmp/lpt-730/pi-scroll-")), log = path.join(cwd, "calls.jsonl");
try {
await mkdir(path.join(cwd, ".pi"));
await writeFile(path.join(cwd, ".pi/computer-use.json"), JSON.stringify({ command: process.execPath,
args: [fixture("coordinate-scroll-driver.mjs"), log] }));
const result = await new PiIntegrationTest({ testName: `${testName}-${providerFirst ? "provider-first" : "core-first"}`,
artifactsDir: testArtifactsDir(import.meta.filename), cwd, rawMode: false,
extensions: providerFirst ? [provider, consumer] : [consumer, provider],
isolateUserResources: true, piCommand: PI_COMMAND, timeoutMs: 90_000,
tools: ["read", "search", "insert", "delete", "verify_coordinate_scroll"],
conversation: [assistantMessage([toolCall({ id: "scroll", name: "verify_coordinate_scroll", arguments: { scenario } })], { stopReason: "toolUse" }),
assistantMessage([text("Finished the synthetic coordinate-scroll fixture.")])],
}).run("Exercise ordinary IDE coordinate scroll against synthetic MCP windows only. Never operate a real desktop or replay unknown input.");
expect(getToolExecution(result, "scroll").isError, getToolResultText(result, "scroll")).toBe(false);
expect(getToolResultText(result, "scroll")).toContain(`Scroll ${scenario} contract passed`);
expect((getToolExecution(result, "scroll").result as { content?: { type: string }[] }).content?.some(block => block.type === "image")).toBe(true);
expect(result.tuiRenderedOutput).not.toContain("Failed to load extension");
expect(result.tuiRenderedOutput).not.toContain("already handles");
return (await readFile(log, "utf8")).trim().split("\n").map(line => JSON.parse(line));
} finally { await rm(cwd, { recursive: true, force: true }); }
}
// Missing native axis fields and explicit zero both mean the same contract default.
const inputCalls = (calls: { name: string; arguments: Record<string, unknown> }[]): { name: string; arguments: Record<string, unknown> }[] => calls.filter(call =>
!["list_apps", "list_windows", "launch_app", "fixture_identity", "get_window_state", "zoom", "fixture_control"].includes(call.name)).map(call => ({
...call, arguments: { ...call.arguments, delta_x: call.arguments.delta_x ?? 0, delta_y: call.arguments.delta_y ?? 0 },
}));
const native = (x: number, y: number, delta_x: number, delta_y: number, delivery_mode = "foreground") => ({
pid: 42, window_id: 91, capture_id: expect.any(String), x, y, delta_x, delta_y, unit: "wheel_detents", delivery_mode,
});
test.each([false, true])("ordinary IDE scroll preserves capture-bound deltas and uncertain prefixes (providerFirst=%s)", async providerFirst => {
const actions = inputCalls(await scenario("coordinate-scroll-receipts", "receipts", providerFirst));
// Every scroll at a point is real input in the foreground.
expect(actions.map(call => call.name)).toEqual(Array(7).fill("scroll_at"));
expect(actions.map(call => call.arguments)).toEqual([
native(100, 80, -2, 3), native(120, 90, 1, 0), ...Array.from({ length: 5 }, () => native(130, 100, 1, 1)),
]);
// Every scroll binds to its own capture.
expect(new Set(actions.map(call => call.arguments.capture_id)).size).toBe(actions.length);
}, 120_000);
test.each([false, true])("ordinary IDE scroll keeps image, detail, capability, Safe and native lifetime boundaries (providerFirst=%s)", async providerFirst => {
expect(inputCalls(await scenario("coordinate-scroll-authority", "authority", providerFirst))).toEqual([
{ name: "scroll_at", arguments: native(100, 80, 0, 2) },
]);
}, 120_000);