Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

20.8 KB
Raw Blame
import { execFileSync } from "node:child_process";
import sharp from "sharp";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { expect, test, vi } from "vitest";
import { computerSource, displaySource } from "../src/computer-address.ts";
import { ComputerRequestError, ComputerResources, requireComputerAddress, type OpenedMachine, type OpenedSandbox } from "../src/computer-resources.ts";
import { loadComputerConfig } from "../src/config.ts";
import { checkRemoteWorker } from "../src/machine.ts";
import { presentComputerResult } from "../src/provider.ts";
import { remoteCommand, type CommandRunner } from "../src/runner.ts";
import { ComputerSession, type Driver } from "../src/session.ts";

const cwd = "/synthetic-remote-fixture";
const address = requireComputerAddress;

test("a remote command keeps every argument and variable exactly, whatever the remote login shell", () => {
  const args = ["it's", "a \"b\" $HOME `x` $(id)", "ü ✓", ""];
  const command = remoteCommand("/usr/bin/printf", ["%s|", ...args], { NAME: "a b'c" });
  // SSH hands the command to the login shell as one string; bash and fish both run it.
  expect(execFileSync("bash", ["-c", command]).toString()).toBe(args.map(arg => `${arg}|`).join(""));
  expect(execFileSync("bash", ["-c", remoteCommand("/usr/bin/env", [], { NAME: "a b'c" })]).toString()).toContain("NAME=a b'c\n");
  expect(() => remoteCommand("/usr/bin/env", [], { "BAD-NAME": "x" })).toThrow(/Invalid environment variable/);
});

test("remote sandbox selects its machine without resolving the local runtime", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "pi-remote-sandbox-"));
  const config = await import("../src/config.ts");
  const { sandboxFactory } = await import("../src/machine.ts");
  const { WindowsSandbox } = await import("../src/sandbox.ts");
  const failure = new Error("sandbox reached");
  const local = vi.spyOn(config, "loadComputerConfig").mockRejectedValue(new Error("local runtime unavailable"));
  const open = vi.spyOn(WindowsSandbox, "open").mockRejectedValue(failure);
  try {
    await mkdir(path.join(dir, ".pi"));
    await writeFile(path.join(dir, ".pi/computer-use.json"), JSON.stringify({ machines: { lab: { ssh: "lab-wsl", command: "/mnt/c/driver.exe" } } }));
    const factory = sandboxFactory({ settings: () => ({ kernel: "lanczos3", maxLongEdge: 1280, scale: 1, version: 0, route: "cua_native" }) });
    await expect(factory(dir, { action: "open_sandbox" }, undefined, "lab")).rejects.toBe(failure);
    expect(local).not.toHaveBeenCalled();
    expect(open).toHaveBeenCalledOnce();
  } finally { local.mockRestore(); open.mockRestore(); await rm(dir, { recursive: true, force: true }); }
});
test("browserOverride preserves a single legacy browser path", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "pi-machines-partial-"));
  try {
    await mkdir(path.join(dir, ".pi"));
    const node = "/mnt/c/Users/test/AppData/Local/Programs/PiComputerUse/node-v22.22.2-win-x64/node.exe";
    const worker = "/home/test/.local/share/pi-computer-use/package";
    await writeFile(path.join(dir, ".pi/computer-use.json"), JSON.stringify({ command: "/x/cua-driver.exe", machines: {
      nodeonly: { ssh: "node-alias", node, browserOverride: true },
      workeronly: { ssh: "worker-alias", package: worker, browserOverride: true },
    } }));
    const machines = (await loadComputerConfig(dir)).machines!;
    expect(machines.nodeonly.node).toBe(node);
    expect(machines.workeronly.package).toBe(worker);
  } finally { await rm(dir, { recursive: true, force: true }); }
});
test("a remote machine needs only its trusted SSH alias", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "pi-machines-auto-"));
  try {
    await mkdir(path.join(dir, ".pi"));
    await writeFile(path.join(dir, ".pi/computer-use.json"), JSON.stringify({ command: "/x/cua-driver.exe", machines: { lab: { ssh: "lab-wsl" } } }));
    expect((await loadComputerConfig(dir)).machines?.lab).toEqual({ ssh: "lab-wsl" });
  } finally { await rm(dir, { recursive: true, force: true }); }
});
test("legacy generated remote paths follow the package, but explicit overrides remain available", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "pi-machines-legacy-"));
  try {
    await mkdir(path.join(dir, ".pi"));
    const legacy = { ssh: "lab-wsl", command: "/mnt/c/Users/u/AppData/Local/Programs/PiComputerUse/cua-driver-0.37.0-33c4b82b/runtime/cua-driver.exe", runtime: "windows-desktop",
      node: "/mnt/c/Users/u/AppData/Local/Programs/PiComputerUse/node-v22.22.2-win-x64/node.exe", package: "/home/u/.local/share/pi-computer-use/package" };
    const write = async (machine: object) => writeFile(path.join(dir, ".pi/computer-use.json"), JSON.stringify({ command: "/x/cua-driver.exe", machines: { lab: machine } }));
    await write(legacy);
    expect((await loadComputerConfig(dir)).machines?.lab).toEqual({ ssh: "lab-wsl" });
    await write({ ...legacy, driverOverride: true, browserOverride: true });
    expect((await loadComputerConfig(dir)).machines?.lab).toEqual({ ...legacy, driverOverride: true, browserOverride: true });
  } finally { await rm(dir, { recursive: true, force: true }); }
});
test("machines come only from validated configuration", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "pi-machines-"));
  await mkdir(path.join(dir, ".pi"));
  const write = (machines: unknown) => writeFile(path.join(dir, ".pi/computer-use.json"), JSON.stringify({ command: "/x/cua-driver.exe", machines }));
  await write({ lab: { ssh: "lab-wsl", command: "/mnt/c/run/cua-driver.exe", runtime: "windows-desktop", node: "/mnt/c/node.exe", package: "/home/u/pkg" } });
  expect((await loadComputerConfig(dir)).machines?.lab).toEqual({ ssh: "lab-wsl", command: "/mnt/c/run/cua-driver.exe", runtime: "windows-desktop", node: "/mnt/c/node.exe", package: "/home/u/pkg" });
  for (const bad of [{ "Bad Name": { ssh: "a", command: "/x" } }, { local: { ssh: "a", command: "/x" } }, { pc: { ssh: "a", command: "relative/cua.exe" } }, { pc: { ssh: "a", command: "/x", extra: 1 } }, { pc: { ssh: "a", command: "/x", runtime: "unknown" } }, []]) {
    await write(bad);
    await expect(loadComputerConfig(dir)).rejects.toThrow(/Invalid machines/);
  }
});

const png = (await sharp({ create: { width: 40, height: 30, channels: 3, background: "red" } }).png().toBuffer()).toString("base64");
const image = { type: "image" as const, mimeType: "image/png", data: png };
/** One display's pixels as the native runtime reports them: 1920x1080 scaled to 40x30 here. */
const pixels = { platform: "windows", screenshot_width: 40, screenshot_height: 30, screenshot_original_width: 1920, screenshot_original_height: 1080 };

/** A fake native backend with one window and one display. */
function backend(pid: number) {
  const display = { display_id: `D${pid}`, primary: true, monitor_handle: `0x${pid}`, monitor_device_paths: [`\\\\?\\DISPLAY#${pid}`],
    bounds: { x: 0, y: 0, width: 1920, height: 1080 }, work_area: { x: 0, y: 0, width: 1920, height: 1040 }, scale_factor: 1, dpi: 96 };
  const driver: Driver = {
    identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}), selectedDisplays: true,
    call: vi.fn<Driver["call"]>(async (name, args) => {
      if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };
      if (name === "list_displays") return { content: [], structuredContent: { platform: "windows", topology_id: `t${pid}`, displays: [display] } };
      if (name === "get_desktop_state") return { content: [image], structuredContent: { ...pixels, display: "primary" } };
      if (name === "get_display_state") return { content: [image], structuredContent: { ...pixels, display, topology_id: `t${pid}` } };
      // A sequence frame: pixels without the UI tree.
      if (name === "get_window_state" && (args as { include_accessibility_tree?: boolean }).include_accessibility_tree === false)
        return { content: [image], structuredContent: { pid, window_id: 7, window_bounds: { x: 0, y: 0, width: 40, height: 30 },
          screenshot_width: 40, screenshot_height: 30, screenshot_original_width: 40, screenshot_original_height: 30 } };
      // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.
      if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],
        ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),
        window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };
      if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };
      if (name === "list_apps") return { content: [], structuredContent: { apps: [{ name: `App ${pid}`, launch_path: `C:\\app${pid}.exe` }] } };
      if (name === "launch_app") throw new Error("launch stopped by the fixture");
      throw new Error(`Unexpected native call ${name}`);
    }),
  };
  return driver;
}

async function fixture() {
  const hostDriver = backend(10), remoteDriver = backend(20), guestDriver = backend(30);
  const host = new ComputerSession(hostDriver, () => "yolo");
  const remote = new ComputerSession(remoteDriver, () => "yolo"), guest = new ComputerSession(guestDriver, () => "yolo");
  const machineClose = vi.fn(() => remote.close()), sandboxClose = vi.fn(async () => {});
  const machine: OpenedMachine = { ref: "lab", session: remote, describe: source => `# Remote machine\nAddress: ${source}`, close: machineClose };
  const openSandbox = vi.fn(async (_cwd: string, _command: Record<string, unknown>, _signal?: AbortSignal, onMachine?: string): Promise<OpenedSandbox> =>
    ({ ref: "sb-1", session: guest, describe: source => source, close: sandboxClose, ...(onMachine ? { machine: onMachine } : {}) }));
  const start = vi.fn(async (_cwd: string, name: string) => {
    if (name !== "lab") throw new ComputerRequestError(`No machine named ${name} is configured in .pi/computer-use.json.`);
    return machine;
  });
  const resources = new ComputerResources(async () => host, async () => host.close(), openSandbox, start);
  const names = (driver: Driver) => vi.mocked(driver.call).mock.calls.map(([name]) => name);
  return { resources, remote, hostDriver, remoteDriver, openSandbox, start, machineClose, sandboxClose, names, machineSource: computerSource("machine", "lab") };
}

test("Safe remains blocked while a remote YOLO browser is closing", async () => {
  const f = await fixture();
  await f.resources.search(address(`${f.machineSource}/windows`), cwd, "");
  let release!: () => void;
  const waiting = new Promise<void>(resolve => { release = resolve; });
  const unsafe = vi.spyOn(f.remote, "hasUnapprovedAgentBrowser").mockReturnValue(true);
  f.machineClose.mockImplementationOnce(async () => { await waiting; unsafe.mockReturnValue(false); return f.remote.close(); });
  const closing = f.resources.delete(f.machineSource, cwd);
  await vi.waitFor(() => expect(f.machineClose).toHaveBeenCalled());
  expect(f.resources.hasUnapprovedAgentBrowser()).toBe(true);
  release(); await closing;
  expect(f.resources.hasUnapprovedAgentBrowser()).toBe(false);
});
test("a machine starts on first use; its windows, apps and displays come only from its own driver", async () => {
  const f = await fixture();
  const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
  const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
  await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "hello" }));
  const displays = await f.resources.read(address(`${f.machineSource}/displays`), cwd);
  expect((displays!.details.state!.displays as { display_id: string }[]).map(display => display.display_id)).toEqual(["D20"]);
  // An app address opens the app on the machine that listed it.
  const apps = await f.resources.search(address(`${f.machineSource}/apps`), cwd, "App");
  const app = (apps.details.state!.apps as { app_ref: string }[])[0].app_ref;
  await f.resources.perform(address(computerSource("app", app)), cwd, JSON.stringify({ action: "open" }));
  // The Write's successor walks the window again; repeated walks are collapsed here.
  expect(f.names(f.remoteDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state", "list_displays", "list_apps", "list_apps", "list_windows", "launch_app"]);
  expect(f.names(f.hostDriver)).toEqual([]);
  // Every use after the first reuses the started machine.
  expect(f.start).toHaveBeenCalledOnce();
});

test("an unknown machine refuses, and closing a machine releases it, its references and its sandboxes", async () => {
  const f = await fixture();
  await expect(f.resources.search(address("computer:machine/other/windows"), cwd, "")).rejects.toThrow(ComputerRequestError);
  const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
  const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
  const sandbox = await f.resources.perform(address(f.machineSource), cwd, JSON.stringify({ action: "open_sandbox" }));
  expect(sandbox.details.sandbox_ref).toBe("sb-1");
  expect(f.openSandbox.mock.calls[0][3]).toBe("lab");
  const closed = await f.resources.delete(f.machineSource, cwd);
  // The report names each released part.
  expect(closed.details.status).toBe("closed");
  expect(closed.content[0]).toMatchObject({ text: expect.stringMatching(/computer:sandbox\/sb-1: Sandbox[^\n]*destroyed[\s\S]*computer:machine\/lab: /) });
  expect(f.sandboxClose).toHaveBeenCalledOnce();
  expect(f.machineClose).toHaveBeenCalledOnce();
  expect(() => f.resources.descriptor(address(computerSource("sandbox", "sb-1")))).toThrow(/expired/);
  await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "late" }))).rejects.toThrow(/expired/);
  // A stale remote window address never reaches this host's desktop.
  await expect(f.resources.read(address(computerSource("window", ref)), cwd, ["ui"])).rejects.toThrow(/Unknown or closed window reference[\s\S]*nothing was sent/);
  expect(f.names(f.hostDriver)).toEqual([]);
});

test("closing the computer session releases every machine, the local one included, and reports each", async () => {
  const f = await fixture();
  await f.resources.search(address(`${f.machineSource}/windows`), cwd, "");
  await f.resources.search(address("computer:machine/local/windows"), cwd, "");
  const closed = await f.resources.delete("computer:session", cwd);
  expect(f.machineClose).toHaveBeenCalledOnce();
  expect(closed.details.status).toBe("closed");
  // What the model sees: a session's own close result carries a JSON projection that must not hide later parts.
  const lines = presentComputerResult(closed);
  expect(lines).toMatch(/computer:machine\/lab: /);
  expect(lines).toMatch(/computer:machine\/local: /);
  // A closed machine starts again on its next use.
  await f.resources.search(address(`${f.machineSource}/windows`), cwd, "");
  expect(f.start).toHaveBeenCalledTimes(2);
});

test("removed address forms refuse", () => {
  for (const source of ["computer:windows", "computer:apps", "computer:displays", "computer:machine/lab/display", "computer:machine/lab/display/a/b"]) expect(() => address(source)).toThrow();
  // A display without its machine names the form to use instead.
  expect(() => address("computer:display/primary")).toThrow(/computer:machine\/<name>\/display\/primary/);
});

// LPT-929, LPT-924: every address reaches the machine that issued it, and every receipt names that machine.
test("window sequences and their frames run on the machine that captured them", async () => {
  const f = await fixture();
  const listed = await f.resources.search(address(`${f.machineSource}/windows`), cwd, "window");
  expect(listed.details.machine).toBe("lab");
  const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const captured = await f.resources.sequence(address(computerSource("window", ref)), cwd, ['sequence:{"duration_ms":1}']);
  expect(captured?.details).toMatchObject({ status: "observed", machine: "lab" });
  const sequence = address(computerSource("sequence", captured!.details.sequence!.sequence_id));
  const frames = await f.resources.read(sequence, cwd, ["frames:endpoints"]);
  expect(frames?.details).toMatchObject({ status: "observed", machine: "lab" });
  expect(frames!.content.filter(block => block.type === "image")).toHaveLength(1);
  expect(presentComputerResult(frames!)).toMatch(/^Computer observed \(yolo\) on lab\n/);
  expect(f.names(f.hostDriver)).toEqual([]);
});

test("a machine's primary and listed displays are read and captured on that machine only", async () => {
  const f = await fixture();
  const primary = address(displaySource("lab", "primary"));
  const overview = await f.resources.read(primary, cwd);
  expect(overview?.details).toMatchObject({ status: "observed", machine: "lab" });
  // The display observation keeps its machine.
  expect(f.resources.descriptor(address(computerSource("observation", overview!.details.agent_image!.image_id as string)))).toContain(`Source: ${displaySource("lab", "primary")}`);
  const sequence = await f.resources.sequence(primary, cwd, ['sequence:{"duration_ms":1}']);
  expect(sequence?.details).toMatchObject({ status: "observed", machine: "lab" });
  const listed = await f.resources.read(address(`${f.machineSource}/displays`), cwd);
  const ref = (listed!.details.state!.displays as { display_ref: string }[])[0].display_ref;
  // The listing shows each display with its machine's address.
  expect(presentComputerResult(listed!)).toContain(displaySource("lab", ref));
  const selected = await f.resources.sequence(address(displaySource("lab", ref)), cwd, ['sequence:{"duration_ms":1}']);
  expect(selected?.details).toMatchObject({ status: "observed", machine: "lab" });
  const frames = await f.resources.read(address(computerSource("sequence", selected!.details.sequence!.sequence_id)), cwd, ["frames:last"]);
  expect(frames?.details.status).toBe("observed");
  // A display reference of one machine is unknown on another.
  const elsewhere = await f.resources.read(address(displaySource("local", ref)), cwd);
  expect(elsewhere?.details).toMatchObject({ status: "refused", machine: "local" });
  expect(presentComputerResult(elsewhere!)).toMatch(/Unknown or unavailable display reference/);
  expect(f.names(f.remoteDriver)).toEqual(["get_desktop_state", "get_desktop_state", "list_displays", "get_display_state"]);
  expect(f.names(f.hostDriver)).toEqual([]);
});

test("an address no machine issued is refused before anything is sent", async () => {
  const f = await fixture();
  await f.resources.search(address(`${f.machineSource}/windows`), cwd, "");
  await expect(f.resources.read(address(computerSource("window", "w-unknown")), cwd, ["ui"])).rejects.toThrow(/Unknown or closed window reference/);
  await expect(f.resources.read(address(computerSource("sequence", "seq-unknown")), cwd, ["frames:last"])).rejects.toThrow(/Sequence is unavailable/);
  await expect(f.resources.read(address(displaySource("other", "primary")), cwd)).rejects.toThrow(/No machine named other/);
  expect(f.names(f.hostDriver)).toEqual([]);
  expect(f.names(f.remoteDriver)).toEqual(["list_windows"]);
});

test("a remote browser refuses before launch when that machine's worker copy is stale or missing", async () => {
  const local = execFileSync("sha256sum", ["src/web-worker.ts", "src/agent-profile-lock.ts", "src/browser-command-line.ts", "src/aria-match.ts", "src/web-geometry.ts", require.resolve("playwright-core/package.json")], { encoding: "utf8" });
  const runner = (output: string | Error): CommandRunner => ({ spawn: vi.fn(), run: vi.fn(async () => {
    if (output instanceof Error) throw output;
    return output;
  }) });
  const current = runner(local);
  await checkRemoteWorker(current, "/pkg", "setup part machine");
  expect(current.run).toHaveBeenCalledWith("sha256sum", ["/pkg/src/web-worker.ts", "/pkg/src/agent-profile-lock.ts", "/pkg/src/browser-command-line.ts", "/pkg/src/aria-match.ts", "/pkg/src/web-geometry.ts", "/pkg/node_modules/playwright-core/package.json"], expect.anything());
  const stale = local.replace(/^[0-9a-f]{64}/, "0".repeat(64));
  await expect(checkRemoteWorker(runner(stale), "/pkg", "setup part machine"))
    .rejects.toThrow(/out of date \(src\/web-worker\.ts\); nothing was launched\. Refresh it from this package with setup part machine/);
  await expect(checkRemoteWorker(runner(new Error("No such file")), "/pkg", "setup"))
    .rejects.toBeInstanceOf(ComputerRequestError);
});