Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

12.2 KB
Raw Blame
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { expect, test, vi } from "vitest";
import { computerSource } from "../src/computer-address.ts";
import { ComputerResources, requireComputerAddress, type OpenedSandbox } from "../src/computer-resources.ts";
import { ComputerSession, type Driver } from "../src/session.ts";
import { WindowsSandbox, sandboxConfig, sandboxOptions, type SandboxHost } from "../src/sandbox.ts";

const cwd = "/synthetic-sandbox-fixture";
const address = requireComputerAddress;

test("open_sandbox file channels default to read-only imports and one writable export", () => {
  expect(sandboxOptions({ action: "open_sandbox", import: ["C:\\data\\inputs", { host: "/mnt/c/repo", guest: "D:\\repo", writable: true }], export: "C:\\out" }).shares).toEqual([
    { host: "C:\\data\\inputs", guest: "C:\\pi\\import\\inputs", writable: false },
    { host: "/mnt/c/repo", guest: "D:\\repo", writable: true },
    { host: "C:\\out", guest: "C:\\pi\\export", writable: true },
  ]);
  expect(sandboxOptions({ action: "open_sandbox" }).shares).toEqual([]);
  expect(() => sandboxOptions({ action: "open_sandbox", clipboard: true })).toThrow(/only import and export/);
  expect(() => sandboxOptions({ action: "open_sandbox", import: [{ host: "C:\\a", guest: "C:\\pi\\runtime" }] })).toThrow(/outside C:\\pi/);
  expect(() => sandboxOptions({ action: "open_sandbox", import: ["C:\\a\\same", "D:\\b\\same"] })).toThrow(/same guest folder/);
});

test("the sandbox shares only the runtime, its own config, an output folder and requested folders", () => {
  const wsb = sandboxConfig("C:\\rt", "C:\\w\\config", "C:\\w\\out", [{ host: "C:\\in & out", guest: "C:\\pi\\import\\x", writable: false }]);
  expect(wsb).toContain("<ClipboardRedirection>Disable</ClipboardRedirection>");
  expect(wsb).toContain("<HostFolder>C:\\rt</HostFolder><SandboxFolder>C:\\pi\\runtime</SandboxFolder><ReadOnly>true</ReadOnly>");
  expect(wsb).toContain("<HostFolder>C:\\w\\out</HostFolder><SandboxFolder>C:\\pi\\out</SandboxFolder><ReadOnly>false</ReadOnly>");
  expect(wsb).toContain("<HostFolder>C:\\in &amp; out</HostFolder><SandboxFolder>C:\\pi\\import\\x</SandboxFolder><ReadOnly>true</ReadOnly>");
  expect(wsb.match(/<MappedFolder>/g)).toHaveLength(4);
});

/** A host whose sandbox launch stops at `failAt`; every script is recorded. */
async function fakeHost(failAt: "session" | "id" | "ready", running = false, failForward?: number) {
  const local = await mkdtemp(path.join(tmpdir(), "pi-sandbox-test-"));
  const scripts: string[] = [];
  const forwarded: number[] = [], closedForwards: number[] = [];
  const host: SandboxHost = {
    async path(value, to) {
      if (to === "windows") return value.startsWith("/mnt/c/") ? "C:\\" + value.slice(7).replaceAll("/", "\\") : "\\\\wsl.localhost\\Ubuntu" + value.replaceAll("/", "\\");
      if (value === "C:\\Local\\PiComputerUse\\sandboxes") return local;
      throw new Error(`unexpected path ${value}`);
    },
    async powershell(script) {
      scripts.push(script);
      if (script.includes("sandbox=")) return "sandbox=True;wsb=True;missing=;local=C:\\Local\n";
      if (script.includes("Get-Process vmmemWindowsSandbox") && script.includes("Select-Object -First 1")) {
        if (failAt === "id") throw new Error("Cancelled while starting the sandbox.");
        return "11111111-2222-3333-4444-555555555555\n";
      }
      if (script.includes("cmd.exe /c exit 0")) {
        if (failAt !== "ready") throw new Error("The sandbox user session did not become available within 120 s.");
        return "";
      }
      if (script.includes(".IpV4Address")) return "172.20.0.2\n";
      if (script.includes("Get-Process WindowsSandboxRemoteSession -ErrorAction SilentlyContinue) { 'running' }")) return running ? "running\n" : "";
      return "";
    },
    mkdir: async dir => { await mkdir(dir, { recursive: true }); },
    writeFile: (file, content) => writeFile(file, content),
    readFile: file => {
      const relay = /relay-(47800|47802)\.txt$/.exec(file);
      if (failAt === "ready" && relay) return Promise.resolve(relay[1] === "47800" ? "12001 4242" : "12003 4243");
      return readFile(file, "utf8").catch(() => "");
    },
    remove: dir => rm(dir, { recursive: true, force: true }),
    forward: async port => {
      forwarded.push(port);
      if (forwarded.length === failForward) throw new Error("Fixture forward failure");
      return { port, close: () => { closedForwards.push(port); } };
    },
  };
  return { host, scripts, local, forwarded, closedForwards };
}

test("Sandbox provisions a separate identity key and returns it only for the identity endpoint", async () => {
  const f = await fakeHost("ready");
  let sandbox: WindowsSandbox | undefined;
  try {
    sandbox = await WindowsSandbox.open(f.host, "/mnt/c/runtime/cua-driver.exe", { shares: [] });
    const config = JSON.parse(await readFile(path.join(f.local, sandbox.ref, "config/identity.json"), "utf8"));
    expect(config.session).toBe(sandbox.ref);
    expect(config.key).toMatch(/^[a-f0-9]{64}$/);
    expect(config.key).not.toBe(sandbox.endpoint?.token);
    expect(sandbox.endpoint?.identityAuth).toEqual(config);
    expect(sandbox.endpoint?.relay).toBeUndefined();
    expect(f.forwarded).toEqual([12001, 12003]);
  } finally { await sandbox?.close(); await rm(f.local, { recursive: true, force: true }); }
  expect(f.closedForwards).toEqual([12001, 12003]);
});

test("a failed identity forward releases the already opened driver tunnel", async () => {
  const f = await fakeHost("ready", false, 2);
  try {
    await expect(WindowsSandbox.open(f.host, "/mnt/c/runtime/cua-driver.exe", { shares: [] })).rejects.toBeInstanceOf(Error);
    expect(f.closedForwards).toEqual([12001]);
    expect(await readdir(f.local)).toEqual([]);
  } finally { await rm(f.local, { recursive: true, force: true }); }
});

test("a sandbox that fails after launch is stopped and its host folder removed", async () => {
  const { host, scripts, local } = await fakeHost("session");
  await expect(WindowsSandbox.open(host, "/mnt/c/runtime/cua-driver.exe", sandboxOptions({ action: "open_sandbox" }))).rejects.toThrow(/user session/);
  const close = scripts.at(-1)!;
  expect(close).toContain("stop --id 11111111-2222-3333-4444-555555555555");
  expect(await readdir(local)).toEqual([]);
});

test("a launch cancelled before its id is known stops its sandbox and the client it started", async () => {
  const { host, scripts, local } = await fakeHost("id");
  await expect(WindowsSandbox.open(host, "/mnt/c/runtime/cua-driver.exe", sandboxOptions({ action: "open_sandbox" }))).rejects.toThrow(/Cancelled/);
  const close = scripts.at(-1)!;
  expect(close).toContain("foreach ($id in $ids)");
  expect(close).toMatch(/WindowsSandboxRemoteSession .*StartTime -ge .*Stop-Process -Force/);
  expect(await readdir(local)).toEqual([]);
});

// Windows runs one sandbox at a time; a second launch only showed an error dialog and timed out after 120 s (LPT-838).
test("another running sandbox refuses the launch at once and starts nothing", async () => {
  const { host, scripts, local } = await fakeHost("id", true);
  await expect(WindowsSandbox.open(host, "/mnt/c/runtime/cua-driver.exe", sandboxOptions({ action: "open_sandbox" })))
    .rejects.toThrow(/already running on this machine, and Windows allows only one.*Nothing was started/);
  expect(scripts.some(script => script.includes("explorer.exe"))).toBe(false);
  expect(scripts.filter(script => !script.includes("'running'")).some(script => script.includes("Stop-Process"))).toBe(false);
  expect(await readdir(local)).toEqual([]);
});

test("runtime folders outside a Windows drive are refused before any sandbox starts", async () => {
  const { host, scripts } = await fakeHost("session");
  await expect(WindowsSandbox.open(host, "/home/user/runtime/cua-driver.exe", sandboxOptions({ action: "open_sandbox" }))).rejects.toThrow(/Windows drive/);
  expect(scripts.some(script => script.includes("explorer.exe"))).toBe(false);
});

/** A fake native backend with one window; `domain` marks a sandbox guest. */
function backend(pid: number, domain?: "windows-sandbox") {
  const driver: Driver = {
    ...(domain ? { sourceDomain: domain } : {}),
    identity: vi.fn(async () => `life-${pid}`), close: vi.fn(async () => {}),
    call: vi.fn<Driver["call"]>(async (name, args) => {
      if (name === "list_windows") return { content: [], structuredContent: { windows: [{ pid, window_id: 7, title: `window ${pid}` }] } };
      // No pixels here: a capture that asks for them reports why, and the successor falls back to the UI.
      if (name === "get_window_state") return { content: [], structuredContent: { pid, window_id: 7, snapshot_id: "s", elements_complete: true, elements: [],
        ...((args as { include_screenshot?: boolean }).include_screenshot !== false ? { screenshot_error: "No surface." } : {}),
        window_bounds: { x: 0, y: 0, width: 100, height: 80 } } };
      if (name === "type_text") return { content: [], structuredContent: { effect: "confirmed" } };
      throw new Error(`Unexpected native call ${name}`);
    }),
  };
  return driver;
}

async function routing() {
  const hostDriver = backend(10), guestDriver = backend(20, "windows-sandbox");
  const host = new ComputerSession(hostDriver, () => "yolo"), guest = new ComputerSession(guestDriver, () => "yolo");
  const close = vi.fn(async () => {});
  const sandbox: OpenedSandbox = { ref: "sb-test", session: guest, describe: source => `# Windows Sandbox\nAddress: ${source}`, close };
  const resources = new ComputerResources(async () => host, async () => host.close(), async () => sandbox);
  const opened = await resources.perform(address("computer:machine/local"), cwd, JSON.stringify({ action: "open_sandbox" }));
  return { hostDriver, guestDriver, resources, opened, close, sandboxSource: computerSource("sandbox", "sb-test") };
}

test("guest windows are found, read and operated only through the sandbox's own driver", async () => {
  const f = await routing();
  expect(f.opened.details).toMatchObject({ status: "delivered", sandbox_ref: "sb-test" });
  const listed = await f.resources.search(address(f.sandboxSource), cwd, "window");
  const row = (listed.details.state!.windows as { window_ref: string; source_identity: { domain: string } }[])[0];
  expect(row.source_identity.domain).toBe("windows-sandbox");
  const view = await f.resources.read(address(computerSource("window", row.window_ref)), cwd, ["ui"]);
  expect(view?.details.status).toBe("observed");
  await f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "hello" }));
  const names = (driver: Driver) => vi.mocked(driver.call).mock.calls.map(([name]) => name);
  // The Write's successor walks the window again; repeated walks are collapsed here.
  expect(names(f.guestDriver).filter((name, i, all) => name !== "get_window_state" || all[i - 1] !== "get_window_state")).toEqual(["list_windows", "get_window_state", "type_text", "get_window_state"]);
  expect(names(f.hostDriver)).toEqual([]);
  expect(f.resources.descriptor(address(f.sandboxSource))).toContain(f.sandboxSource);
});

test("destroying a sandbox retires its address and guest references", async () => {
  const f = await routing();
  const listed = await f.resources.search(address(f.sandboxSource), cwd, "window");
  const ref = (listed.details.state!.windows as { window_ref: string }[])[0].window_ref;
  const view = await f.resources.read(address(computerSource("window", ref)), cwd, ["ui"]);
  const closed = await f.resources.delete(f.sandboxSource, cwd);
  expect(closed.details).toMatchObject({ status: "closed", sandbox_ref: "sb-test" });
  expect(f.close).toHaveBeenCalledOnce();
  expect(() => f.resources.descriptor(address(f.sandboxSource))).toThrow(/expired/);
  await expect(f.resources.perform(address(computerSource("observation", view!.details.observation_id!)), cwd, JSON.stringify({ action: "type_text", text: "late" }))).rejects.toThrow(/expired/);
});

test("closing the computer session destroys its sandboxes", async () => {
  const f = await routing();
  await f.resources.delete("computer:session", cwd);
  expect(f.close).toHaveBeenCalledOnce();
  expect(() => f.resources.descriptor(address(f.sandboxSource))).toThrow(/expired/);
});