Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

8.2 KB
Raw Blame
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { fileURLToPath } from "node:url";
import { loadComputerConfig, readMachineSettings, type ComputerConfig } from "./config.ts";
import { prepareRemoteRuntime } from "./remote-runtime.ts";
import { prepareRemoteBrowser } from "./remote-browser.ts";
import { RemotePreparationUncertain } from "./remote-provision-error.ts";
import { ComputerRequestError, machineGuide, type MachineFactory, type SandboxFactory } from "./computer-resources.ts";
import type { CuaVisionSettings } from "./cua-image-settings.ts";
import { McpDriver } from "./driver.ts";
import { MachineHost } from "./machine-host.ts";
import { localRunner, sshRunner, type CommandRunner } from "./runner.ts";
import { WindowsSandbox, runnerHost, sandboxOptions, waitForGuestDriver } from "./sandbox.ts";
import { IsolatedSession } from "./isolated-session.ts";
import { ComputerSession, type ComputerMode } from "./session.ts";
import { WebBrowser, WebError } from "./web-browser.ts";
import type { VisualSettings } from "./native-visuals.ts";
import type { HumanStop } from "./human-stop.ts";

/** The configured machine for a connect command; unknown names refuse before any SSH work. */
export async function machineConfig(cwd: string, name: string): Promise<NonNullable<ComputerConfig["machines"]>[string]> {
  const machine = (await readMachineSettings(cwd)).machines[name];
  if (!machine) throw new ComputerRequestError(`No machine named ${name} is configured in .pi/computer-use.json.`);
  return machine;
}

/** The worker files whose version decides the remote browser's behavior, relative to its package copy. */
const WORKER_FILES: [string, string][] = [
  ["src/web-worker.ts", fileURLToPath(new URL("./web-worker.ts", import.meta.url))],
  ["src/agent-profile-lock.ts", fileURLToPath(new URL("./agent-profile-lock.ts", import.meta.url))],
  ["src/browser-command-line.ts", fileURLToPath(new URL("./browser-command-line.ts", import.meta.url))],
  ["src/aria-match.ts", fileURLToPath(new URL("./aria-match.ts", import.meta.url))],
  ["src/web-geometry.ts", fileURLToPath(new URL("./web-geometry.ts", import.meta.url))],
  ["node_modules/playwright-core/package.json", createRequire(import.meta.url).resolve("playwright-core/package.json")],
];

/**
 * Refuse a remote browser before launch when that machine's worker copy differs from this package, so an old
 * worker never silently ignores newer options. `refresh` is the setup command that updates the copy.
 */
export async function checkRemoteWorker(runner: CommandRunner, remotePackage: string, refresh: string, signal?: AbortSignal): Promise<void> {
  const local = await Promise.all(WORKER_FILES.map(async ([, file]) => createHash("sha256").update(await readFile(file)).digest("hex")));
  // A missing file fails sha256sum: then every file counts as stale.
  const output = await runner.run("sha256sum", WORKER_FILES.map(([name]) => `${remotePackage}/${name}`), { signal, timeoutMs: 10_000 })
    .catch(() => "");
  const remote = output.split("\n").map(line => line.split(" ")[0]);
  const stale = WORKER_FILES.filter((_, index) => remote[index] !== local[index]).map(([name]) => name);
  if (stale.length) throw new ComputerRequestError(`The browser worker on this machine is missing or out of date (${stale.join(", ")}); nothing was launched. `
    + `Refresh it from this package with ${refresh}, then open the browser again.`);
}

/**
 * Each sandbox or isolated agent session gets its own guest driver and session: no host overlay, no browser launch, every guest window is the agent's own.
 * On another machine the sandbox host and its files are reached over SSH, and the guest relays through SSH tunnels.
 */
export function sandboxFactory(options: { settings: () => CuaVisionSettings; visuals?: VisualSettings }): SandboxFactory {
  return async (cwd, command, signal, machineName) => {
    let parsed;
    try {
      if (command.action === "open_session" && Object.keys(command).some(key => key !== "action")) throw new Error("open_session takes no options.");
      parsed = command.action === "open_session" ? undefined : sandboxOptions(command);
    } catch (error) { throw new ComputerRequestError(error instanceof Error ? error.message : String(error)); }
    const machine = machineName ? await machineConfig(cwd, machineName) : undefined;
    const config = machine ? undefined : await loadComputerConfig(cwd);
    const runner = machine ? sshRunner(machine.ssh) : localRunner(config!.interop);
    const commandPath = machine ? machine.command ?? await prepareRemoteRuntime(runner, signal) : config!.command;
    const runtime = machine ? machine.command ? machine.runtime : "windows-desktop" : config!.runtime;
    const sandbox = parsed ? await WindowsSandbox.open(runnerHost(runner), commandPath, parsed, signal)
      : await IsolatedSession.open(runnerHost(runner), commandPath, signal);
    // The endpoint selects the guest transport; ssh only makes the runtime hash check read the machine's own bytes.
    const override = machine ? machine.driverOverride : config!.driverOverride;
    const driver = new McpDriver({ command: commandPath, ...(runtime ? { runtime } : {}), ...(override ? { driverOverride: true as const } : {}),
      endpoint: sandbox.endpoint, ...(machine ? { ssh: machine.ssh } : {}) }, undefined, options.visuals);
    try { await waitForGuestDriver(driver, signal); }
    catch (error) { await driver.close().catch(() => undefined); await sandbox.close(); throw error; }
    const guest = new ComputerSession(driver, () => "yolo", options.settings);
    return { ref: sandbox.ref, session: guest, describe: source => sandbox.describe(source), ...(machineName ? { machine: machineName } : {}),
      close: async () => { try { await guest.close(); } finally { await sandbox.close(); } } };
  };
}

/**
 * Another machine's desktop gets its own driver and session over SSH. Safe/YOLO is the user's choice there too;
 * the host overlay is not shown, and browsers start on that machine.
 */
export function machineFactory(options: { getMode: () => ComputerMode; settings: () => CuaVisionSettings; visuals?: VisualSettings; humanStop?: HumanStop; sessionLabel?: () => string | undefined; escapeStop?: (cwd: string) => Promise<boolean | undefined> }): MachineFactory {
  return async (cwd, name, signal) => {
    const machine = await machineConfig(cwd, name);
    const escapeStop = options.escapeStop ? await options.escapeStop(cwd) : (await readMachineSettings(cwd)).escapeStop;
    const runner = sshRunner(machine.ssh);
    const command = machine.command ?? await prepareRemoteRuntime(runner, signal);
    const runtime = machine.command ? machine.runtime : "windows-desktop";
    const driver = new McpDriver({ command, ...(runtime ? { runtime } : {}), ...(machine.driverOverride ? { driverOverride: true as const } : {}), ...(escapeStop !== undefined ? { escapeStop } : {}), ssh: machine.ssh }, undefined, options.visuals, options.humanStop, options.sessionLabel);
    try { await waitForGuestDriver(driver, signal, 30_000); }
    catch (error) { await driver.close().catch(() => undefined); throw error; }
    const launchWeb = async (launch: Parameters<typeof WebBrowser.launch>[0], webSignal?: AbortSignal) => {
      const prepared = await prepareRemoteBrowser(runner, webSignal, { node: machine.node, package: machine.package }).catch(error => {
        if (error instanceof RemotePreparationUncertain) throw new WebError(error.message, false, true);
        throw error;
      });
      const node = prepared.node;
      const workerPackage = prepared.package;
      if (machine.package) await checkRemoteWorker(runner, workerPackage, "the explicit browser worker override", webSignal);
      return WebBrowser.launch({ ...launch, remote: { runner, node, worker: `${workerPackage}/src/web-worker.ts` } }, webSignal);
    };
    const remote = new ComputerSession(driver, options.getMode, options.settings, undefined, launchWeb, new MachineHost(runner));
    return { ref: name, session: remote,
      describe: source => [`# Machine ${name}`, `Address: ${source}`, `SSH: ${machine.ssh}`,
        "Browser: prepared automatically on first use", ...machineGuide(source)].join("\n"),
      close: () => remote.close() };
  };
}