Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { spawn } from "node:child_process";
import { randomBytes, randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import { connect, createServer, type AddressInfo } from "node:net";
import path from "node:path";
import type { ComputerConfig } from "./config.ts";
import { ComputerRequestError } from "./computer-resources.ts";
import { SSH_OPTIONS, powershellOf, runnerFiles, type CommandRunner } from "./runner.ts";
import { startSessionRelay } from "./session-relay-host.ts";
import type { SessionRelay, SessionRelayKeys } from "./session-relay.ts";
import type { RelayCredentials } from "./relay-auth.ts";
/** Host-side commands for the sandbox lifecycle; replaced by a fake in tests. */
export interface SandboxHost {
/** Run one Windows PowerShell script on the host and return its stdout. */
powershell(script: string, signal?: AbortSignal, timeoutMs?: number): Promise<string>;
/** Convert a WSL path to a Windows path, or the reverse with `unix`. */
path(value: string, to: "windows" | "unix"): Promise<string>;
/** Files in the host's WSL. readFile returns "" for a missing file. */
mkdir(path: string): Promise<void>;
writeFile(path: string, content: string): Promise<void>;
readFile(path: string): Promise<string>;
remove(path: string): Promise<void>;
/** Reach a port on the host's loopback from this machine: the same port locally, an SSH tunnel for another machine. */
forward(port: number, signal?: AbortSignal): Promise<{ port: number; close(): void }>;
/** Host-owned Node reverse relay; only needed for MXC, not Sandbox. */
reverseRelay?(workdir: string, keys: SessionRelayKeys, signal?: AbortSignal): Promise<SessionRelay>;
}
/** One host folder shared with the guest. Import is read-only unless requested; export is writable. */
export interface SandboxShare { host: string; guest: string; writable: boolean }
export interface SandboxOptions { shares: SandboxShare[] }
/** A host relay port that reaches one guest port, with the relay process to stop. */
interface Relay { port: number; pid: number }
const GUEST_ROOT = "C:\\pi";
const DRIVER_GUEST_PORT = 47800;
const IDENTITY_GUEST_PORT = 47802;
const SCRIPTS = new URL("../scripts/sandbox/", import.meta.url);
const COMMON_SCRIPTS = new URL("../scripts/", import.meta.url);
const RELAY_AUTH_SCRIPT = new URL("../scripts/session/relay-auth.ps1", import.meta.url);
const psString = (value: string) => `'${value.replace(/'/g, "''")}'`;
const xml = (value: string) => value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
/** Validate open_sandbox's explicit file channels before any host work. */
export function sandboxOptions(command: Record<string, unknown>): SandboxOptions {
const allowed = ["action", "import", "export"];
if (Object.keys(command).some(key => !allowed.includes(key))) throw new Error("open_sandbox accepts only import and export.");
const shares: SandboxShare[] = [];
const imports = command.import ?? [];
if (!Array.isArray(imports) || imports.length > 8) throw new Error("import is a list of up to 8 host folders.");
for (const entry of imports) {
const item = typeof entry === "string" ? { host: entry } : entry;
if (!item || typeof item !== "object" || Array.isArray(item)) throw new Error("Each import is a host folder path or {host, guest?, writable?}.");
const { host, guest, writable } = item as Record<string, unknown>;
if (Object.keys(item).some(key => !["host", "guest", "writable"].includes(key)) || typeof host !== "string" || !host
|| (guest !== undefined && typeof guest !== "string") || (writable !== undefined && typeof writable !== "boolean"))
throw new Error("Each import is a host folder path or {host, guest?, writable?}.");
shares.push({ host, guest: guest as string | undefined ?? `${GUEST_ROOT}\\import\\${path.win32.basename(host.replace(/[\\/]+$/, "")) || "folder"}`, writable: writable === true });
}
if (command.export !== undefined) {
if (typeof command.export !== "string" || !command.export) throw new Error("export is one host folder path.");
shares.push({ host: command.export, guest: `${GUEST_ROOT}\\export`, writable: true });
}
const guests = shares.map(share => share.guest.toLowerCase());
if (new Set(guests).size !== guests.length) throw new Error("Two shares use the same guest folder; set guest explicitly.");
for (const share of shares) {
const guest = share.guest.toLowerCase();
const reserved = guest === "c:\\pi" || guest.startsWith("c:\\pi\\") && !guest.startsWith("c:\\pi\\import\\") && guest !== "c:\\pi\\export";
if (!/^[a-z]:\\/.test(guest) || reserved) throw new Error(`Guest folder ${share.guest} must be an absolute Windows path outside ${GUEST_ROOT}.`);
}
return { shares };
}
/** Windows Sandbox configuration: no clipboard, audio, video or printer redirection; explicit folders only. */
export function sandboxConfig(runtime: string, config: string, out: string, shares: readonly SandboxShare[]): string {
const folder = (host: string, guest: string, readOnly: boolean) =>
` <MappedFolder><HostFolder>${xml(host)}</HostFolder><SandboxFolder>${xml(guest)}</SandboxFolder><ReadOnly>${readOnly}</ReadOnly></MappedFolder>`;
return ["<Configuration>",
" <VGpu>Disable</VGpu>", " <Networking>Enable</Networking>", " <ClipboardRedirection>Disable</ClipboardRedirection>",
" <AudioInput>Disable</AudioInput>", " <VideoInput>Disable</VideoInput>", " <PrinterRedirection>Disable</PrinterRedirection>",
" <MappedFolders>",
folder(runtime, `${GUEST_ROOT}\\runtime`, true), folder(config, `${GUEST_ROOT}\\config`, true), folder(out, `${GUEST_ROOT}\\out`, false),
...shares.map(share => folder(share.host, share.guest, !share.writable)),
" </MappedFolders>", "</Configuration>", ""].join("\n");
}
/** Run the sandbox host through a command runner: this machine's Windows, or another machine's over SSH. */
export function runnerHost(runner: CommandRunner): SandboxHost {
const run = async (file: string, args: string[], signal?: AbortSignal, timeoutMs = 60_000, input?: string) =>
(await runner.run(file, args, { signal, timeoutMs, input })).replace(/\r/g, "");
const files = runnerFiles(runner);
return {
// Plain-text errors: an encoded command otherwise reports progress and errors as CLIXML on stderr.
powershell: (script, signal, timeout) => run(powershellOf(runner),
["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", Buffer.from(
// Continue: under Stop, Windows PowerShell turns redirected native stderr into exceptions. Scripts throw real failures.
// Explicit exit 0: -Command otherwise exits 1 when the last command only set $? to false.
`$ProgressPreference = 'SilentlyContinue'; $ErrorActionPreference = 'Continue'\ntry {\n${script}\nexit 0\n} catch { [Console]::Error.WriteLine($_.Exception.Message); exit 1 }`,
"utf16le").toString("base64")], signal, timeout),
path: (value, to) => to === "windows" ? files.windows(value) : files.native(value),
mkdir: dir => files.mkdir(dir),
writeFile: (file, content) => files.write(file, content),
readFile: file => files.read(file).then(bytes => bytes.toString("utf8").replace(/\r/g, ""), () => ""),
remove: dir => files.remove(dir),
forward: async (port, signal) => runner.ssh ? tunnel(runner.ssh, port, signal) : { port, close: () => {} },
reverseRelay: (workdir, keys, signal) => startSessionRelay(runner, workdir, keys, signal),
};
}
/** Forward a free local port to a port on the remote machine's loopback over SSH. */
async function tunnel(alias: string, remotePort: number, signal?: AbortSignal): Promise<{ port: number; close(): void }> {
const local = await new Promise<number>((resolve, reject) => {
const server = createServer().listen(0, "127.0.0.1", () => { const { port } = server.address() as AddressInfo; server.close(() => resolve(port)); });
server.on("error", reject);
});
const child = spawn("ssh", [...SSH_OPTIONS, "-N", "-o", "ExitOnForwardFailure=yes", "-L", `127.0.0.1:${local}:127.0.0.1:${remotePort}`, alias], { stdio: "ignore" });
const close = () => { if (child.exitCode === null) child.kill(); };
for (let i = 0; i < 100; i++) {
if (child.exitCode !== null) throw new Error("The SSH tunnel to the sandbox did not start.");
if (signal?.aborted) { close(); throw new Error("Cancelled while starting the sandbox."); }
const open = await new Promise<boolean>(resolve => {
const socket = connect({ host: "127.0.0.1", port: local }, () => { socket.destroy(); resolve(true); });
socket.on("error", () => resolve(false));
});
if (open) return { port: local, close };
await new Promise(resolve => setTimeout(resolve, 100));
}
close();
throw new Error("The SSH tunnel to the sandbox did not start.");
}
const sleep = (ms: number, signal?: AbortSignal) => new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => { signal?.removeEventListener("abort", abort); resolve(); }, ms);
const abort = () => { clearTimeout(timer); reject(new Error("Cancelled while starting the sandbox.")); };
signal?.addEventListener("abort", abort, { once: true });
});
/** Wait until the guest driver answers; it starts a few seconds after the guest session. */
export async function waitForGuestDriver(driver: { call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<{ isError?: boolean }>; close(): Promise<void> },
signal?: AbortSignal, timeoutMs = 60_000): Promise<void> {
const deadline = Date.now() + timeoutMs;
let last = "";
while (Date.now() < deadline) {
try {
if (!(await driver.call("list_windows", {}, signal)).isError) return;
last = "list_windows failed";
} catch (error) {
last = error instanceof Error ? error.message : String(error);
await driver.close().catch(() => undefined);
}
await sleep(1000, signal);
}
throw new Error(`The sandbox driver did not answer within ${timeoutMs / 1000} s. ${last}`);
}
/**
* One disposable Windows Sandbox with its own CUA driver. It exists for self-isolation, so the agent's work
* does not touch the human's desktop. The sandbox window is minimized without activation and must stay open.
*/
export class WindowsSandbox {
readonly ref = "sb-" + randomUUID().slice(0, 8);
id?: string;
endpoint?: NonNullable<ComputerConfig["endpoint"]>;
private relays: Relay[] = [];
private tunnels: { port: number; close(): void }[] = [];
/** When this launch started; the sandbox client started since then is this sandbox's window. */
private startedAt?: string;
private launched = false;
private workdir?: { windows: string; unix: string };
private closed = false;
private constructor(private readonly host: SandboxHost, readonly shares: readonly SandboxShare[]) {}
/** Start a sandbox and its guest driver. Any failure or cancellation stops everything started so far. */
static async open(host: SandboxHost, runtimeCommand: string, options: SandboxOptions, signal?: AbortSignal): Promise<WindowsSandbox> {
const sandbox = new WindowsSandbox(host, options.shares);
try { await sandbox.start(runtimeCommand, signal); }
catch (error) { await sandbox.close(); throw error; }
return sandbox;
}
private ps(script: string, signal?: AbortSignal, timeoutMs?: number) { return this.host.powershell(script, signal, timeoutMs); }
private wsb(args: string) { return `& (Join-Path $env:LOCALAPPDATA 'Microsoft\\WindowsApps\\wsb.exe') ${args}`; }
private async start(runtimeCommand: string, signal?: AbortSignal): Promise<void> {
const runtime = await this.host.path(path.dirname(runtimeCommand), "windows");
const shares: SandboxShare[] = [];
for (const share of this.shares) {
const windows = /^[A-Za-z]:\\/.test(share.host) ? share.host : await this.host.path(share.host, "windows");
if (!/^[A-Za-z]:\\/.test(windows)) throw new ComputerRequestError(`Shared folder ${share.host} must be on a Windows drive; Windows Sandbox cannot map ${windows}.`);
shares.push({ ...share, host: windows });
}
if (!/^[A-Za-z]:\\/.test(runtime)) throw new ComputerRequestError("The driver runtime must be on a Windows drive to map it into the sandbox.");
const check = await this.ps(`$missing = @(${shares.map(share => psString(share.host)).join(",")}) | Where-Object { -not (Test-Path -LiteralPath $_ -PathType Container) }
"sandbox=" + (Test-Path 'C:\\Windows\\System32\\WindowsSandbox.exe') + ";wsb=" + (Test-Path (Join-Path $env:LOCALAPPDATA 'Microsoft\\WindowsApps\\wsb.exe')) + ";missing=" + ($missing -join '|') + ";local=" + $env:LOCALAPPDATA`, signal);
const facts = Object.fromEntries(check.trim().split(";").map(part => part.split("=") as [string, string]));
// Nothing has started yet: these are refusals, not unknown outcomes.
if (facts.sandbox !== "True" || facts.wsb !== "True") throw new ComputerRequestError("Windows Sandbox and its wsb command are not available on this host. Enable the Windows Sandbox feature first.");
if (facts.missing) throw new ComputerRequestError(`Shared host folders do not exist: ${facts.missing}`);
const windows = `${facts.local}\\PiComputerUse\\sandboxes\\${this.ref}`;
const unix = await this.host.path(windows.replace(/\\[^\\]+$/, ""), "unix").then(parent => `${parent}/${this.ref}`);
this.workdir = { windows, unix };
await this.host.mkdir(`${unix}/config`);
await this.host.mkdir(`${unix}/out`);
for (const name of ["guest-setup.ps1", "guest-start.ps1", "guest-identity-bridge.ps1", "host-relay.ps1"])
await this.host.writeFile(`${unix}/config/${name}`, await readFile(new URL(name, SCRIPTS), "utf8"));
for (const name of ["window-identity.ps1", "identity-auth.ps1", "identity-channel.ps1", "identity-channel.cs"])
await this.host.writeFile(`${unix}/config/${name}`, await readFile(new URL(name, COMMON_SCRIPTS), "utf8"));
await this.host.writeFile(`${unix}/config/relay-auth.ps1`, await readFile(RELAY_AUTH_SCRIPT, "utf8"));
const token = randomBytes(32).toString("hex");
const identityAuth = { session: this.ref, key: randomBytes(32).toString("hex") };
await this.host.writeFile(`${unix}/config/token.txt`, token);
await this.host.writeFile(`${unix}/config/identity.json`, JSON.stringify(identityAuth));
await this.host.writeFile(`${unix}/sandbox.wsb`, sandboxConfig(runtime, `${windows}\\config`, `${windows}\\out`, shares));
// A WindowsSandboxServer started by wsb.exe or WindowsSandbox.exe from WSL/SSH fails; launch through the user's shell
// and call wsb only once a sandbox VM exists.
const ids = `@(${this.wsb("list")} | Where-Object { $_ -match '^[0-9a-f-]{36}$' })`;
// Windows runs one sandbox at a time; launching beside another only shows an error dialog on the desktop.
const running = await this.ps(`if (Get-Process WindowsSandboxRemoteSession -ErrorAction SilentlyContinue) { 'running' }
else { Get-Process WindowsSandboxServer -ErrorAction SilentlyContinue | Stop-Process -Force }`, signal);
if (running.includes("running")) throw new ComputerRequestError("Windows Sandbox is already running on this machine, and Windows allows only one. "
+ "Close it first (Delete its computer:sandbox address if this session opened it), then open a new one. Nothing was started.");
const startedAt = this.startedAt = new Date().toISOString();
this.launched = true;
await this.ps(`Start-Process explorer.exe -ArgumentList ${psString(`${windows}\\sandbox.wsb`)}`, signal);
const id = await this.ps(`for ($i = 0; $i -lt 120; $i++) {
Start-Sleep -Milliseconds 1000
if (Get-Process vmmemWindowsSandbox -ErrorAction SilentlyContinue) {
$id = ${ids} | Select-Object -First 1
if ($id) { $id; exit }
}
}
throw 'The sandbox did not start within 120 s.'`, signal, 150_000);
this.id = id.trim();
// Hide the sandbox window from the human without activating it; closing it would stop the sandbox.
await this.ps(`Add-Type -Namespace PiSandbox -Name User32 -MemberDefinition '[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow);'
$since = [DateTime]::Parse(${psString(startedAt)}).AddSeconds(-5)
for ($i = 0; $i -lt 30; $i++) {
$client = Get-Process WindowsSandboxRemoteSession -ErrorAction SilentlyContinue | Where-Object { $_.StartTime -ge $since -and $_.MainWindowHandle -ne 0 } | Select-Object -First 1
if ($client) { [void][PiSandbox.User32]::ShowWindow($client.MainWindowHandle, 7); exit }
Start-Sleep -Milliseconds 500
}`, signal);
await this.ps(`for ($i = 0; $i -lt 120; $i++) {
${this.wsb(`exec --id ${this.id} --run-as ExistingLogin -c 'cmd.exe /c exit 0'`)} *> $null
if ($LASTEXITCODE -eq 0) { exit }
Start-Sleep -Seconds 1
}
throw 'The sandbox user session did not become available within 120 s.'`, signal, 180_000);
await this.ps(`${this.wsb(`exec --id ${this.id} --run-as System -c 'powershell.exe -NoProfile -ExecutionPolicy Bypass -File ${GUEST_ROOT}\\config\\guest-setup.ps1'`)} *> $null
${this.wsb(`exec --id ${this.id} --run-as ExistingLogin -c 'powershell.exe -NoProfile -ExecutionPolicy Bypass -File ${GUEST_ROOT}\\config\\guest-start.ps1'`)} *> $null
((${this.wsb(`ip --id ${this.id} --raw`)} | Out-String | ConvertFrom-Json).Networks | Select-Object -First 1).IpV4Address`, signal, 120_000)
.then(ip => this.startRelays(ip.trim(), token, identityAuth, signal));
}
private async startRelays(ip: string, token: string, identityAuth: RelayCredentials, signal?: AbortSignal): Promise<void> {
if (!/^\d+\.\d+\.\d+\.\d+$/.test(ip)) throw new Error("The sandbox reported no IPv4 address.");
const { windows, unix } = this.workdir!;
const relay = async (guestPort: number): Promise<Relay> => {
const file = `${windows}\\out\\relay-${guestPort}.txt`;
// No stdio redirection: it would inherit the interop pipes and keep this call waiting.
await this.ps(`Start-Process powershell.exe -WindowStyle Hidden -ArgumentList ${psString(`-NoProfile -ExecutionPolicy Bypass -File "${windows}\\config\\host-relay.ps1" -GuestIp ${ip} -GuestPort ${guestPort} -PortFile "${file}"`)}`, signal);
for (let i = 0; i < 40; i++) {
const text = await this.host.readFile(`${unix}/out/relay-${guestPort}.txt`);
const [port, pid] = text.trim().split(/\s+/).map(Number);
if (port > 0 && pid > 0) return { port, pid };
await sleep(250, signal);
}
throw new Error("The host relay did not start.");
};
this.relays.push(await relay(DRIVER_GUEST_PORT));
this.relays.push(await relay(IDENTITY_GUEST_PORT));
// Another machine's relays listen on its loopback; reach them through SSH tunnels.
this.tunnels.push(await this.host.forward(this.relays[0].port, signal));
this.tunnels.push(await this.host.forward(this.relays[1].port, signal));
this.endpoint = { url: `http://127.0.0.1:${this.tunnels[0].port}/mcp`, token, identityPort: this.tunnels[1].port, identityAuth };
}
/** Stop the sandbox, its relays and its host folder. Safe to call more than once. */
async close(): Promise<void> {
if (this.closed) return;
this.closed = true;
const pids = this.relays.map(relay => relay.pid);
this.relays = [];
for (const tunnel of this.tunnels) tunnel.close();
this.tunnels = [];
// A cancelled launch may have started a sandbox before its id was known; only this launch's sandbox can be running.
// Call wsb only once a VM exists (see start).
const unknown = this.launched && !this.id ? [`$ids = @()`,
`for ($i = 0; $i -lt 60 -and -not $ids; $i++) { if (Get-Process vmmemWindowsSandbox -ErrorAction SilentlyContinue) { $ids = @(${this.wsb("list")} | Where-Object { $_ -match '^[0-9a-f-]{36}$' }) }; if (-not $ids) { Start-Sleep 1 } }`,
`foreach ($id in $ids) { ${this.wsb("stop --id $id")} *> $null }`] : [];
await this.ps([
...unknown,
...(this.id ? [`${this.wsb(`stop --id ${this.id}`)} *> $null`] : []),
// wsb stop can fail over SSH; closing the sandbox client this launch started also stops the sandbox.
...(this.startedAt ? [`Get-Process WindowsSandboxRemoteSession -ErrorAction SilentlyContinue | Where-Object { $_.StartTime -ge [DateTime]::Parse(${psString(this.startedAt)}).AddSeconds(-5) } | Stop-Process -Force`] : []),
...(pids.length ? [`Stop-Process -Id ${pids.join(",")} -Force -ErrorAction SilentlyContinue`] : []),
"exit 0",
].join("\n"), undefined, 120_000).catch(() => undefined);
if (this.workdir) await this.host.remove(this.workdir.unix).catch(() => undefined);
}
/** Readable description for the sandbox address. */
describe(source: string): string {
const lines = [`# Windows Sandbox`, `Address: ${source}`, `Sandbox id: ${this.id ?? "unknown"}`,
"The sandbox window stays minimized on the host; closing it stops the sandbox. Clipboard, audio, video and printers are not shared."];
for (const share of this.shares) lines.push(`Shared: ${share.host} -> ${share.guest} (${share.writable ? "read/write" : "read-only"})`);
lines.push("Search this address for guest windows. computer_open at this address starts a guest app; Delete destroys the sandbox.");
return lines.join("\n");
}
}