Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { execFile } from "node:child_process";
import { copyFile, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { loadComputerConfig, readMachineSettings, type MachineConfig } from "./config.ts";
import { isWsl } from "./native-runtime.ts";
import { WINDOWS_POWERSHELL, localRunner, runPowerShell, sshRunner, type CommandRunner } from "./runner.ts";
import { MXC, PERCEPTION, WINDOWS_NODE } from "./setup-pins.ts";
import { ensureWindowsNode } from "./windows-node.ts";
import { prepareRemoteBrowser } from "./remote-browser.ts";
import { prepareRemoteRuntime } from "./remote-runtime.ts";
/** Optional parts and what each adds; `machine` prepares another machine instead of installing on one. */
export const SETUP_PARTS = [
{ name: "browser", description: "Windows Node for open_browser and attach_browser when Pi runs in WSL (about 30 MB)" },
{ name: "firefox", description: "Playwright's Firefox for \"browser\":\"firefox\"" },
{ name: "perception", description: "Cua perception for the regions view (about 420 MB, AGPL-3.0 icon detector)" },
{ name: "mxc", description: "Microsoft MXC for open_session (Windows 11 24H2 or later)" },
{ name: "vscode-bridge", description: "Pi Browser Bridge for VS Code's integrated browser" },
{ name: "machine", description: "Prepare another Windows machine with WSL, reached over SSH" },
] as const;
export type SetupPart = (typeof SETUP_PARTS)[number]["name"];
export interface SetupRequest {
part: SetupPart;
/** A configured machine name; this machine when omitted or "local". Not used by `machine`. */
machine?: string;
/** For `machine`: the SSH alias and the name to configure it under. */
ssh?: string;
name?: string;
/** For `vscode-bridge` with a portable VS Code: its bin\code.cmd and data\argv.json on the target. */
code?: string;
argv?: string;
}
/** Thrown for a request that cannot start; nothing was installed. */
export class SetupError extends Error {}
/** The Windows machine a part installs on, and how its commands run. */
interface Target {
label: string;
runner: CommandRunner;
/** PowerShell as the runner sees it. */
powershell: string;
/** Pi itself runs on this Windows: paths are Windows paths and no WSL is involved. */
native: boolean;
machine?: MachineConfig;
/** WSL interop socket from the trusted settings. */
interop?: string;
}
export interface SetupDeps {
platform: NodeJS.Platform;
wsl: () => Promise<boolean>;
local: (interop?: string) => CommandRunner;
ssh: (alias: string) => CommandRunner;
}
const defaultDeps: SetupDeps = { platform: process.platform, wsl: isWsl, local: localRunner, ssh: sshRunner };
type Progress = (message: string) => void;
const BRIDGE_ID = "alexshpunt.pi-browser-bridge";
const PACKAGE = fileURLToPath(new URL("..", import.meta.url));
const psString = (value: string) => `'${value.replace(/'/g, "''")}'`;
/**
* Downloads are checked on the target before use; native tools print their own output, and their failure stops
* the script with that output. tar.exe ships with Windows 10 and later and unpacks zip much faster than .NET ZipFile.
*/
const PRELUDE = `
# A WSL working directory is a UNC path that cmd-based tools such as code.cmd cannot use.
Set-Location -LiteralPath $env:TEMP
[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
# .NET directly: Get-FileHash and Expand-Archive come from script modules that some PowerShell setups fail to load,
# and WebClient downloads far faster than Invoke-WebRequest in Windows PowerShell.
function Get-Sha256([string]$File) {
$stream = [IO.File]::OpenRead($File)
try { -join ([Security.Cryptography.SHA256]::Create().ComputeHash($stream) | ForEach-Object { $_.ToString('x2') }) } finally { $stream.Dispose() }
}
function Get-Pinned([string]$Url, [string]$File, [string]$Sha256) {
if ((Test-Path -LiteralPath $File) -and (Get-Sha256 $File) -eq $Sha256) { return }
(New-Object Net.WebClient).DownloadFile($Url, $File)
if ((Get-Sha256 $File) -ne $Sha256) {
Remove-Item -LiteralPath $File -Force
throw "The download from $Url does not match its pinned SHA-256; nothing was installed."
}
}
function Invoke-Native([string]$Exe, [string[]]$Arguments, [switch]$AllowFailure) {
$ErrorActionPreference = 'Continue'
$out = (& $Exe @Arguments 2>&1 | ForEach-Object { "$_" }) -join [Environment]::NewLine
if ($LASTEXITCODE -ne 0 -and !$AllowFailure) { throw ([IO.Path]::GetFileName($Exe) + " exited with $($LASTEXITCODE): " + $out) }
$out
}
function Expand-Zip([string]$Zip, [string]$Destination) {
New-Item -ItemType Directory -Force -Path $Destination | Out-Null
$tar = Join-Path $env:SystemRoot 'System32\\tar.exe'
if (Test-Path -LiteralPath $tar) { Invoke-Native $tar @('-xf', $Zip, '-C', $Destination) | Out-Null }
else { Add-Type -AssemblyName System.IO.Compression.FileSystem; [IO.Compression.ZipFile]::ExtractToDirectory($Zip, $Destination) }
}
`;
const ps = (target: Target, script: string, timeoutMs: number, signal?: AbortSignal) =>
runPowerShell(target.runner, target.powershell, PRELUDE + script, { timeoutMs, scriptOnStdin: true, ...(signal ? { signal } : {}) });
/** A Windows path on the target as its runner addresses files. */
async function targetPath(target: Target, windows: string): Promise<string> {
return target.native ? windows : (await target.runner.run("wslpath", ["-u", windows])).trim();
}
/** A target path (Windows, or the target's WSL path) as Windows sees it. */
async function windowsPath(target: Target, value: string): Promise<string> {
return target.native || /^[A-Za-z]:\\/.test(value) ? value : (await target.runner.run("wslpath", ["-w", value])).trim();
}
/** Write bytes to a Windows path on the target. */
async function put(target: Target, windows: string, bytes: Buffer): Promise<void> {
const file = await targetPath(target, windows);
if (!target.machine) await writeFile(file, bytes);
else await target.runner.run("dd", ["status=none", `of=${file}`], { input: bytes, timeoutMs: 600_000 });
}
async function resolveTarget(cwd: string, machine: string | undefined, deps: SetupDeps): Promise<Target> {
const settings = await readMachineSettings(cwd);
if (machine && machine !== "local") {
const config = settings.machines[machine];
if (!config) throw new SetupError(`No machine named ${machine} is configured in .pi/computer-use.json; nothing was installed.`);
return { label: `machine ${machine}`, runner: deps.ssh(config.ssh), powershell: WINDOWS_POWERSHELL, native: false, machine: config };
}
if (deps.platform === "win32") return { label: "this machine", runner: deps.local(), powershell: "powershell.exe", native: true };
// Pi in WSL: Windows reached through interop.
if (deps.platform === "linux" && await deps.wsl())
return { label: "this machine", runner: deps.local(settings.interop), powershell: WINDOWS_POWERSHELL, native: false, ...(settings.interop ? { interop: settings.interop } : {}) };
throw new SetupError("Setup needs Pi on Windows or in WSL; nothing was installed.");
}
/** Install the pinned portable Node on the target and return its node.exe as a Windows path. */
async function installNode(target: Target, signal?: AbortSignal): Promise<{ exe: string; installed: boolean }> {
const name = `node-${WINDOWS_NODE.version}-win-x64`;
const out = await ps(target, `
$root = Join-Path $env:LOCALAPPDATA 'Programs\\PiComputerUse'
$exe = Join-Path $root ${psString(`${name}\\node.exe`)}
if (Test-Path -LiteralPath $exe) { 'present'; $exe; return }
New-Item -ItemType Directory -Force -Path $root | Out-Null
$zip = Join-Path $env:TEMP ${psString(`pi-${name}.zip`)}
Get-Pinned ${psString(WINDOWS_NODE.url)} $zip ${psString(WINDOWS_NODE.sha256)}
$stage = Join-Path $root ('.staging-' + [guid]::NewGuid())
Expand-Zip $zip $stage
Move-Item -LiteralPath (Join-Path $stage ${psString(name)}) -Destination $root
Remove-Item -LiteralPath $stage, $zip -Recurse -Force
'installed'; $exe`, 600_000, signal);
const [state, exe] = out.split("\n").slice(-2);
return { exe, installed: state === "installed" };
}
async function browser(target: Target, signal?: AbortSignal): Promise<string> {
if (target.native) return "Pi runs on Windows here, so browsers use Pi's own Node. Nothing to install.";
const node = await installNode(target, signal);
return node.installed ? `Installed Windows Node ${WINDOWS_NODE.version} on ${target.label}: ${node.exe}` : `Windows Node is already installed on ${target.label}: ${node.exe}`;
}
async function firefox(target: Target, signal?: AbortSignal): Promise<string> {
const cli = path.join(path.dirname(createRequire(import.meta.url).resolve("playwright-core/package.json")), "cli.js");
let node: string, script: string;
if (target.machine) {
const prepared = await prepareRemoteBrowser(target.runner, signal, { node: target.machine.node, package: target.machine.package });
node = prepared.node;
script = await windowsPath(target, `${prepared.package}/node_modules/playwright-core/cli.js`);
} else if (target.native) { node = process.execPath; script = cli; }
else {
node = await ensureWindowsNode(target.interop, signal);
script = await windowsPath(target, cli);
}
const out = await target.runner.run(node, [script, "install", "firefox"], { timeoutMs: 900_000, ...(signal ? { signal } : {}) });
return `Playwright's Firefox is ready on ${target.label}.${tail(out)}`;
}
async function perception(target: Target, cwd: string, signal?: AbortSignal): Promise<string> {
const driver = await windowsPath(target, target.machine ? target.machine.command ?? await prepareRemoteRuntime(target.runner, signal) : (await loadComputerConfig(cwd)).command);
const { name, release, catalog, archive, version } = PERCEPTION;
const out = await ps(target, `
$driver = ${psString(driver)}
$status = Invoke-Native $driver @('extension', 'status', 'perception') -AllowFailure
if ($status -notmatch ${psString(`Active version: ${version.replace(/\./g, "\\.")}`)}) {
$dir = Join-Path $env:TEMP 'pi-perception'
New-Item -ItemType Directory -Force -Path $dir | Out-Null
$catalog = Join-Path $dir ${psString(`${name}.catalog.json`)}
Get-Pinned ${psString(`${release}${name}.catalog.json`)} $catalog ${psString(catalog)}
Get-Pinned ${psString(`${release}${name}.tar.gz`)} (Join-Path $dir ${psString(`${name}.tar.gz`)}) ${psString(archive)}
Invoke-Native $driver @('extension', 'install', 'perception', '--catalog', $catalog)
Remove-Item -LiteralPath $dir -Recurse -Force
}
Invoke-Native $driver @('extension', 'status', 'perception', '--self-test')`, 2_400_000, signal);
return `Cua perception ${version} is ready on ${target.label}.${tail(out)}`;
}
async function mxc(target: Target, signal?: AbortSignal): Promise<string> {
const out = await ps(target, `
$dir = Join-Path $env:LOCALAPPDATA 'PiComputerUse\\mxc'
$zip = Join-Path $env:TEMP ${psString(`pi-mxc-${MXC.version}.zip`)}
Get-Pinned ${psString(MXC.url)} $zip ${psString(MXC.sha256)}
$stage = Join-Path $env:TEMP ('pi-mxc-' + [guid]::NewGuid())
Expand-Zip $zip $stage
New-Item -ItemType Directory -Force -Path $dir | Out-Null
Copy-Item -LiteralPath (Join-Path $stage 'x64\\wxc-exec.exe') -Destination $dir -Force
Remove-Item -LiteralPath $stage, $zip -Recurse -Force
Join-Path $dir 'wxc-exec.exe'`, 600_000, signal);
return `Installed MXC ${MXC.version} on ${target.label}: ${out.split("\n").at(-1)}`;
}
/** Add the bridge to VS Code's argv.json (JSONC) without rewriting the rest; undefined when it is already allowed. */
export function allowProposedApi(text: string, id = BRIDGE_ID): string | undefined {
if (text.includes(id)) return undefined;
if (!text.trim()) text = "{\n}\n";
// Whether only whitespace and comments come before the closing bracket, so no separating comma is needed.
const empty = (rest: string, close: string) => rest.replace(/\/\/[^\n]*|\/\*[\s\S]*?\*\/|\s/g, "").startsWith(close);
const list = /"enable-proposed-api"\s*:\s*\[/.exec(text);
if (list) {
const at = list.index + list[0].length;
return text.slice(0, at) + JSON.stringify(id) + (empty(text.slice(at), "]") ? "" : ", ") + text.slice(at);
}
const brace = text.indexOf("{") + 1;
if (!brace) throw new SetupError("VS Code's argv.json has no JSON object; fix or remove it, then run setup again.");
return `${text.slice(0, brace)}\n\t"enable-proposed-api": [${JSON.stringify(id)}]${empty(text.slice(brace), "}") ? "" : ","}${text.slice(brace)}`;
}
/** Run a command on the machine where Pi runs; npm and npx are .cmd files on Windows. */
function local(command: string, args: string[], cwd: string, signal?: AbortSignal): Promise<string> {
return new Promise((resolve, reject) => execFile(command, args, { cwd, shell: process.platform === "win32", timeout: 600_000, maxBuffer: 16 * 1024 * 1024, ...(signal ? { signal } : {}) },
(error, stdout, stderr) => error ? reject(new Error(`${command} ${args[0]} failed: ${(stderr || error.message).trim().slice(-800)}`)) : resolve(stdout)));
}
/** Package the bridge with its ws dependency into a .vsix. */
async function buildBridge(signal?: AbortSignal): Promise<Buffer> {
const work = await mkdtemp(path.join(tmpdir(), "pi-browser-bridge-"));
try {
for (const file of ["package.json", "extension.js", "README.md"]) await copyFile(path.join(PACKAGE, "vscode-browser-bridge", file), path.join(work, file));
await local("npm", ["install", "--omit=dev", "--no-audit", "--no-fund", "--silent"], work, signal);
await local("npx", ["--yes", "@vscode/[email protected]", "package", "--skip-license", "-o", "pi-browser-bridge.vsix"], work, signal);
return await readFile(path.join(work, "pi-browser-bridge.vsix"));
} finally { await rm(work, { recursive: true, force: true }); }
}
async function vscodeBridge(target: Target, request: SetupRequest, signal?: AbortSignal): Promise<string> {
if (!request.code !== !request.argv) throw new SetupError("For a portable VS Code pass both its bin\\code.cmd and its data\\argv.json; nothing was installed.");
const vsix = await buildBridge(signal);
const temp = await ps(target, "$env:TEMP", 30_000, signal);
const file = `${temp}\\pi-browser-bridge.vsix`;
await put(target, file, vsix);
const code = request.code ? await windowsPath(target, request.code) : "";
const argv = request.argv ? await windowsPath(target, request.argv) : "";
const found = await ps(target, `
$code = ${psString(code)}
if (!$code) {
foreach ($candidate in @((Join-Path $env:LOCALAPPDATA 'Programs\\Microsoft VS Code\\bin\\code.cmd'), (Join-Path $env:ProgramFiles 'Microsoft VS Code\\bin\\code.cmd'))) {
if (Test-Path -LiteralPath $candidate) { $code = $candidate; break }
}
}
if (!$code) { throw 'No installed VS Code found. For a portable VS Code pass its bin\\code.cmd and data\\argv.json.' }
$argv = ${psString(argv)}
if (!$argv) { $argv = Join-Path $env:USERPROFILE '.vscode\\argv.json' }
Invoke-Native $code @('--install-extension', ${psString(file)}, '--force') | Out-Null
Remove-Item -LiteralPath ${psString(file)} -Force
$text = if (Test-Path -LiteralPath $argv) { [IO.File]::ReadAllText($argv) } else { '' }
$code
$argv
[Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($text))`, 300_000, signal);
const [codeCmd, argvJson, encoded] = found.split("\n").slice(-3);
const next = allowProposedApi(Buffer.from(encoded, "base64").toString("utf8"));
if (next !== undefined) await ps(target, `
$argv = ${psString(argvJson)}
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $argv) | Out-Null
[IO.File]::WriteAllText($argv, [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String(${psString(Buffer.from(next).toString("base64"))})), (New-Object Text.UTF8Encoding $false))`, 30_000, signal);
return `Installed the Pi Browser Bridge into ${codeCmd} on ${target.label}. ${argvJson} ${next === undefined ? "already allows" : "now allows"} its proposed API. Quit VS Code fully and start it again; Reload Window does not reread argv.json.`;
}
async function prepareMachine(request: SetupRequest, deps: SetupDeps, progress: Progress, signal?: AbortSignal): Promise<string> {
const alias = request.ssh;
if (!alias || !/^[A-Za-z0-9][A-Za-z0-9._@-]{0,127}$/.test(alias)) throw new SetupError("machine needs the SSH alias of a Windows machine with WSL; nothing was installed.");
const name = request.name ?? alias;
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(name) || name === "local") throw new SetupError("The machine name must be letters, digits, - or _, and not local.");
const runner = deps.ssh(alias);
progress("preparing the driver");
await prepareRemoteRuntime(runner, signal);
progress("preparing the browser worker");
await prepareRemoteBrowser(runner, signal);
return `Prepared ${alias} for computer use. Add this trusted machine under "machines" in .pi/computer-use.json:\n${JSON.stringify({ [name]: { ssh: alias } }, null, 2)}`;
}
const tail = (out: string) => {
const lines = out.replace(/\r/g, "").split("\n").map(line => line.trimEnd()).filter(Boolean);
return lines.length ? `\n${lines.slice(-8).join("\n")}` : "";
};
/** Install one optional part on this machine or a configured one, or prepare another machine. */
export async function runSetup(request: SetupRequest, cwd: string, options: { progress?: Progress; signal?: AbortSignal; deps?: Partial<SetupDeps> } = {}): Promise<string> {
const deps = { ...defaultDeps, ...options.deps };
const progress = options.progress ?? (() => undefined);
if (!SETUP_PARTS.some(part => part.name === request.part)) throw new SetupError(`Unknown part ${request.part}. Parts: ${SETUP_PARTS.map(part => part.name).join(", ")}.`);
if (request.part === "machine") return prepareMachine(request, deps, progress, options.signal);
const target = await resolveTarget(cwd, request.machine, deps);
progress(`working on ${target.label}`);
switch (request.part) {
case "browser": return browser(target, options.signal);
case "firefox": return firefox(target, options.signal);
case "perception": return perception(target, cwd, options.signal);
case "mxc": return mxc(target, options.signal);
case "vscode-bridge": return vscodeBridge(target, request, options.signal);
}
}