Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import sharp from "sharp";
import { displayModes as nativeDisplayModes, type DisplayModes } from "./display-mode.ts";
import { powershellOf, runPowerShell, runnerFiles, type CommandRunner, type RunnerFiles } from "./runner.ts";
/** The Chromium switch a WebView2 app gets for debugging; port 0 lets each app choose a free port. */
export const WEBVIEW2_DEBUGGING = "--remote-debugging-port=0";
/** Quote one argument for a Windows command line as CommandLineToArgvW and the C runtime read it back. */
export function windowsArgument(value: string): string {
if (value && !/[\s"]/.test(value)) return value;
return `"${value.replace(/(\\*)"/g, '$1$1\\"').replace(/(\\+)$/, "$1$1")}"`;
}
/** One app process as Windows reports it; `responding` is false when its UI thread stopped handling messages. */
export interface ProcessHealth { pid: number; name: string; responding: boolean; memory_mb: number; cpu_seconds: number }
/** A recognised line of text with its box in the pixels of the image that was read. */
export interface TextLine { text: string; x: number; y: number; width: number; height: number; words: { text: string; x: number; y: number; width: number; height: number }[] }
/**
* Work on the controlled machine that is neither capture nor input: display modes, an owned folder for files,
* process health and Windows' built-in OCR. Commands run through the machine's runner, locally or over SSH.
*/
export class MachineHost {
private folder?: Promise<{ windows: string; wsl: string }>;
private readonly folderId = randomUUID();
private readonly fs: RunnerFiles;
constructor(private readonly runner: CommandRunner, readonly displayModes: DisplayModes = nativeDisplayModes(runner)) {
this.fs = runnerFiles(runner);
}
private powershell(script: string, input?: string | Buffer, timeoutMs = 60_000): Promise<string> {
return runPowerShell(this.runner, powershellOf(this.runner), script, { ...(input !== undefined ? { input } : {}), timeoutMs });
}
/** A file in the owned folder as the runner addresses it. */
private async inFolder(name: string): Promise<string> {
return this.fs.join((await this.files()).wsl, checkedName(name));
}
/** This session's folder in the machine's Windows temp directory, created on first use; `wsl` is the runner's own path. */
files(): Promise<{ windows: string; wsl: string }> {
this.folder ??= (async () => {
const windows = await this.powershell(`$d = Join-Path ([IO.Path]::GetTempPath()) 'pi-computer-use\\${this.folderId}'; New-Item -ItemType Directory -Force -Path $d | Out-Null; $d`);
const wsl = await this.fs.native(windows);
return { windows, wsl };
})();
this.folder.catch(() => { this.folder = undefined; });
return this.folder;
}
/** Write bytes as a file in the owned folder and return its Windows path. */
async put(name: string, bytes: Buffer): Promise<string> {
const folder = await this.files();
await this.fs.write(await this.inFolder(name), bytes);
return `${folder.windows}\\${name}`;
}
/** Read a file from the owned folder. */
async get(name: string): Promise<Buffer> {
return this.fs.read(await this.inFolder(name));
}
/** Rename a file inside the owned folder. */
async rename(from: string, to: string): Promise<void> {
await this.fs.rename(await this.inFolder(from), await this.inFolder(to));
}
/** Files in the owned folder with their sizes. */
async list(): Promise<{ name: string; bytes: number }[]> {
return this.fs.list((await this.files()).wsl);
}
/** Remove the owned folder; called when the computer session closes. */
async removeFiles(): Promise<void> {
if (!this.folder) return;
const folder = await this.folder.catch(() => undefined);
this.folder = undefined;
if (folder) await this.fs.remove(folder.wsl);
}
/**
* Post WM_CLOSE to a window of one process and wait up to `timeoutMs` for it to go: `closed` when it is gone (or was
* already), `open` when the app kept it, `refused` when Windows would not deliver the message.
*/
async closeWindow(pid: number, window: number, timeoutMs: number): Promise<"closed" | "open" | "refused"> {
if (![pid, window, timeoutMs].every(value => Number.isSafeInteger(value) && value > 0)) throw new Error("Invalid window to close.");
const out = await this.powershell(`Add-Type -Name Close -Namespace PiComputerUse -MemberDefinition '[DllImport("user32.dll")] public static extern bool IsWindow(IntPtr h); [DllImport("user32.dll")] public static extern bool PostMessage(IntPtr h, uint m, IntPtr w, IntPtr l); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint p);'
$h = [IntPtr]${window}
function Present { $p = [uint32]0; [void][PiComputerUse.Close]::GetWindowThreadProcessId($h, [ref]$p); [PiComputerUse.Close]::IsWindow($h) -and $p -eq ${pid} }
if (-not (Present)) { 'closed'; exit }
if (-not [PiComputerUse.Close]::PostMessage($h, 0x10, [IntPtr]::Zero, [IntPtr]::Zero)) { 'refused'; exit }
$deadline = (Get-Date).AddMilliseconds(${timeoutMs})
while ((Get-Date) -lt $deadline) { if (-not (Present)) { 'closed'; exit }; Start-Sleep -Milliseconds 100 }
'open'`, undefined, timeoutMs + 20_000);
const outcome = out.trim();
if (outcome !== "closed" && outcome !== "open" && outcome !== "refused") throw new Error(`Unexpected close result: ${outcome.slice(0, 200)}`);
return outcome;
}
/**
* Whether a window's process still handles messages, and how much memory and CPU it has used.
* With a window handle, Windows' own hung-window test decides; the process's main window may be another one.
*/
async processHealth(pid: number, window?: number): Promise<ProcessHealth | undefined> {
if (!Number.isSafeInteger(pid) || pid <= 0 || (window !== undefined && (!Number.isSafeInteger(window) || window <= 0))) return undefined;
const responding = window === undefined ? "$p.Responding"
: `-not (Add-Type -PassThru -Name Hung -Namespace PiComputerUse -MemberDefinition '[DllImport("user32.dll")] public static extern bool IsHungAppWindow(IntPtr window);')::IsHungAppWindow([IntPtr]${window})`;
const out = await this.powershell(`$p = Get-Process -Id ${pid} -ErrorAction SilentlyContinue; if ($p) { @{ pid = $p.Id; name = $p.ProcessName; responding = [bool](${responding}); memory_mb = [math]::Round($p.WorkingSet64 / 1MB); cpu_seconds = [math]::Round($p.CPU, 1) } | ConvertTo-Json -Compress }`, undefined, 20_000);
return out ? JSON.parse(out) as ProcessHealth : undefined;
}
/**
* Record what the default playback device plays (WASAPI loopback) for `seconds` into a WAV file in the owned
* folder and return its name. Fails with a clear message when the machine has no active playback device.
*/
async recordAudio(seconds: number, name: string): Promise<string> {
const script = await this.put("audio-loopback.ps1", await readFile(new URL("../scripts/audio-loopback.ps1", import.meta.url)));
const folder = await this.files();
try {
await this.runner.run(powershellOf(this.runner), ["-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script, String(seconds), `${folder.windows}\\${checkedName(name)}`],
{ timeoutMs: seconds * 1000 + 30_000 });
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(/0x80070490|Element not found/.test(message) ? "That machine has no active playback device to record from."
: /muted/.test(message) ? "The default playback device on that machine is muted, so loopback would record only silence. Ask the user to unmute it." : message);
}
return name;
}
/**
* Turn WebView2 remote debugging on or off for one app through the user's per-exe WebView2 policy
* (HKCU\Software\Policies\Microsoft\Edge\WebView2\AdditionalBrowserArguments). It applies to the app's next start,
* whoever starts it, and stays until turned off. Returns the value now set for that exe, if any.
*/
async webview2Debugging(app: string, enabled: boolean): Promise<string | undefined> {
const key = "HKCU:\\Software\\Policies\\Microsoft\\Edge\\WebView2\\AdditionalBrowserArguments";
const name = app.replace(/'/g, "''");
const change = enabled
? `New-Item -Path '${key}' -Force | Out-Null; New-ItemProperty -Path '${key}' -Name '${name}' -Value '${WEBVIEW2_DEBUGGING}' -PropertyType String -Force | Out-Null`
: `Remove-ItemProperty -Path '${key}' -Name '${name}' -ErrorAction SilentlyContinue`;
// Policy keys are writable only with administrator rights: with UAC, Windows asks the person at that machine to
// approve the elevated change on its secure desktop, which no injected input can reach.
const encoded = Buffer.from(change, "utf16le").toString("base64");
await this.powershell(`try { ${change} } catch [System.Security.SecurityException], [System.UnauthorizedAccessException] {
$p = Start-Process powershell.exe -Verb RunAs -WindowStyle Hidden -Wait -PassThru -ArgumentList '-NoProfile -EncodedCommand ${encoded}'
if ($p.ExitCode -ne 0) { throw "The elevated change failed with exit code $($p.ExitCode)." }
}`, undefined, 180_000);
const out = await this.powershell(`(Get-ItemProperty -Path '${key}' -Name '${name}' -ErrorAction SilentlyContinue).'${name}'`, undefined, 20_000);
return out.trim() || undefined;
}
/**
* Start an executable with extra environment variables and return its process ID. CUA's launch has no environment,
* so this starts the process from PowerShell; the caller binds its window as for any launch.
*/
async startProcess(path: string, args: string[], environment: Record<string, string>): Promise<number> {
const spec = Buffer.from(JSON.stringify({ path, args: args.map(windowsArgument).join(" "), environment }), "utf8").toString("base64");
// The variables go into this PowerShell's own environment, which Start-Process passes on. Start-Process also keeps
// the app from inheriting this call's output pipes, which would hold the call open for the app's lifetime.
const out = await this.powershell(`$s = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${spec}')) | ConvertFrom-Json
foreach ($v in $s.environment.PSObject.Properties) { [Environment]::SetEnvironmentVariable($v.Name, [string]$v.Value) }
$a = @{ FilePath = $s.path; PassThru = $true }; if ($s.args) { $a.ArgumentList = $s.args }
(Start-Process @a).Id`, undefined, 30_000);
const pid = Number(out.trim());
if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("The process did not report an ID; do not relaunch blindly.");
return pid;
}
/** The process that owns the foreground window. */
async foregroundOwner(): Promise<{ pid: number; name: string } | undefined> {
const out = await this.powershell(`Add-Type -Name F -Namespace PiComputerUse -MemberDefinition '[DllImport("user32.dll")] public static extern System.IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(System.IntPtr h, out uint p);'
$p = 0; [void][PiComputerUse.F]::GetWindowThreadProcessId([PiComputerUse.F]::GetForegroundWindow(), [ref]$p)
$proc = Get-Process -Id $p -ErrorAction SilentlyContinue
if ($proc) { @{ pid = [int]$p; name = $proc.ProcessName } | ConvertTo-Json -Compress }`, undefined, 20_000);
return out ? JSON.parse(out) : undefined;
}
/** The clipboard's image as PNG bytes, or undefined when it holds no image. */
async clipboardImage(): Promise<Buffer | undefined> {
const folder = await this.files();
const name = `clipboard-${Date.now()}.png`;
const saved = await this.powershell(`Add-Type -AssemblyName System.Windows.Forms, System.Drawing; $i = [Windows.Forms.Clipboard]::GetImage(); if ($i) { $i.Save('${folder.windows.replace(/'/g, "''")}\\${name}', [Drawing.Imaging.ImageFormat]::Png); 'saved' }`, undefined, 20_000);
if (saved !== "saved") return undefined;
try { return await this.get(name); }
finally { await this.fs.remove(await this.inFolder(name)).catch(() => undefined); }
}
/**
* Recognise text with Windows.Media.Ocr in the user's languages. Boxes are in the given image's pixels;
* a region is cut from it, enlarged up to 2x for small glyphs, and its boxes are mapped back.
*/
async ocr(image: Buffer, region?: { x: number; y: number; width: number; height: number }): Promise<TextLine[]> {
let png = image, box = (value: Omit<TextLine, "words">) => value;
if (region) {
const meta = await sharp(image).metadata();
const left = Math.max(0, Math.floor(region.x)), top = Math.max(0, Math.floor(region.y));
const width = Math.min(meta.width! - left, Math.ceil(region.x + region.width) - left), height = Math.min(meta.height! - top, Math.ceil(region.y + region.height) - top);
if (width <= 0 || height <= 0) throw new Error("The region is outside this observation's image.");
const scale = Math.max(1, Math.min(2, Math.floor(2600 / Math.max(width, height))));
png = await sharp(image).extract({ left, top, width, height }).resize(width * scale, height * scale, { kernel: "lanczos3" }).png().toBuffer();
box = value => ({ ...value, x: Math.round(left + value.x / scale), y: Math.round(top + value.y / scale), width: Math.round(value.width / scale), height: Math.round(value.height / scale) });
}
const name = `ocr-${Date.now()}-${Math.random().toString(36).slice(2)}.png`;
const path = await this.put(name, png);
try {
const out = await this.powershell(OCR_SCRIPT.replace("__PATH__", path.replace(/'/g, "''")), undefined, 60_000);
const lines = out ? JSON.parse(out) as TextLine[] : [];
return lines.map(line => ({ ...box(line), words: line.words.map(word => box(word) as TextLine["words"][number]) }));
} finally {
await this.fs.remove(await this.inFolder(name)).catch(() => undefined);
}
}
}
/** Accept plain file names only, so a name never leaves the owned folder. */
export function checkedName(name: string): string {
if (typeof name !== "string" || !/^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,127}$/.test(name) || name.includes("..")) throw new Error("Use a plain file name: letters, digits, space, dot, dash, underscore or brackets.");
return name;
}
const OCR_SCRIPT = String.raw`
Add-Type -AssemblyName System.Runtime.WindowsRuntime
$null = [Windows.Storage.StorageFile, Windows.Storage, ContentType = WindowsRuntime]
$null = [Windows.Media.Ocr.OcrEngine, Windows.Foundation, ContentType = WindowsRuntime]
$null = [Windows.Graphics.Imaging.BitmapDecoder, Windows.Graphics, ContentType = WindowsRuntime]
$asTask = ([System.WindowsRuntimeSystemExtensions].GetMethods() | Where-Object { $_.Name -eq 'AsTask' -and $_.GetParameters().Count -eq 1 -and $_.GetParameters()[0].ParameterType.Name -eq 'IAsyncOperation` + "`" + String.raw`1' })[0]
function Await($operation, $type) { $task = $asTask.MakeGenericMethod($type).Invoke($null, @($operation)); $task.Wait(-1) | Out-Null; $task.Result }
$file = Await ([Windows.Storage.StorageFile]::GetFileFromPathAsync('__PATH__')) ([Windows.Storage.StorageFile])
$stream = Await ($file.OpenAsync([Windows.Storage.FileAccessMode]::Read)) ([Windows.Storage.Streams.IRandomAccessStream])
$decoder = Await ([Windows.Graphics.Imaging.BitmapDecoder]::CreateAsync($stream)) ([Windows.Graphics.Imaging.BitmapDecoder])
$bitmap = Await ($decoder.GetSoftwareBitmapAsync()) ([Windows.Graphics.Imaging.SoftwareBitmap])
$engine = [Windows.Media.Ocr.OcrEngine]::TryCreateFromUserProfileLanguages()
if (-not $engine) { throw 'Windows OCR has no installed recognition language.' }
$result = Await ($engine.RecognizeAsync($bitmap)) ([Windows.Media.Ocr.OcrResult])
$stream.Dispose()
$lines = @(foreach ($line in $result.Lines) {
$words = @(foreach ($w in $line.Words) { $r = $w.BoundingRect; [pscustomobject]@{ text = $w.Text; x = [int]$r.X; y = [int]$r.Y; width = [int]$r.Width; height = [int]$r.Height } })
$x = ($words | Measure-Object -Property x -Minimum).Minimum; $y = ($words | Measure-Object -Property y -Minimum).Minimum
$right = ($words | ForEach-Object { $_.x + $_.width } | Measure-Object -Maximum).Maximum; $bottom = ($words | ForEach-Object { $_.y + $_.height } | Measure-Object -Maximum).Maximum
[pscustomobject]@{ text = $line.Text; x = [int]$x; y = [int]$y; width = [int]($right - $x); height = [int]($bottom - $y); words = $words }
})
ConvertTo-Json -InputObject $lines -Depth 5 -Compress
`;