Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
/** A machine's discovery resources: its windows and installed apps (Search) and its displays (Read). */
export type MachinePart = "windows" | "apps" | "displays";
/** A namespaced computer resource; references are session-issued, never native PIDs. */
export type ComputerAddress =
| { kind: "session"; source: string }
/** A machine by its name: `local` for this one, otherwise a name under `machines` in .pi/computer-use.json. */
| { kind: "machine"; source: string; ref: string; part?: MachinePart }
| { kind: "app" | "window" | "sequence" | "sandbox"; source: string; ref: string }
/** A display of one machine: `primary`, or a reference listed by that machine's displays. */
| { kind: "display"; source: string; machine: string; ref: string }
/** `hint` names one element of the observation for an internal input step; parsed addresses never carry it. */
| { kind: "observation"; source: string; ref: string; hint?: string }
/** A named rectangle of a window: `ref` is the window's reference, `name` the viewport's name in that window. */
| { kind: "viewport"; source: string; ref: string; name: string };
/** A name the agent gives a window (computer:<name>) or a viewport of a window (computer:<window>/<name>). */
export const NAME = /^[a-z][a-z0-9_-]{0,31}$/;
const references = new Set(["app", "window", "observation", "sequence", "sandbox", "machine"]);
/** Words a long address starts with; machine names and window names never take them. */
export const ADDRESS_KINDS: ReadonlySet<string> = new Set([...references, "session", "display"]);
const parts = new Set(["windows", "apps", "displays"]);
/** Resolve this namespace or reject malformed owned addresses before any native work. */
export function parseComputerAddress(source: string): ComputerAddress | undefined {
if (!source.startsWith("computer:")) return undefined;
const invalid = () => { throw new Error("Invalid computer resource. Use computer:session, computer:machine/<name> or an address returned by Search or Read."); };
if (source.length > 1024 || source.includes("#") || /[\s\x00-\x1f]/.test(source)) return invalid();
const [path, query] = source.slice(9).split("?");
if (source.split("?").length > 2) return invalid();
const segments = path.split("/");
const kind = segments[0];
if (kind === "session") {
if (segments.length !== 1 || query !== undefined) return invalid();
return { kind, source };
}
if (kind === "display")
throw new Error("Display addresses name their machine: computer:machine/<name>/display/primary, or a display listed by Read computer:machine/<name>/displays.");
const reference = (segment: string) => {
let ref: string;
try { ref = decodeURIComponent(segment); } catch { return invalid(); }
return /^[a-zA-Z0-9_-]{1,160}$/.test(ref) ? ref : invalid();
};
if (kind === "machine" && segments.length === 4 && segments[2] === "display") {
if (query !== undefined) return invalid();
return { kind: "display", source, machine: reference(segments[1]), ref: reference(segments[3]) };
}
if (kind === "window" && segments.length === 3) {
if (query !== undefined || !NAME.test(segments[2])) return invalid();
return { kind: "viewport", source, ref: reference(segments[1]), name: segments[2] };
}
const machinePart = kind === "machine" && segments.length === 3 ? segments[2] : undefined;
if (!references.has(kind) || (segments.length !== 2 && !(machinePart && parts.has(machinePart)))) return invalid();
const ref = reference(segments[1]);
if (query !== undefined) return invalid();
if (kind === "machine") return { kind, ref, source, ...(machinePart ? { part: machinePart as MachinePart } : {}) };
return { kind: kind as "app" | "window" | "observation" | "sequence" | "sandbox", ref, source };
}
/** Build a computer address from an opaque session-issued reference, or a machine's address from its name. */
export function computerSource(kind: "app" | "window" | "observation" | "sequence" | "sandbox" | "machine", ref: string, part?: MachinePart): string {
const source = `computer:${kind}/${encodeURIComponent(ref)}${part ? `/${part}` : ""}`;
parseComputerAddress(source);
return source;
}
/** The long address of a window's viewport. */
export function viewportSource(window: string, name: string): string {
const source = `${computerSource("window", window)}/${name}`;
parseComputerAddress(source);
return source;
}
/** The address of one machine's display: `primary` or a reference its displays listing issued. */
export function displaySource(machine: string, ref: string): string {
const source = `computer:machine/${encodeURIComponent(machine)}/display/${encodeURIComponent(ref)}`;
parseComputerAddress(source);
return source;
}