Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
import { validateUiCapture, type UiCaptureOptions } from "./ui-resources.ts";
import { SEQUENCE_LIMITS, validateSequence, type SequenceOptions, type FrameSelection } from "./temporal-capture.ts";
import { validateWebSnapshot, validateWebTarget, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
import { validateRendering, type FrameRendering } from "./contact-sheet.ts";
/** One native Read mode. Details and sampled frames never grant input authority. */
export type ComputerViews =
| { kind: "state" }
| { kind: "dom"; options: WebSnapshotOptions }
| { kind: "overview"; capture: UiCaptureOptions }
| { kind: "region"; region: { x: number; y: number; width: number; height: number } }
| { kind: "sequence"; options: SequenceOptions }
/** Selected frames, one image each, or one sheet or composite image of them. */
| { kind: "frames"; selection: FrameSelection; rendering?: FrameRendering }
/** An element by a UI hint or an aria ref, or, for crop and highlight of a browser page, by a Playwright locator in `locator`. */
| { kind: "element"; operation: "inspect" | "scope" | "highlight" | "crop" | "text"; hint: string; locator?: WebTarget };
function json(value: string): Record<string, unknown> {
const parsed: unknown = JSON.parse(value);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw Error("Use a JSON options object.");
return parsed as Record<string, unknown>;
}
function keys(value: Record<string, unknown>, allowed: readonly string[]): void {
const unknown = Object.keys(value).filter(key => !allowed.includes(key));
if (unknown.length) throw Error(`Read view contains unsupported options: ${unknown.join(", ")}. Allowed: ${allowed.join(", ")}.`);
}
/** Validate owned views before capturing; text offsets are never image coordinates.
* `ocr` adds Windows text recognition to a window overview, an observation or a viewport.
* `changes` reads a window's UI and pixels and lists only what changed since its previous observation.
* `regions` adds icons and text areas found by Cua perception to a window overview or its current observation.
* `aria` reads a window's UI as a Playwright aria snapshot whose refs are the observation's hints. */
export function parseComputerViews(views: readonly string[] = []): ComputerViews & { ocr?: true; changes?: true; regions?: true; aria?: true } {
if (views.includes("aria")) {
const view = parseView(views.filter(view => view !== "aria"));
if (view.kind !== "overview" || (view.capture.granularity ?? "ui") !== "ui")
throw Error("Use aria alone or with ui capture options such as max_elements; it renders the UI tree as a Playwright aria snapshot.");
return { kind: "overview", capture: { ...view.capture, granularity: "ui" }, aria: true };
}
if (views.includes("regions")) {
const rest = parseComputerViews(views.filter(view => view !== "regions"));
if (rest.ocr || rest.changes || rest.kind !== "overview" || rest.capture.granularity === "ui")
throw Error("Use regions alone or with image/full capture options; it finds icons and text in pixels.");
return { ...rest, regions: true };
}
if (views.includes("changes")) {
const view = parseView(views.filter(view => view !== "changes"));
if (view.kind !== "overview")
throw Error("Use changes alone (UI and pixels), or with ui, image or full and their capture options; it compares a fresh capture with the window's previous observation.");
return { kind: "overview", capture: { ...view.capture, granularity: view.capture.granularity ?? "full" }, changes: true };
}
if (!views.includes("ocr")) return parseView(views);
const view = parseView(views.filter(view => view !== "ocr"));
if (view.kind !== "overview") throw Error("Use ocr alone or with capture options; it reads text from pixels. For the text of one part of a window, Read a viewport of it with ocr.");
if (view.capture.granularity === "ui") throw Error("ocr needs pixels; use image or full granularity.");
return { ...view, ocr: true };
}
function parseView(views: readonly string[]): ComputerViews {
const first = views[0] ?? "";
if (views.some(view => view === "dom" || view.startsWith("dom:"))) {
if (views.length !== 1) throw Error("Use dom alone: a DOM snapshot has no image, UI, state or capture options.");
return { kind: "dom", options: validateWebSnapshot(first === "dom" ? {} : json(first.slice(4))) };
}
if (views.includes("state")) {
if (views.length !== 1) throw Error("Use state alone: window metadata has no pixels, UI or capture options.");
return { kind: "state" };
}
if (/^(sequence|frames|element):/.test(first)) {
if (views.length !== 1) throw Error("Use one sequence, frames or element view without overview options.");
const value = first.slice(first.indexOf(":") + 1);
if (first.startsWith("sequence:")) {
const raw = json(value);
keys(raw, ["duration_ms", "fps", "max_frames", "timeout_ms", "maxLongEdge", "scale", "region"]);
const options = raw as unknown as SequenceOptions;
validateSequence(options);
if (options.maxLongEdge !== undefined && (!Number.isSafeInteger(options.maxLongEdge) || options.maxLongEdge < 64 || options.maxLongEdge > 4096) ||
options.scale !== undefined && (!Number.isFinite(options.scale) || options.scale < 0.05 || options.scale > 1)) throw Error("Invalid sequence image size/scale.");
if (options.region) keys(options.region, ["x", "y", "width", "height"]);
return { kind: "sequence", options };
}
if (first.startsWith("element:")) {
// The target is everything after the first comma: a Playwright locator can contain commas of its own.
const comma = value.indexOf(","), operation = value.slice(0, comma < 0 ? undefined : comma), hint = comma < 0 ? "" : value.slice(comma + 1);
const usage = "Use element:inspect|scope|highlight|crop|text,<UI hint>, element:crop|highlight,<aria ref> on a DOM observation, or element:crop|highlight,<Playwright locator> on a browser window.";
if (!["inspect", "scope", "highlight", "crop", "text"].includes(operation) || !hint) throw Error(usage);
const kind = operation as Extract<ComputerViews, { kind: "element" }>["operation"];
// UI hints look like "3:ab"; Playwright aria refs look like "e12" or "f1e3".
if (/^[a-z0-9]+(:[a-z]+)?$/i.test(hint)) return { kind: "element", operation: kind, hint };
if (kind !== "crop" && kind !== "highlight") throw Error(usage);
return { kind: "element", operation: kind, hint, locator: validateWebTarget(hint) };
}
if (["all", "first", "last", "endpoints", "changed"].includes(value)) return { kind: "frames", selection: value as FrameSelection };
const raw = json(value);
// Rendering options (as, region, columns, maxLongEdge) go with any selection: select, indexes or from/to.
const rendering = validateRendering(raw);
for (const key of ["as", "region", "columns", "maxLongEdge"]) delete raw[key];
if (rendering && !Object.keys(raw).length) raw.select = "all";
if (raw.select !== undefined) {
keys(raw, ["select"]);
if (!["all", "first", "last", "endpoints", "changed"].includes(raw.select as string)) throw Error("select is all, first, last, endpoints or changed.");
return { kind: "frames", selection: raw.select as FrameSelection, ...(rendering ? { rendering } : {}) };
}
const indexes = raw.indexes;
if (indexes !== undefined) {
keys(raw, ["indexes"]);
if (!Array.isArray(indexes) || indexes.length < 1 || indexes.length > SEQUENCE_LIMITS.selected || new Set(indexes).size !== indexes.length ||
indexes.some(i => !Number.isSafeInteger(i) || i < 0 || i >= SEQUENCE_LIMITS.max_frames)) throw Error(`Select 1..${SEQUENCE_LIMITS.selected} distinct frame indexes 0..${SEQUENCE_LIMITS.max_frames - 1}.`);
} else {
keys(raw, ["from", "to"]);
if (!Number.isSafeInteger(raw.from) || !Number.isSafeInteger(raw.to) || (raw.from as number) < 0 ||
(raw.to as number) < (raw.from as number) || (raw.to as number) >= SEQUENCE_LIMITS.max_frames || (raw.to as number) - (raw.from as number) >= SEQUENCE_LIMITS.selected)
throw Error(`Use an inclusive frame range within 0..${SEQUENCE_LIMITS.max_frames - 1} of at most ${SEQUENCE_LIMITS.selected} frames.`);
}
return { kind: "frames", selection: raw as FrameSelection, ...(rendering ? { rendering } : {}) };
}
const capture: UiCaptureOptions = {};
for (const view of views) {
if (view.startsWith("region:")) {
if (views.length !== 1) throw Error("Use one rectangular region view without overview options.");
const values = view.slice(7).split(",");
const [x, y, width, height] = values.map(Number);
if (values.length !== 4 || values.some(value => !value.trim()) ||
![x, y, width, height].every(Number.isFinite) || x < 0 || y < 0 || width <= 0 || height <= 0)
throw Error("Use region:x,y,width,height in local overview-image pixels.");
return { kind: "region", region: { x, y, width, height } };
}
if (view.startsWith("capture:")) {
const options = json(view.slice(8));
// granularity may also come from the image/ui/full views; both forms mean the same.
keys(options, ["granularity", "maxLongEdge", "scale", "max_elements", "max_depth", "hints", "guides"]);
const duplicate = Object.keys(options).filter(key => key in capture);
if (duplicate.length) throw Error(`${duplicate.join(", ")} is given twice: once as a view and once in capture.`);
Object.assign(capture, options);
} else if (["image", "ui", "full"].includes(view)) {
if (capture.granularity !== undefined) throw Error(`Choose one of image, ui or full; got ${capture.granularity} and ${view}.`);
capture.granularity = view as UiCaptureOptions["granularity"];
} else throw Error(`Unknown view ${view}. Choose image/ui/full with optional capture options, or one region/element/sequence/frames view.`);
}
validateUiCapture(capture);
return { kind: "overview", capture };
}