Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

Computer Use draws its tool calls like Pi Agent IDE's tools (LPT-856)

alexshpunt-clanker[bot]3ea1c1397bea
diff --git a/src/computer-action-renderer.ts b/src/computer-action-renderer.ts
deleted file mode 100644
index 313e274..0000000
--- a/src/computer-action-renderer.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { Text } from "@earendil-works/pi-tui";
-import type { TextEditorPluginApi } from "pi-agent-ide/api/text-editor";
-
-type Registration = Parameters<TextEditorPluginApi["addToolRenderer"]>[0];
-const record = (value: unknown): Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
-// App text is data, never terminal control sequences. Keep newlines for the existing receipt layout.
-const plain = (value: string) => value.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/g, "");
-
-/** Preserve computer receipts without replacing IDE's renderer for unrelated edits. */
-export function computerActionRenderer(tool: string): Registration {
- return {
- tool,
- matches(value, phase) {
- if (phase === "call") return typeof record(value).path === "string" && String(record(value).path).startsWith("computer:");
- return record(record(record(record(value).details).metadata).semanticAction).kind === "computer-action";
- },
- renderCall(args) {
- return new Text(`${tool} ${plain(String(record(args).path))}`, 0, 0);
- },
- renderResult(result) {
- // content is the provider's safe summary; metadata.receipt can contain private native diagnostics.
- const summary = result.content.find(block => block.type === "text")?.text ?? "";
- const body = summary.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "");
- return new Text(plain(body), 0, 0);
- },
- };
-}
diff --git a/src/computer-read-renderer.ts b/src/computer-read-renderer.ts
deleted file mode 100644
index e26edde..0000000
--- a/src/computer-read-renderer.ts
+++ /dev/null
@@ -1,18 +0,0 @@
-import { Text } from "@earendil-works/pi-tui";
-import { createReadResultRenderer, type ReadResultRenderer } from "pi-agent-ide/api/read";
-
-const fallback = createReadResultRenderer({ kind: "source" });
-const record = (value: unknown): Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
-const clean = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, "").slice(0, 200);
-
-/** Show a short provenance line above Playwright page images; other reads retain IDE's source renderer. */
-export const computerReadRenderer: ReadResultRenderer = (result, options, theme, context) => {
- const details = record(result.details), computer = record(details.computer), image = record(computer.agent_image);
- if (details.resolvedBy !== "computer" || image.route !== "playwright_screenshot") return fallback(result, options, theme, context);
- const output = record(image.output), state = record(computer.state);
- // Pi appends the enlarged image below this component; keep the receipt short enough to stay above it.
- return new Text([
- `Playwright ${clean(image.kind)} screenshot: ${output.width}x${output.height} PNG | page pixels, input=none`,
- `${clean(state.title)} | ${clean(state.url)}`,
- ].join("\n"), 0, 0);
-};
diff --git a/src/computer-renderers.ts b/src/computer-renderers.ts
new file mode 100644
index 0000000..b46cf81
--- /dev/null
+++ b/src/computer-renderers.ts
@@ -0,0 +1,311 @@
+import type { Component } from "@earendil-works/pi-tui";
+import type { Theme } from "@earendil-works/pi-coding-agent";
+import {
+ actionState, elementRows, firstReason, HeadingLine, OutcomeCard, oneLine, problemNotes, record, size, successorNotes, targetNames,
+ type ComputerState, type DescribeAddress, type Heading, type Note, type Outcome,
+} from "./computer-tui.ts";
+
+/** A tool result as Pi gives it to a renderer; only the typed fields are read, never the receipt text. */
+interface ToolResult { content?: { type: string; text?: string }[]; details?: unknown; isError?: boolean }
+interface RenderOptions { expanded: boolean; isPartial: boolean }
+interface RenderContext { isError?: boolean; lastComponent?: Component; args?: unknown }
+
+const array = (value: unknown): unknown[] => Array.isArray(value) ? value : [];
+const str = (value: unknown): string | undefined => typeof value === "string" && value.trim() ? value : undefined;
+const num = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? value : undefined;
+const last = (address: string) => address.replace(/^computer:/, "").replace(/\?.*$/, "");
+
+/** The window a person is looking at: its title when this run listed one, otherwise its short address. */
+function windowLabel(describe: DescribeAddress | undefined, address: unknown): string {
+ const source = str(address);
+ if (!source) return "window";
+ const title = describe?.(source)?.title;
+ return title ? `“${oneLine(title, 60)}”` : last(source);
+}
+const machineOf = (describe: DescribeAddress | undefined, at: unknown) => {
+ const source = str(at);
+ return source ? describe?.(source)?.machine ?? /^computer:machine\/([^/?]+)/.exec(source)?.[1] ?? /^computer:([a-z0-9_-]+)\/[a-z]\d/.exec(source)?.[1] : undefined;
+};
+const base = (path: string) => path.split(/[\\/]/).filter(Boolean).pop() ?? path;
+/** Where an input is aimed: a ref or locator, or a point. */
+const aimed = (args: Record<string, unknown>) => args.target !== undefined ? oneLine(args.target, 60) : args.x !== undefined ? `${args.x},${args.y}` : "";
+/** Argument names whose values are what the agent typed or pasted; headings and details never repeat them. */
+const LITERAL = new Set(["text", "content", "value", "values", "data", "script", "code"]);
+
+/** Exact scalar arguments for the expanded heading, without literal input. */
+function argumentDetails(args: Record<string, unknown>): [string, string][] {
+ return Object.entries(args).filter(([key, value]) => !LITERAL.has(key) && ["string", "number", "boolean"].includes(typeof value))
+ .map(([key, value]) => [key, oneLine(value, 120)] as [string, string]);
+}
+
+/** The heading of one typed computer_* call: what it acts on, then the options that matter. */
+export function typedHeading(tool: string, rawArgs: unknown, describe?: DescribeAddress): Heading {
+ const args = record(rawArgs), window = windowLabel(describe, args.window);
+ const base_: Heading = { tool, details: argumentDetails(args) };
+ const heading = (primary: string, ...qualifiers: (string | undefined | false)[]): Heading =>
+ ({ ...base_, primary: { text: primary }, qualifiers: qualifiers.filter((q): q is string => typeof q === "string" && q.length > 0) });
+ switch (tool) {
+ case "computer_open": {
+ const what = str(args.app) ?? (str(args.path) ? base(String(args.path)) : undefined) ?? str(args.url) ?? str(args.browser) ??
+ (args.sandbox ? "sandbox" : args.isolated_session ? "isolated session" : args.webview2_debugging ? "WebView2 debugging" : undefined) ?? window;
+ return heading(what, machineOf(describe, args.at), str(args.browser) && str(args.url) ? str(args.browser) : undefined);
+ }
+ case "computer_window": {
+ const action = String(args.action ?? "");
+ const detail = action === "move" ? `move ${args.x},${args.y}` : action === "resize" ? `resize ${args.width}×${args.height}` : action === "name" ? `name ${args.as}` : action;
+ return heading(window, detail);
+ }
+ case "computer_view": {
+ const at = args.x !== undefined || args.y !== undefined ? `at ${args.x ?? "_"},${args.y ?? "_"}` : undefined;
+ const dims = args.width !== undefined || args.height !== undefined ? `${args.width ?? "_"}×${args.height ?? "_"}` : undefined;
+ return heading(last(String(args.viewport ?? "viewport")), at, dims);
+ }
+ case "computer_wait": return heading(window, String(args.mode ?? "wait"), `${array(args.conditions).length} condition(s)`);
+ case "computer_browser": return heading(window, String(args.action ?? ""), str(args.url) ?? str(args.tab));
+ case "computer_files": return heading(String(args.machine ?? "machine"), String(args.action ?? ""), str(args.from) ?? str(args.name));
+ case "computer_clipboard": return heading(String(args.machine ?? "machine"), String(args.action ?? ""), str(args.image));
+ case "computer_record": return heading(windowLabel(describe, args.at), String(args.action ?? ""), num(args.seconds) !== undefined ? `${args.seconds} s` : num(args.duration_ms) !== undefined ? `${args.duration_ms} ms` : undefined);
+ case "computer_click": return heading(window, aimed(args), num(args.count) ? `×${args.count}` : undefined, str(args.button) && args.button !== "left" ? String(args.button) : undefined);
+ case "computer_hover": return heading(window, aimed(args));
+ case "computer_drag": return heading(window, `${aimed(record(args.from)) || "?"} → ${aimed(record(args.to)) || "?"}`);
+ case "computer_scroll": return heading(window, aimed(args), `${args.delta_x ?? 0},${args.delta_y ?? 0}`);
+ case "computer_type": return heading(window, aimed(args), `${String(args.text ?? "").length} chars`, args.submit ? "+ Enter" : undefined);
+ case "computer_press": return heading(window, String(args.key ?? ""), num(args.hold_ms) ? `${args.hold_ms} ms` : undefined);
+ case "computer_select": {
+ const choice = Array.isArray(args.menu) ? (args.menu as string[]).map(item => oneLine(item, 30)).join(" > ")
+ : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : array(args.values).length ? `${array(args.values).length} value(s)` : "";
+ return heading(window, aimed(args), choice);
+ }
+ case "computer_timeline": return heading(window, `${array(args.events).length} events`);
+ case "computer_display": return heading(String(args.display ?? "display"), `${args.width}×${args.height}`);
+ default: return heading(last(String(args.path ?? args.window ?? args.at ?? "")));
+ }
+}
+
+const KINDS: Record<string, string> = { w: "window", o: "observation", s: "sequence", a: "app", b: "sandbox", d: "display" };
+/** A short computer address in words: `computer:local/w1tt` is "window w1tt" on machine local. */
+function addressWords(address: string): { what: string; machine?: string } {
+ const short = /^computer:([a-z0-9_-]+)\/([wosabd])(\d+[a-z]{2})(?:\/([a-z0-9_-]+))?$/.exec(address.replace(/\?.*$/, ""));
+ if (short) return { what: `${KINDS[short[2]]} ${short[2]}${short[3]}${short[4] ? `/${short[4]}` : ""}`, machine: short[1] };
+ const machine = /^computer:machine\/([^/?]+)(?:\/(windows|apps|displays))?/.exec(address);
+ if (machine) return { what: machine[2] ? `${machine[1]} ${machine[2]}` : `machine ${machine[1]}`, machine: machine[1] };
+ if (address === "computer:session") return { what: "session" };
+ return { what: last(address) };
+}
+
+/** The heading of an IDE tool call on a computer address (Read/Search/Write/Insert/Delete and the standalone twins). */
+export function resourceHeading(tool: string, rawArgs: unknown, describe?: DescribeAddress): Heading {
+ const args = record(rawArgs), path = str(args.path) ?? "";
+ const views = array(args.views).filter((v): v is string => typeof v === "string");
+ const { what, machine } = addressWords(path);
+ const title = path.startsWith("computer:") ? describe?.(path)?.title : undefined;
+ const qualifiers: string[] = [];
+ if (typeof args.query === "string") qualifiers.push(`“${oneLine(args.query, 60)}”`);
+ if (views.length) qualifiers.push(`views ${views.join(",")}`);
+ if (machine && !/^machine /.test(what) && !/^[a-z0-9_-]+ (windows|apps|displays)$/.test(what)) qualifiers.push(machine);
+ return { tool, primary: { text: title ? `“${oneLine(title, 60)}”` : what }, qualifiers, details: [["path", path], ...argumentDetails(args).filter(([key]) => key !== "path")] };
+}
+
+const sizeFact = (bounds: unknown) => { const b = record(bounds); return size(b.width, b.height); };
+
+/** Facts, notes and rows of a window-management receipt. */
+function managementFacts(state: Record<string, unknown>): { facts: string[]; notes: Note[] } {
+ const operation = str(state.operation), observed = record(record(state.observed).bounds), requested = record(state.requested_bounds);
+ const facts: string[] = [];
+ if (operation) facts.push(operation);
+ const now = size(observed.width, observed.height), want = size(requested.width, requested.height);
+ if (now) facts.push(now);
+ const notes: Note[] = [];
+ if (now && want && now !== want && state.postcondition_confirmed === true) notes.push({ tone: "muted", text: `Windows settled at ${now} for the requested ${want}.` });
+ return { facts, notes };
+}
+
+/** The outcome of one typed computer_* action, from the semantic data its result carries. */
+export function actionOutcome(result: ToolResult, isError: boolean, describe?: DescribeAddress): Outcome {
+ const data = record(record(record(result.details).metadata).semanticAction);
+ if (data.kind !== "computer-action") {
+ // A result that is not a computer receipt (a schema or runtime failure): say that it failed, and why in one line.
+ const text = result.content?.find(block => block.type === "text")?.text;
+ return { state: isError ? "failed" : "delivered", notes: isError && firstReason(text) ? [{ tone: "error", text: firstReason(text)! }] : [] };
+ }
+ const script = record(data.script), state = actionState(data, isError);
+ const windowAddress = str(script.window) ?? str(data.source);
+ const named = targetNames(describe, windowAddress);
+ // The heading already names the window by its title; the status line adds only where it ran.
+ const facts = [str(data.machine) ?? named.machine ?? machineOf(describe, windowAddress)].filter((f): f is string => !!f);
+ const notes = problemNotes(state, data);
+ const rows: string[] = [];
+ const details: [string, string][] = [];
+ const stateRecord = record(data.state);
+ if (typeof stateRecord.mutation_attempted === "boolean") {
+ const managed = managementFacts(stateRecord);
+ facts.push(...managed.facts); notes.push(...managed.notes);
+ }
+ if (state === "delivered" || state === "observed") {
+ const ui = record(data.ui), image = record(record(data.agent_image).output);
+ const dims = size(image.width, image.height);
+ if (dims) facts.push(`image ${dims}`);
+ if (num(ui.collected) !== undefined) { facts.push(`UI ${ui.collected} element${ui.collected === 1 ? "" : "s"}${ui.truncated === true ? " (truncated)" : ""}`); rows.push(...elementRows(ui.elements)); }
+ const successor = successorNotes(data.successor);
+ notes.push(...successor.notes); rows.push(...successor.rows);
+ }
+ const code = str(script.code) ?? str(record(data.error).code);
+ if (code) details.push(["code", code]);
+ if (str(script.stage)) details.push(["stage", String(script.stage)]);
+ if (str(script.window)) details.push(["window", String(script.window)]);
+ if (str(script.observation)) details.push(["observation", String(script.observation)]);
+ if (str(data.mode)) details.push(["mode", String(data.mode)]);
+ const delivery = str(stateRecord.delivery_mode);
+ if (delivery) details.push(["delivery", delivery]);
+ if (str(script.application_effect)) details.push(["application effect", String(script.application_effect)]);
+ return { state, facts, notes, rows, details };
+}
+
+/** The human-readable lines of a result's text, without the model-only envelope and attached guides. */
+function contentRows(result: ToolResult): string[] {
+ const text = (result.content ?? []).filter(block => block.type === "text" && typeof block.text === "string" && !block.text.startsWith("\n\n---\n\n# Guide:"))
+ .map(block => block.text!.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "")).join("\n");
+ return text.split(/\r?\n/).map(line => line.trimEnd()).filter(Boolean);
+}
+
+const READ_STATES = new Set<ComputerState>(["observed", "listed", "delivered", "closed", "refused", "unknown"]);
+const stateOf = (status: unknown, isError: boolean, fallback: ComputerState): ComputerState =>
+ READ_STATES.has(status as ComputerState) ? status as ComputerState : isError ? "failed" : fallback;
+
+/** The outcome of a Read of a computer address, whether through the IDE or computer_read. */
+export function readOutcome(result: ToolResult, isError: boolean, describe?: DescribeAddress): Outcome {
+ const outer = record(result.details), d = record(outer.computer ?? outer.payload ?? result.details);
+ const state = stateOf(d.status, isError, "observed");
+ const source = str(d.window_ref) ? `computer:window/${d.window_ref}` : str(outer.source);
+ const named = targetNames(describe, source);
+ const facts = [str(d.machine) ?? named.machine, named.title].filter((f): f is string => !!f);
+ const notes: Note[] = [], rows: string[] = [], details: [string, string][] = [];
+ const image = record(record(d.agent_image).output), dims = size(image.width, image.height);
+ if (dims) facts.push(`image ${dims}`);
+ const page = record(d.state);
+ if (record(d.agent_image).route === "playwright_screenshot") {
+ facts.push(`${oneLine(record(d.agent_image).kind ?? "page", 20)} screenshot`);
+ if (str(page.title)) facts.push(`“${oneLine(page.title, 60)}”`);
+ if (str(page.url)) facts.push(oneLine(page.url, 80));
+ }
+ const crop = record(d.screenshot_crop);
+ if (size(crop.width, crop.height)) facts.push(`crop ${size(crop.width, crop.height)}`);
+ const ui = record(d.ui);
+ if (num(ui.collected) !== undefined) {
+ facts.push(`UI ${ui.collected}${num(ui.native_total) !== undefined && ui.native_total !== ui.collected ? `/${ui.native_total}` : ""} element${ui.collected === 1 ? "" : "s"}${ui.truncated === true ? " (truncated)" : ""}`);
+ rows.push(...elementRows(ui.elements));
+ }
+ const dom = record(d.dom);
+ if (str(dom.snapshot)) {
+ facts.push(`page ${oneLine(dom.title ?? dom.url ?? "", 50)}`.trim());
+ rows.push(...String(dom.snapshot).split("\n").map(line => line.trimEnd()).filter(Boolean));
+ if (dom.truncated === true) notes.push({ tone: "warning", text: "The page snapshot is truncated; narrow it with dom:{target,depth}." });
+ }
+ const lines = array(d.text_lines);
+ if (lines.length) { facts.push(`OCR ${lines.length} line${lines.length === 1 ? "" : "s"}`); rows.push(...lines.map(line => oneLine(array(record(line).words).map(w => record(w).text).join(" ")))); }
+ const regions = array(d.visual_regions);
+ if (regions.length) facts.push(`${regions.length} region${regions.length === 1 ? "" : "s"}`);
+ const element = record(d.element_text);
+ if (str(element.text)) { facts.push(`element ${oneLine(element.role ?? "", 20)} ${oneLine(element.name ?? "", 40)}`.trim()); rows.push(...String(element.text).split(/\r?\n/)); if (element.cut === true) notes.push({ tone: "muted", text: "The element's text is cut." }); }
+ const sequence = record(d.sequence);
+ if (num(sequence.frame_count) !== undefined) facts.push(`${sequence.frame_count} frames`);
+ const changes = successorNotes(d.changes); notes.push(...changes.notes); rows.push(...changes.rows);
+ const stateRecord = record(d.state);
+ if (d.window_state_kind === "management" || stateRecord.bounds !== undefined) {
+ const sample = d.window_state_kind === "management" ? record(stateRecord.state) : stateRecord, bs = sizeFact(sample.bounds);
+ if (bs) facts.push(bs);
+ if (sample.minimized === true) facts.push("minimized"); else if (sample.maximized === true) facts.push("maximized");
+ }
+ const windows = array(record(d.state).windows), apps = array(record(d.state).apps), displays = array(record(d.state).displays);
+ if (windows.length || apps.length || displays.length) {
+ const [kind, items] = windows.length ? ["window", windows] as const : apps.length ? ["app", apps] as const : ["display", displays] as const;
+ facts.push(`${items.length} ${kind}${items.length === 1 ? "" : "s"}`);
+ rows.push(...items.map(item => listedRow(kind, record(item), describe)));
+ }
+ const verification = record(d.verification);
+ if (array(verification.conditions).length) {
+ const conditions = array(verification.conditions).map(raw => record(raw));
+ facts.push(`${conditions.filter(c => c.verdict === "satisfied").length}/${conditions.length} conditions`);
+ rows.push(...conditions.map((c, i) => `${i + 1}. ${oneLine(c.verdict ?? "")}${str(c.reason) ? ` — ${oneLine(c.reason, 120)}` : ""}`));
+ }
+ if (state === "refused" || state === "unknown" || state === "failed") {
+ const reason = firstReason(result.content?.find(block => block.type === "text")?.text?.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "").split("\n").slice(1).join("\n"));
+ notes.push(...problemNotes(state, {}, reason));
+ }
+ // A resource with no typed observation fields, such as the session descriptor: its own text is the readable form.
+ if (!facts.length && !rows.length && !notes.length && (state === "observed" || state === "listed")) rows.push(...contentRows(result));
+ const captured = str(record(d.source).captured_at);
+ if (captured) details.push(["captured", captured]);
+ const freshness = str(record(record(d.source).visibility).paint_freshness);
+ if (freshness) details.push(["paint", freshness]);
+ if (str(d.mode)) details.push(["mode", String(d.mode)]);
+ return { state, facts, notes, rows, details };
+}
+
+function listedRow(kind: "window" | "app" | "display", item: Record<string, unknown>, describe?: DescribeAddress): string {
+ void describe;
+ if (kind === "window") return [str(item.app_name), str(item.title) ? `“${oneLine(item.title, 70)}”` : undefined, sizeFact(item.bounds), item.owned === true ? "yours" : undefined].filter(Boolean).join(" ");
+ if (kind === "app") return oneLine(item.name ?? item.launch_path ?? "");
+ return [str(item.display_id) ?? str(item.name), sizeFact(item.bounds), item.primary === true ? "primary" : undefined].filter(Boolean).join(" ");
+}
+
+// --- Components Pi mounts ---
+
+const reuse = <T extends Component>(previous: Component | undefined, type: new (...args: never[]) => T): T | undefined => previous instanceof type ? previous : undefined;
+
+/** A heading component, reusing the previous one so a redraw does not flicker. */
+export function headingComponent(heading: Heading, theme: Theme, expanded: boolean, previous?: Component): Component {
+ const component = reuse(previous, HeadingLine) as HeadingLine | undefined;
+ if (component) { component.update(heading, theme, expanded); return component; }
+ return new HeadingLine(heading, theme, expanded);
+}
+/** An outcome component, reusing the previous one. */
+export function outcomeComponent(outcome: Outcome, theme: Theme, expanded: boolean, previous?: Component): Component {
+ const component = reuse(previous, OutcomeCard) as OutcomeCard | undefined;
+ if (component) { component.update(outcome, theme, expanded); return component; }
+ return new OutcomeCard(outcome, theme, expanded);
+}
+
+/** renderCall and renderResult of one typed computer_* tool. */
+export function typedRenderers(tool: string, describe?: DescribeAddress) {
+ return {
+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>
+ headingComponent(typedHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),
+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>
+ outcomeComponent(options.isPartial ? { state: "running" } : actionOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent),
+ };
+}
+
+/** Pi Agent IDE's Read result renderer for computer addresses. */
+export function computerReadRenderer(describe?: DescribeAddress) {
+ return (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>
+ outcomeComponent(options.isPartial ? { state: "running" } : readOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent);
+}
+
+/** Pi Agent IDE's Search result renderer for computer addresses. Search results carry the listing under `payload`. */
+export const computerSearchRenderer = computerReadRenderer;
+
+/** A Write, Insert or Delete on a computer address: refused or closed, with a heading that names the target. */
+export function computerActionRenderer(tool: "write" | "insert" | "delete", describe?: DescribeAddress) {
+ const owns = (value: unknown) => typeof record(value).path === "string" && String(record(value).path).startsWith("computer:");
+ return {
+ tool,
+ matches: (value: unknown, phase: "call" | "result") => phase === "call" ? owns(value)
+ : record(record(record(value).details).metadata).semanticAction !== undefined && record(record(record(record(value).details).metadata).semanticAction).kind === "computer-action",
+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>
+ headingComponent(resourceHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),
+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>
+ outcomeComponent(options.isPartial ? { state: "running" } : actionOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent),
+ };
+}
+
+/** renderCall and renderResult of the standalone computer_read / computer_search / computer_delete tools. */
+export function resourceRenderers(tool: "computer_read" | "computer_search" | "computer_delete", describe?: DescribeAddress) {
+ return {
+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>
+ headingComponent(resourceHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),
+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>
+ outcomeComponent(options.isPartial ? { state: "running" } : tool === "computer_delete" ? actionOutcome(result, context.isError === true, describe) : readOutcome(result, context.isError === true, describe),
+ theme, options.expanded, context.lastComponent),
+ };
+}
diff --git a/src/computer-resources.ts b/src/computer-resources.ts
index 2a4e4b8..6b3c8d4 100644
--- a/src/computer-resources.ts
+++ b/src/computer-resources.ts
@@ -424,6 +424,19 @@ export class ComputerResources {
shorten(text: string): string {
return this.shorts.shorten(text, (kind, ref) => this.place(requireComputerAddress(computerSource(kind, ref)))?.machine);
}
+ /**
+ * What a person calls the thing an address points at: its machine and, for a window, the title its last listing
+ * gave. For display only: it never throws, issues no alias and starts no native work, so an address from an
+ * earlier run or a window that is gone simply has no name.
+ */
+ describe(source: string): { machine?: string; title?: string } | undefined {
+ try {
+ const address = this.address(source);
+ const place = this.place(address);
+ const title = address.kind === "window" || address.kind === "viewport" ? this.titles.get(address.ref) : undefined;
+ return place || title ? { ...(place ? { machine: place.machine } : {}), ...(title ? { title } : {}) } : undefined;
+ } catch { return undefined; }
+ }
/** The owner of a window, or of one machine's display. */
private async route(cwd: string, target: ObservationTarget, signal?: AbortSignal): Promise<Owner> {
if (target.window_ref !== undefined)
diff --git a/src/computer-tools.ts b/src/computer-tools.ts
index 30213ce..b2a2aa1 100644
--- a/src/computer-tools.ts
+++ b/src/computer-tools.ts
@@ -1,13 +1,11 @@
import { Type, type JsonValue, type Static } from "@earendil-works/pi-ai";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { Text } from "@earendil-works/pi-tui";
import { ComputerRequestError, refKind, type ComputerResources, type InputPlan, type InputPoint } from "./computer-resources.ts";
import type { ComputerAddress } from "./computer-address.ts";
-import { computerActionRenderer } from "./computer-action-renderer.ts";
+import { resourceRenderers, typedRenderers } from "./computer-renderers.ts";
import { parseComputerViews } from "./computer-views.ts";
import { COMPUTER_DELETE_HELP, computerAction, deleteComputer } from "./provider.ts";
-const plain = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, " ");
type Action = Awaited<ReturnType<typeof computerAction>>;
/**
@@ -492,16 +490,14 @@ export function timelinePlan(params: Static<typeof timelineParameters>): InputPl
export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResources): void {
const namespace = computerNamespace();
const common = { exposure: "deferred" as const, namespace, outputSchema: receiptSchema };
- const rendered = (name: string, call: (args: Record<string, unknown>) => string) => ({
- renderCall: (args: unknown) => new Text(`${name} ${plain(call(args as Record<string, unknown>))}`, 0, 0),
- renderResult: computerActionRenderer(name).renderResult,
- });
+ const describe = (source: string) => resources.describe(source);
+ const rendered = (name: string) => typedRenderers(name, describe);
pi.registerTool({
...common, name: "computer_open", label: "Computer open",
description: "Use computer_open to start an app, a Playwright browser, a Windows Sandbox or an isolated session on a machine, or to attach Playwright to a running browser window. The result gives the new window or sandbox address.",
parameters: openParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_open", args => [args.at, args.app ?? args.path ?? args.url ?? args.browser ?? (args.sandbox ? "sandbox" : args.isolated_session ? "isolated session" : "")].filter(Boolean).join(" ")),
+ ...rendered("computer_open"),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.at, "open", () => {
@@ -516,7 +512,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address without keys or clicks, or to give it a short name.",
parameters: windowParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : args.action === "name" ? ` ${args.as}` : ""}`),
+ ...rendered("computer_window"),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.window, params.action, () => {
@@ -530,7 +526,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_view to make, move or resize a viewport: a named rectangle of a window that follows the window. The result shows the viewport at the window's own resolution. Give its address to the computer_* input tools to act at points of that image and see what changed only there; Read it to look again.",
parameters: viewParameters,
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_view", args => `${args.viewport}${args.x !== undefined || args.y !== undefined ? ` at ${args.x ?? "_"},${args.y ?? "_"}` : ""}${args.width !== undefined || args.height !== undefined ? ` ${args.width ?? "_"}x${args.height ?? "_"}` : ""}`),
+ ...rendered("computer_view"),
async execute(_id, params, signal, _update, ctx) {
const { viewport, ...rect } = params;
return actionResult(await computerAction(resources, viewport, "view", () => resources.address(viewport),
@@ -542,7 +538,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_wait to wait until a window's UI or a browser page meets conditions, or to check them once, without sending input. The last sample becomes the window's current observation.",
parameters: waitParameters,
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_wait", args => `${args.window} ${args.mode ?? "wait"} ${(args.conditions as unknown[] | undefined)?.length ?? 0} condition(s)`),
+ ...rendered("computer_wait"),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.window, params.mode === "check" ? "verify" : "wait_for", () => {
@@ -558,7 +554,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_browser to navigate, manage tabs, answer JavaScript dialogs, upload or download files, and run page JavaScript in a browser window. The result shows the page's fresh snapshot.",
parameters: browserParameters,
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true },
- ...rendered("computer_browser", args => `${args.window} ${args.action}${args.url ? ` ${args.url}` : args.tab ? ` ${args.tab}` : ""}`),
+ ...rendered("computer_browser"),
async execute(_id, params, signal, _update, ctx) {
const { window, ...command } = params;
return actionResult(await computerAction(resources, window, params.action, () => resources.address(window),
@@ -570,7 +566,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_files to copy files between the workspace and this session's own folder on a machine, or to list that folder, so apps there can open or save them.",
parameters: filesParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_files", args => `${args.machine} ${args.action}${args.from ? ` ${args.from}` : args.name ? ` ${args.name}` : ""}`),
+ ...rendered("computer_files"),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.machine, params.action, () => {
@@ -584,7 +580,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_clipboard to read or replace a machine's clipboard in YOLO mode, with text or an image.",
parameters: clipboardParameters,
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
- ...rendered("computer_clipboard", args => `${args.machine} ${args.action}${args.image ? ` ${args.image}` : ""}`),
+ ...rendered("computer_clipboard"),
async execute(_id, params, signal, _update, ctx) {
let command: Record<string, unknown> = {};
return actionResult(await computerAction(resources, params.machine, `clipboard_${params.action}`, () => {
@@ -598,7 +594,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_record to record a machine's display to video or its playback audio, to capture frames of a window or display over time for motion and animation, or to watch a window or viewport in the background while acting on it, such as panning a panorama or dragging a 3D camera.",
parameters: recordParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_record", args => `${args.at} ${args.action}${args.seconds ? ` ${args.seconds} s` : args.duration_ms ? ` ${args.duration_ms} ms` : ""}`),
+ ...rendered("computer_record"),
async execute(_id, params, signal, _update, ctx) {
let run: RecordRun = { command: {} };
return actionResult(await computerAction(resources, params.at, params.action, () => {
@@ -628,56 +624,56 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
...common, name: "computer_click", label: "Computer click",
description: "Use computer_click to click an element or a point of a window: any button, double or triple clicks, with modifier keys. The result lists what changed.",
parameters: clickParameters, annotations: inputAnnotations,
- ...rendered("computer_click", args => `${args.window} ${aimed(args)}${args.count ? ` x${args.count}` : ""}${args.button && args.button !== "left" ? ` ${args.button}` : ""}`),
+ ...rendered("computer_click"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "click", () => clickPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_hover", label: "Computer hover",
description: "Use computer_hover to rest the pointer on an element or a point without clicking, for tooltips, hover menus and hover states.",
parameters: hoverParameters, annotations: inputAnnotations,
- ...rendered("computer_hover", args => `${args.window} ${aimed(args)}`),
+ ...rendered("computer_hover"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "hover", () => hoverPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_drag", label: "Computer drag",
description: "Use computer_drag to press a button at one element or point, move to another, optionally through a path, and release there, for sliders, drag and drop and drawing.",
parameters: dragParameters, annotations: inputAnnotations,
- ...rendered("computer_drag", args => `${args.window} ${aimed(args.from as Record<string, unknown>)} → ${aimed(args.to as Record<string, unknown>)}`),
+ ...rendered("computer_drag"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "drag", () => dragPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_scroll", label: "Computer scroll",
description: "Use computer_scroll to turn the mouse wheel over an element, a point or the middle of a window or page.",
parameters: scrollParameters, annotations: inputAnnotations,
- ...rendered("computer_scroll", args => `${args.window} ${aimed(args)} ${args.delta_x ?? 0},${args.delta_y ?? 0}`),
+ ...rendered("computer_scroll"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "scroll", () => scrollPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_type", label: "Computer type",
description: "Use computer_type to type text into a field or the focused element, or to replace a field's value, optionally pressing Enter after it.",
parameters: typeParameters, annotations: inputAnnotations,
- ...rendered("computer_type", args => `${args.window} ${aimed(args)} ${String(args.text ?? "").length} chars${args.submit ? " + Enter" : ""}`),
+ ...rendered("computer_type"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "type", () => typePlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_press", label: "Computer press",
description: "Use computer_press to press a key or shortcut such as Enter or ctrl+s, or to hold keys down for a time.",
parameters: pressParameters, annotations: inputAnnotations,
- ...rendered("computer_press", args => `${args.window} ${args.key}${args.hold_ms ? ` for ${args.hold_ms} ms` : ""}`),
+ ...rendered("computer_press"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "press", () => pressPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_select", label: "Computer select",
description: "Use computer_select to choose options in a list or select element, set a checkbox, radio or toggle, or open an app's menu item.",
parameters: selectParameters, annotations: inputAnnotations,
- ...rendered("computer_select", args => `${args.window} ${args.menu ? (args.menu as string[]).join(" > ") : `${args.target ?? ""} ${args.values ? (args.values as string[]).join(", ") : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : ""}`}`),
+ ...rendered("computer_select"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "select", () => selectPlan(params), ctx.cwd, signal); },
});
pi.registerTool({
...common, name: "computer_timeline", label: "Computer timeline",
description: "Use computer_timeline to play timed key, button, pointer, wheel and pen events in one call, for games, held keys across moves and pressure-sensitive strokes.",
parameters: timelineParameters, annotations: inputAnnotations,
- ...rendered("computer_timeline", args => `${args.window} ${(args.events as unknown[] | undefined)?.length ?? 0} events`),
+ ...rendered("computer_timeline"),
async execute(_id, params, signal, _update, ctx) { return input(params.window, "timeline", () => timelinePlan(params), ctx.cwd, signal); },
});
pi.registerTool({
@@ -685,7 +681,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou
description: "Use computer_display to change a display's resolution in YOLO mode, for example to test an app at another size; closing computer use restores it.",
parameters: displayParameters,
annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },
- ...rendered("computer_display", args => `${args.display} ${args.width}x${args.height}`),
+ ...rendered("computer_display"),
async execute(_id, params, signal, _update, ctx) {
return actionResult(await computerAction(resources, params.display, "resize", () => resources.address(params.display),
address => resources.perform(address, ctx.cwd, JSON.stringify({ action: "resize", width: params.width, height: params.height }), signal)));
@@ -700,8 +696,7 @@ export function registerComputerDelete(pi: ExtensionAPI, resources: ComputerReso
description: "Use computer_delete to close an app window, or to release a sandbox, isolated session, machine or all of computer use.",
parameters: Type.Object({ path: Type.String({ description: COMPUTER_DELETE_HELP }) }, { additionalProperties: false }),
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
- renderCall: args => new Text(`computer_delete ${plain(args.path)}`, 0, 0),
- renderResult: computerActionRenderer("computer_delete").renderResult,
+ ...resourceRenderers("computer_delete", resource => resources.describe(resource)),
async execute(_id, params, signal, _update, ctx) {
return actionResult(await deleteComputer(resources, params.path, params, ctx.cwd, signal));
},
diff --git a/src/computer-tui.ts b/src/computer-tui.ts
new file mode 100644
index 0000000..f435829
--- /dev/null
+++ b/src/computer-tui.ts
@@ -0,0 +1,208 @@
+import type { Theme, ThemeColor } from "@earendil-works/pi-coding-agent";
+import { keyText } from "@earendil-works/pi-coding-agent";
+import { type Component, truncateToWidth, visibleWidth, wrapTextWithAnsi } from "@earendil-works/pi-tui";
+
+/**
+ * What a person sees of a Computer Use call and its outcome in the native Pi TUI.
+ *
+ * Everything here is presentation: it reads typed result fields (never the agent's receipt text) and draws them in the
+ * conventions of Pi Agent IDE's tool cards, a bold tool title with a primary value and muted qualifiers, and a status
+ * line with an icon, so the two kinds of tool read alike. Nothing here changes what the agent receives.
+ */
+
+/** The state of a call, as a person reads it. Refused means nothing was sent; unknown means it may have been. */
+export type ComputerState = "running" | "observed" | "listed" | "delivered" | "closed" | "refused" | "unknown" | "failed";
+
+const STATES: Record<ComputerState, { icon: string; tone: ThemeColor }> = {
+ running: { icon: "●", tone: "accent" },
+ observed: { icon: "✓", tone: "success" },
+ listed: { icon: "✓", tone: "success" },
+ delivered: { icon: "✓", tone: "success" },
+ closed: { icon: "✓", tone: "success" },
+ refused: { icon: "✗", tone: "warning" },
+ unknown: { icon: "?", tone: "warning" },
+ failed: { icon: "✗", tone: "error" },
+};
+
+/** One styled value of a call heading. */
+export interface HeadingPart { text: string; color?: ThemeColor }
+
+/** A call heading: the tool, what it acts on, and the options that matter, most important first. */
+export interface Heading {
+ tool: string;
+ primary?: HeadingPart;
+ qualifiers?: readonly string[];
+ /** Exact arguments, shown only when the card is expanded. */
+ details?: readonly (readonly [label: string, value: string])[];
+}
+
+/** A note under the status line. */
+export interface Note { tone: ThemeColor; text: string }
+
+/** The outcome of a call, ready to draw. */
+export interface Outcome {
+ state: ComputerState;
+ /** What the status line says after the state: where, on what, how big. */
+ facts?: readonly string[];
+ /** Why, or what to know, always shown (wrapped). */
+ notes?: readonly Note[];
+ /** Short rows such as UI elements or listed windows. Compact shows the first few. */
+ rows?: readonly string[];
+ /** More rows than those given, when the source was already bounded. */
+ moreRows?: number;
+ /** Labelled facts shown only when expanded. */
+ details?: readonly (readonly [label: string, value: string])[];
+}
+
+/** Rows and note lines a compact card shows before it offers to expand. */
+export const COMPACT_ROWS = 6;
+const EXPANDED_ROWS = 40;
+
+/** App text is data, never terminal control sequences. One line, no layout characters. */
+export function oneLine(value: unknown, max = 240): string {
+ const text = String(value ?? "").replace(/[\x00-\x1f\x7f-\x9f]+/g, " ").replace(/\s+/g, " ").trim();
+ return text.length > max ? `${text.slice(0, max - 1)}…` : text;
+}
+
+const separator = (theme: Theme) => theme.fg("dim", " · ");
+
+/** A width-safe tool-call heading: `tool primary · qualifier · qualifier`, dropping qualifiers before it clips the primary. */
+export class HeadingLine implements Component {
+ constructor(private heading: Heading, private theme: Theme, private expanded: boolean) {}
+ update(heading: Heading, theme: Theme, expanded: boolean): void { this.heading = heading; this.theme = theme; this.expanded = expanded; }
+ invalidate(): void {}
+ render(width: number): string[] {
+ const w = Math.max(1, width), theme = this.theme, { tool, primary, details } = this.heading;
+ const title = theme.fg("toolTitle", theme.bold(oneLine(tool)));
+ let line = title;
+ if (primary && visibleWidth(title) < w) {
+ const qualifiers = (this.heading.qualifiers ?? []).map(q => oneLine(q)).filter(Boolean);
+ const sep = separator(theme), sepWidth = visibleWidth(sep);
+ let kept = qualifiers.length;
+ const total = () => visibleWidth(title) + 1 + visibleWidth(oneLine(primary.text)) +
+ qualifiers.slice(0, kept).reduce((sum, q) => sum + sepWidth + visibleWidth(q), 0) + (kept < qualifiers.length ? sepWidth + 4 : 0);
+ while (kept > 0 && total() > w) kept--;
+ const shown = qualifiers.slice(0, kept).map(q => `${sep}${theme.fg("muted", q)}`).join("");
+ // The "… +N" marker is worth its room only when the target fits whole beside it.
+ const more = kept < qualifiers.length && total() <= w ? `${sep}${theme.fg("dim", `… +${qualifiers.length - kept}`)}` : "";
+ const room = Math.max(1, w - visibleWidth(title) - 1 - visibleWidth(shown) - visibleWidth(more));
+ line = `${title} ${theme.fg(primary.color ?? "accent", truncateToWidth(oneLine(primary.text), room))}${shown}${more}`;
+ }
+ const lines = [truncateToWidth(line, w)];
+ if (this.expanded && details?.length) {
+ const text = [theme.fg("dim", "args ")].concat(details.map(([label, value], i) =>
+ `${i === 0 ? "" : theme.fg("dim", " · ")}${theme.fg("muted", `${oneLine(label)}=`)}${theme.fg("toolOutput", oneLine(value))}`)).join("");
+ lines.push(...wrapTextWithAnsi(text, w).map(l => truncateToWidth(l, w)));
+ }
+ return lines;
+ }
+}
+
+/** The outcome of a call below its heading: a status line, notes, and a few rows that expand on demand. */
+export class OutcomeCard implements Component {
+ constructor(private outcome: Outcome, private theme: Theme, private expanded: boolean) {}
+ update(outcome: Outcome, theme: Theme, expanded: boolean): void { this.outcome = outcome; this.theme = theme; this.expanded = expanded; }
+ invalidate(): void {}
+ render(width: number): string[] {
+ const w = Math.max(1, width), theme = this.theme, { state, facts = [], notes = [], rows = [], moreRows = 0, details = [] } = this.outcome;
+ const { icon, tone } = STATES[state];
+ const status = theme.fg(tone, `${icon} ${state}`);
+ const line = [status, ...facts.filter(Boolean).map(f => theme.fg("muted", oneLine(f)))].join(separator(theme));
+ const lines = [truncateToWidth(line, w)];
+ const indent = " ", inner = Math.max(1, w - indent.length);
+ for (const note of notes) for (const part of wrapTextWithAnsi(theme.fg(note.tone, oneLine(note.text, 600)), inner)) lines.push(indent + truncateToWidth(part, inner));
+ const limit = this.expanded ? EXPANDED_ROWS : COMPACT_ROWS;
+ for (const row of rows.slice(0, limit)) lines.push(indent + truncateToWidth(theme.fg("dim", oneLine(row, 400)), inner));
+ const hidden = Math.max(0, rows.length - limit) + moreRows;
+ if (this.expanded) {
+ for (const [label, value] of details) lines.push(indent + truncateToWidth(`${theme.fg("muted", `${oneLine(label)} `)}${theme.fg("toolOutput", oneLine(value, 400))}`, inner));
+ if (hidden > 0) lines.push(indent + truncateToWidth(theme.fg("muted", `… ${hidden} more`), inner));
+ } else if (hidden > 0) {
+ lines.push(indent + truncateToWidth(theme.fg("muted", `… ${hidden} more · ${keyText("app.tools.expand")} to expand`), inner));
+ }
+ return lines;
+ }
+}
+
+// --- Reading typed result fields. Anything absent or of another type is simply not shown. ---
+
+export const record = (value: unknown): Record<string, unknown> =>
+ value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+const array = (value: unknown): unknown[] => Array.isArray(value) ? value : [];
+const str = (value: unknown): string | undefined => typeof value === "string" && value.trim() ? value : undefined;
+const num = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? value : undefined;
+
+/** The opening of a message for a status note: its first line, up to two sentences. */
+export function firstReason(message: unknown, max = 220): string | undefined {
+ const text = str(message);
+ if (!text) return undefined;
+ const meaningful = text.split(/\r?\n/).map(l => l.trim()).filter(l => l && !l.startsWith("{") && !l.startsWith("}"));
+ // A header that ends in a colon ("Validation failed for tool x:") is only worth showing with the line it introduces.
+ const first = meaningful[0] ?? text;
+ const line = oneLine(first.endsWith(":") && meaningful[1] ? `${first} ${meaningful[1].replace(/^-\s*/, "")}` : first, 2000);
+ const sentences = line.split(/(?<=[.!?])\s+/);
+ let reason = sentences[0];
+ if (sentences[1] && reason.length + sentences[1].length < max) reason += ` ${sentences[1]}`;
+ return oneLine(reason, max);
+}
+
+/** Names for what an address points at, supplied by the live session; absent for an address from an earlier run. */
+export type DescribeAddress = (source: string) => { machine?: string; title?: string } | undefined;
+
+/** The machine and window title an address names in this run, for display. */
+export function targetNames(describe: DescribeAddress | undefined, source: unknown): { machine?: string; title?: string } {
+ const address = str(source);
+ const named = address ? describe?.(address) : undefined;
+ return { ...(named?.machine ? { machine: named.machine } : {}), ...(named?.title ? { title: `“${oneLine(named.title, 60)}”` } : {}) };
+}
+
+/** The state of a typed action from its semantic data. */
+export function actionState(data: Record<string, unknown>, isError: boolean): ComputerState {
+ const status = str(data.status);
+ if (status === "delivered" || status === "observed" || status === "listed" || status === "closed" || status === "unknown" || status === "refused") {
+ // A delivered receipt that the adapter still marked not ok (a failed postcondition) is not a success.
+ return data.ok === false && status !== "unknown" && status !== "refused" ? "unknown" : status;
+ }
+ if (data.ok === true && !isError) return "delivered";
+ if (data.effect === "not-applied") return "refused";
+ if (data.effect === "unknown") return "unknown";
+ return isError ? "failed" : "delivered";
+}
+
+/** Notes for a refused, unknown or failed action: the reason in a person's terms, and what to do not do. */
+export function problemNotes(state: ComputerState, data: Record<string, unknown>, fallback?: string): Note[] {
+ if (state !== "refused" && state !== "unknown" && state !== "failed") return [];
+ const script = record(data.script), error = record(data.error);
+ const reason = firstReason(script.reason) ?? firstReason(error.message) ?? fallback;
+ const notes: Note[] = [];
+ if (reason) notes.push({ tone: state === "failed" ? "error" : "warning", text: reason });
+ if (state === "unknown") notes.push({ tone: "muted", text: "It may have been applied. Read the window before acting again; do not repeat it blindly." });
+ return notes;
+}
+
+/** What changed in the UI after an action, from the successor observation's diff. */
+export function successorNotes(successor: unknown): { notes: Note[]; rows: string[] } {
+ const diff = record(record(successor).diff);
+ if (!Object.keys(diff).length) return { notes: [], rows: [] };
+ const counts = record(diff.counts), changed = num(counts.changed) ?? 0, added = num(counts.added) ?? 0, removed = num(counts.removed) ?? 0;
+ const waited = num(diff.waited_ms);
+ const what = diff.kind === "dom" ? "page" : diff.kind === "pixels" ? "pixels" : "UI";
+ if (changed + added + removed === 0) {
+ const pixels = record(diff.pixels);
+ return { notes: [{ tone: "warning", text: `Nothing changed in the ${what}${waited !== undefined ? ` within ${waited} ms` : ""}. Check the target before acting again.${pixels.changed === true ? "" : ""}` }], rows: [] };
+ }
+ const parts = [changed && `${changed} changed`, added && `${added} added`, removed && `${removed} removed`].filter(Boolean).join(", ");
+ const rows = [...array(diff.changed).map(l => `~ ${oneLine(l, 160)}`), ...array(diff.added).map(l => `+ ${oneLine(l, 160)}`), ...array(diff.removed).map(l => `- ${oneLine(l, 160)}`)];
+ return { notes: [{ tone: "muted", text: `${what} ${parts}${waited !== undefined ? ` within ${waited} ms` : ""}` }], rows };
+}
+
+/** `500×350` from a width and height. */
+export const size = (width: unknown, height: unknown): string | undefined => num(width) !== undefined && num(height) !== undefined ? `${width}×${height}` : undefined;
+
+/** UI element rows `hint role name`; values are left out, they can be typed text or secrets. */
+export function elementRows(elements: unknown): string[] {
+ return array(elements).map(raw => {
+ const e = record(raw);
+ return [str(e.hint), str(e.role), str(e.name) ? `“${oneLine(e.name, 80)}”` : undefined, e.enabled === false ? "(disabled)" : undefined].filter(Boolean).join(" ");
+ }).filter(Boolean);
+}
diff --git a/src/provider.ts b/src/provider.ts
index 30972fd..3913544 100644
--- a/src/provider.ts
+++ b/src/provider.ts
@@ -11,8 +11,7 @@ import { REMOTE_DESKTOP_DISPLAY } from "./display-mode.ts";
import type { WebLogEntry } from "./web-browser.ts";
import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";
import type { PixelBox } from "./pixel-diff.ts";
-import { computerActionRenderer } from "./computer-action-renderer.ts";
-import { computerReadRenderer } from "./computer-read-renderer.ts";
+import { computerActionRenderer, computerReadRenderer, computerSearchRenderer } from "./computer-renderers.ts";
import type { ComputerSession, ComputerResult, ComputerMode, Content } from "./session.ts";
const owns = (source: unknown): source is string => typeof source === "string" && source.startsWith("computer:");
@@ -452,6 +451,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
markdown: guide,
triggers: ["read", "search"].map(tool => ({ tool, resourcePrefixes: ["computer:"] })) }]);
const resources = new ComputerResources(getSession, closeSession, openSandbox, connectMachine, getMode);
+ const describe = (source: string) => resources.describe(source);
// Pi shutdown releases every machine, sandbox and session this computer use started.
pi.on("session_shutdown", () => resources.closeAll().then(() => undefined));
const resolver: ResourceResolver = {
@@ -475,7 +475,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
await Promise.all([
optional(connectReadPlugin(pi, { protocol: READ_PROTOCOL, apiVersion: READ_API_VERSION, id: "computer", setup(api) {
accepted.add("read");
- api.addResolver({ resolver, priority: -250, renderResult: computerReadRenderer });
+ api.addResolver({ resolver, priority: -250, renderResult: computerReadRenderer(describe) as never });
for (const view of ["image", "ui", "full", "state", "dom", "region", "sequence", "frames", "capture", "element", "ocr", "changes", "regions", "aria"])
api.addView({ view, contentKind: "any", presenter: { id: `computer-${view}`, present: document => document } });
api.addHandler({ stage: "read", when: { resolvedBy: resolver.id, contentKind: "any" }, async handler(context) {
@@ -489,7 +489,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
} })),
optional(connectSearchPlugin(pi, { protocol: SEARCH_PROTOCOL, apiVersion: SEARCH_API_VERSION, id: "computer", setup(api) {
accepted.add("search");
- api.addResolver({ priority: -250, resolver: { id: "computer", readResources: request => owns(request.path) ? [request.path] : [],
+ api.addResolver({ priority: -250, resolver: { id: "computer", renderResult: computerSearchRenderer(describe) as never, readResources: request => owns(request.path) ? [request.path] : [],
async tryResolve(request, context) {
if (!owns(request.path)) return { kind: "not-handled" };
try {
@@ -511,7 +511,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw
let renderersRegistered = false;
pi.on("session_start", () => {
if (renderersRegistered) return;
- for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool));
+ for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool, describe) as never);
renderersRegistered = true;
});
// Write and Insert do not act on computer addresses; they refuse instead of treating the address as a file.
diff --git a/src/standalone-tools.ts b/src/standalone-tools.ts
index c59bc55..f3b18e6 100644
--- a/src/standalone-tools.ts
+++ b/src/standalone-tools.ts
@@ -1,20 +1,19 @@
import { Type, type JsonValue } from "@earendil-works/pi-ai";
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { Text } from "@earendil-works/pi-tui";
import type { ComputerResources } from "./computer-resources.ts";
-import { computerReadRenderer } from "./computer-read-renderer.ts";
+import { resourceRenderers } from "./computer-renderers.ts";
import { computerNamespace, registerComputerDelete } from "./computer-tools.ts";
import { COMPUTER_PATH_HELP, COMPUTER_SEARCH_HELP, COMPUTER_VIEWS_HELP, presentComputerResult, readComputer } from "./provider.ts";
const GUIDE = "docs:computer";
const image = Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String() });
-const plain = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, " ");
/**
* Offer computer use without Pi Agent IDE: computer_read, computer_search and computer_delete stand in for the IDE's Read,
* Search and Delete. Like the other computer_* tools they are deferred, off the model's tool list until tool_search loads them.
*/
export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerResources, guide: string): void {
+ const describe = (source: string) => resources.describe(source);
const namespace = computerNamespace();
pi.registerTool({
name: "computer_read", label: "Computer read", exposure: "deferred", namespace,
@@ -31,8 +30,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerRes
Type.Object({ type: Type.Literal("images"), note: Type.String(), images: Type.Array(image) }),
]),
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
- renderCall: args => new Text(`computer_read ${plain(args.path)}${args.views?.length ? ` [${plain(args.views.join(", "))}]` : ""}`, 0, 0),
- renderResult: computerReadRenderer,
+ ...resourceRenderers("computer_read", describe),
async execute(_id, params, signal, _update, ctx) {
if (params.path === GUIDE) {
const lines = guide.replace(/\n$/, "").split("\n"), offset = params.offset ?? 1;
@@ -59,7 +57,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerRes
limit: Type.Optional(Type.Integer({ minimum: 1, description: "Maximum results." })),
}, { additionalProperties: false }),
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
- renderCall: args => new Text(`computer_search ${plain(args.path)} ${plain(JSON.stringify(args.query))}`, 0, 0),
+ ...resourceRenderers("computer_search", describe),
async execute(_id, params, signal, _update, ctx) {
const result = await resources.search(resources.address(params.path), ctx.cwd, params.query, params.limit, signal);
return { content: [{ type: "text", text: resources.shorten(presentComputerResult(result)) }], details: result.details, ...(result.isError ? { isError: true } : {}) };
diff --git a/test/computer-action-renderer.test.ts b/test/computer-action-renderer.test.ts
deleted file mode 100644
index 3f93c1f..0000000
--- a/test/computer-action-renderer.test.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-import { visibleWidth } from "@earendil-works/pi-tui";
-import { expect, test } from "vitest";
-import { computerActionRenderer } from "../src/computer-action-renderer.ts";
-
-function render(summary: string, width = 80, guide = "") {
- const renderer = computerActionRenderer("insert");
- return renderer.renderResult!({ content: [{ type: "text", text: summary }, { type: "text", text: guide }], details: {
- metadata: { semanticAction: { kind: "computer-action", receipt: "PRIVATE_NATIVE_DIAGNOSTIC", error: { message: "PRIVATE_NATIVE_ERROR" } } },
- } } as never, { expanded: false, isPartial: false }, {} as never, {} as never).render(width);
-}
-
-test("computer renderers select only computer calls and semantic computer results", () => {
- for (const tool of ["write", "insert"] as const) {
- const renderer = computerActionRenderer(tool);
- expect(renderer.matches?.({ path: "computer:observation/owned" }, "call")).toBe(true);
- expect(renderer.matches?.({ path: "notes.txt" }, "call")).toBe(false);
- expect(renderer.matches?.({ path: ["computer:session"] }, "call")).toBe(false);
- expect(renderer.matches?.({ details: { metadata: { semanticAction: { kind: "computer-action" } } } }, "result")).toBe(true);
- for (const result of [undefined, { details: {} }, { details: { metadata: { semanticAction: { kind: "file-action" } } } }])
- expect(renderer.matches?.(result, "result")).toBe(false);
- }
-});
-
-test.each([
- "Computer refused\nDOM action: refused; trusted=false; atomic=false; effects=unverified\nAttempted steps: []; acknowledged: []\nIn flight: none; reason=dom_action_trusted_route_unavailable; cleanup=not_started",
- 'Computer unknown\nDOM action: refused; effects=unverified\nAttempted steps: ["click","dblclick"]; acknowledged: ["click"]\nIn flight: dblclick; progress=unknown; cleanup=unconfirmed',
- 'Computer delivered\nDOM action: dispatched; effects=unverified\nAttempted steps: ["click"]; acknowledged: ["click"]\nIn flight: none; cleanup=confirmed',
-])("keeps the sanitized receipt, not private metadata, even when collapsed", summary => {
- const rendered = render(summary, 180).join("\n");
- for (const line of summary.split("\n")) expect(rendered).toContain(line);
- expect(rendered).not.toContain("PRIVATE_NATIVE");
- expect(rendered).not.toContain("edit failed");
-});
-
-test("wraps wide receipt text and never emits app-provided terminal controls", () => {
- const rendered = render("Computer refused\nDOM refusal: 测试 e\u0301 \u001b[2J unsafe", 18);
- expect(rendered.every(line => visibleWidth(line) <= 18)).toBe(true);
- expect(rendered.join("\n")).not.toContain("\u001b");
- expect(rendered.join("\n")).toContain("测试");
-});
-
-test("keeps model-only result envelopes and attached guides out of the terminal receipt", () => {
- const rendered = render('<system-result note="Internal reference"><uuid>PRIVATE_REFERENCE</uuid></system-result>\nComputer refused', 80, "PRIVATE_AGENT_GUIDE").join("\n");
- expect(rendered).toContain("Computer refused");
- expect(rendered).not.toContain("PRIVATE_");
- expect(rendered).not.toContain("system-result");
-});
-test("computer call headings omit literal input and native data", () => {
- const renderer = computerActionRenderer("write");
- const lines = renderer.renderCall!({ path: "computer:observation/owned", content: "PRIVATE_TYPED_TEXT" }, {} as never, {} as never).render(100);
- expect(lines.join("\n")).toContain("write computer:observation/owned");
- expect(lines.join("\n")).not.toContain("PRIVATE_TYPED_TEXT");
-});
diff --git a/test/computer-read-renderer.test.ts b/test/computer-read-renderer.test.ts
deleted file mode 100644
index c017308..0000000
--- a/test/computer-read-renderer.test.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { visibleWidth } from "@earendil-works/pi-tui";
-import { expect, test } from "vitest";
-import { computerReadRenderer } from "../src/computer-read-renderer.ts";
-
-const theme = { fg: (_: unknown, text: string) => text, bold: (text: string) => text };
-const context = { args: { path: "computer:observation/current" }, cwd: "/", isError: false };
-const render = (text: string, computer: unknown, width = 79) =>
- computerReadRenderer({ content: [{ type: "text", text }, { type: "text", text: "PRIVATE_AGENT_GUIDE" }],
- details: { source: context.args.path, resolvedBy: "computer", computer } } as never,
- { expanded: false, isPartial: false }, theme as never, context as never).render(width);
-
-test("a Playwright screenshot shows a short sanitized provenance line above Pi's enlarged image", () => {
- const lines = render("Computer observed (yolo)\nImage: 36x36 image/png", {
- status: "observed", state: { title: "Form\u001b[2J", url: "http://127.0.0.1:8123/form" },
- agent_image: { route: "playwright_screenshot", kind: "crop", output: { width: 36, height: 36 } },
- });
- expect(lines).toHaveLength(2);
- const output = lines.join("\n");
- for (const value of ["Playwright crop screenshot", "36x36 PNG", "page pixels, input=none", "http://127.0.0.1:8123/form"]) expect(output).toContain(value);
- expect(output).not.toContain("\u001b");
- expect(output).not.toContain("PRIVATE_");
- expect(lines.every(line => visibleWidth(line) <= 79)).toBe(true);
-});
-test("ordinary computer reads retain the standard source renderer", () => {
- const lines = render("ordinary window overview", { status: "observed", agent_image: { route: "cua_native" } });
- expect(lines.join("\n")).toContain("ordinary window overview");
- expect(lines.join("\n")).toMatch(/[╭┌]/);
-});