Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.
TypeScript accessibilityagent-toolsai-agentbrowser-automationcomputer-usecuadesktop-automationelectronpipi-coding-agentpi-extensionpi-packageplaywrighttypescriptui-automationvisionwebview2windowswindows-automationwsl
Computer Use draws its tool calls like Pi Agent IDE's tools (LPT-856)
alexshpunt-clanker[bot]3ea1c1397bea
diff --git a/src/computer-action-renderer.ts b/src/computer-action-renderer.tsdeleted file mode 100644index 313e274..0000000--- a/src/computer-action-renderer.ts+++ /dev/null@@ -1,27 +0,0 @@-import { Text } from "@earendil-works/pi-tui";-import type { TextEditorPluginApi } from "pi-agent-ide/api/text-editor";--type Registration = Parameters<TextEditorPluginApi["addToolRenderer"]>[0];-const record = (value: unknown): Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};-// App text is data, never terminal control sequences. Keep newlines for the existing receipt layout.-const plain = (value: string) => value.replace(/[\x00-\x08\x0b-\x1f\x7f-\x9f]/g, "");--/** Preserve computer receipts without replacing IDE's renderer for unrelated edits. */-export function computerActionRenderer(tool: string): Registration {- return {- tool,- matches(value, phase) {- if (phase === "call") return typeof record(value).path === "string" && String(record(value).path).startsWith("computer:");- return record(record(record(record(value).details).metadata).semanticAction).kind === "computer-action";- },- renderCall(args) {- return new Text(`${tool} ${plain(String(record(args).path))}`, 0, 0);- },- renderResult(result) {- // content is the provider's safe summary; metadata.receipt can contain private native diagnostics.- const summary = result.content.find(block => block.type === "text")?.text ?? "";- const body = summary.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "");- return new Text(plain(body), 0, 0);- },- };-}diff --git a/src/computer-read-renderer.ts b/src/computer-read-renderer.tsdeleted file mode 100644index e26edde..0000000--- a/src/computer-read-renderer.ts+++ /dev/null@@ -1,18 +0,0 @@-import { Text } from "@earendil-works/pi-tui";-import { createReadResultRenderer, type ReadResultRenderer } from "pi-agent-ide/api/read";--const fallback = createReadResultRenderer({ kind: "source" });-const record = (value: unknown): Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};-const clean = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, "").slice(0, 200);--/** Show a short provenance line above Playwright page images; other reads retain IDE's source renderer. */-export const computerReadRenderer: ReadResultRenderer = (result, options, theme, context) => {- const details = record(result.details), computer = record(details.computer), image = record(computer.agent_image);- if (details.resolvedBy !== "computer" || image.route !== "playwright_screenshot") return fallback(result, options, theme, context);- const output = record(image.output), state = record(computer.state);- // Pi appends the enlarged image below this component; keep the receipt short enough to stay above it.- return new Text([- `Playwright ${clean(image.kind)} screenshot: ${output.width}x${output.height} PNG | page pixels, input=none`,- `${clean(state.title)} | ${clean(state.url)}`,- ].join("\n"), 0, 0);-};diff --git a/src/computer-renderers.ts b/src/computer-renderers.tsnew file mode 100644index 0000000..b46cf81--- /dev/null+++ b/src/computer-renderers.ts@@ -0,0 +1,311 @@+import type { Component } from "@earendil-works/pi-tui";+import type { Theme } from "@earendil-works/pi-coding-agent";+import {+ actionState, elementRows, firstReason, HeadingLine, OutcomeCard, oneLine, problemNotes, record, size, successorNotes, targetNames,+ type ComputerState, type DescribeAddress, type Heading, type Note, type Outcome,+} from "./computer-tui.ts";++/** A tool result as Pi gives it to a renderer; only the typed fields are read, never the receipt text. */+interface ToolResult { content?: { type: string; text?: string }[]; details?: unknown; isError?: boolean }+interface RenderOptions { expanded: boolean; isPartial: boolean }+interface RenderContext { isError?: boolean; lastComponent?: Component; args?: unknown }++const array = (value: unknown): unknown[] => Array.isArray(value) ? value : [];+const str = (value: unknown): string | undefined => typeof value === "string" && value.trim() ? value : undefined;+const num = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? value : undefined;+const last = (address: string) => address.replace(/^computer:/, "").replace(/\?.*$/, "");++/** The window a person is looking at: its title when this run listed one, otherwise its short address. */+function windowLabel(describe: DescribeAddress | undefined, address: unknown): string {+ const source = str(address);+ if (!source) return "window";+ const title = describe?.(source)?.title;+ return title ? `“${oneLine(title, 60)}”` : last(source);+}+const machineOf = (describe: DescribeAddress | undefined, at: unknown) => {+ const source = str(at);+ return source ? describe?.(source)?.machine ?? /^computer:machine\/([^/?]+)/.exec(source)?.[1] ?? /^computer:([a-z0-9_-]+)\/[a-z]\d/.exec(source)?.[1] : undefined;+};+const base = (path: string) => path.split(/[\\/]/).filter(Boolean).pop() ?? path;+/** Where an input is aimed: a ref or locator, or a point. */+const aimed = (args: Record<string, unknown>) => args.target !== undefined ? oneLine(args.target, 60) : args.x !== undefined ? `${args.x},${args.y}` : "";+/** Argument names whose values are what the agent typed or pasted; headings and details never repeat them. */+const LITERAL = new Set(["text", "content", "value", "values", "data", "script", "code"]);++/** Exact scalar arguments for the expanded heading, without literal input. */+function argumentDetails(args: Record<string, unknown>): [string, string][] {+ return Object.entries(args).filter(([key, value]) => !LITERAL.has(key) && ["string", "number", "boolean"].includes(typeof value))+ .map(([key, value]) => [key, oneLine(value, 120)] as [string, string]);+}++/** The heading of one typed computer_* call: what it acts on, then the options that matter. */+export function typedHeading(tool: string, rawArgs: unknown, describe?: DescribeAddress): Heading {+ const args = record(rawArgs), window = windowLabel(describe, args.window);+ const base_: Heading = { tool, details: argumentDetails(args) };+ const heading = (primary: string, ...qualifiers: (string | undefined | false)[]): Heading =>+ ({ ...base_, primary: { text: primary }, qualifiers: qualifiers.filter((q): q is string => typeof q === "string" && q.length > 0) });+ switch (tool) {+ case "computer_open": {+ const what = str(args.app) ?? (str(args.path) ? base(String(args.path)) : undefined) ?? str(args.url) ?? str(args.browser) ??+ (args.sandbox ? "sandbox" : args.isolated_session ? "isolated session" : args.webview2_debugging ? "WebView2 debugging" : undefined) ?? window;+ return heading(what, machineOf(describe, args.at), str(args.browser) && str(args.url) ? str(args.browser) : undefined);+ }+ case "computer_window": {+ const action = String(args.action ?? "");+ const detail = action === "move" ? `move ${args.x},${args.y}` : action === "resize" ? `resize ${args.width}×${args.height}` : action === "name" ? `name ${args.as}` : action;+ return heading(window, detail);+ }+ case "computer_view": {+ const at = args.x !== undefined || args.y !== undefined ? `at ${args.x ?? "_"},${args.y ?? "_"}` : undefined;+ const dims = args.width !== undefined || args.height !== undefined ? `${args.width ?? "_"}×${args.height ?? "_"}` : undefined;+ return heading(last(String(args.viewport ?? "viewport")), at, dims);+ }+ case "computer_wait": return heading(window, String(args.mode ?? "wait"), `${array(args.conditions).length} condition(s)`);+ case "computer_browser": return heading(window, String(args.action ?? ""), str(args.url) ?? str(args.tab));+ case "computer_files": return heading(String(args.machine ?? "machine"), String(args.action ?? ""), str(args.from) ?? str(args.name));+ case "computer_clipboard": return heading(String(args.machine ?? "machine"), String(args.action ?? ""), str(args.image));+ case "computer_record": return heading(windowLabel(describe, args.at), String(args.action ?? ""), num(args.seconds) !== undefined ? `${args.seconds} s` : num(args.duration_ms) !== undefined ? `${args.duration_ms} ms` : undefined);+ case "computer_click": return heading(window, aimed(args), num(args.count) ? `×${args.count}` : undefined, str(args.button) && args.button !== "left" ? String(args.button) : undefined);+ case "computer_hover": return heading(window, aimed(args));+ case "computer_drag": return heading(window, `${aimed(record(args.from)) || "?"} → ${aimed(record(args.to)) || "?"}`);+ case "computer_scroll": return heading(window, aimed(args), `${args.delta_x ?? 0},${args.delta_y ?? 0}`);+ case "computer_type": return heading(window, aimed(args), `${String(args.text ?? "").length} chars`, args.submit ? "+ Enter" : undefined);+ case "computer_press": return heading(window, String(args.key ?? ""), num(args.hold_ms) ? `${args.hold_ms} ms` : undefined);+ case "computer_select": {+ const choice = Array.isArray(args.menu) ? (args.menu as string[]).map(item => oneLine(item, 30)).join(" > ")+ : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : array(args.values).length ? `${array(args.values).length} value(s)` : "";+ return heading(window, aimed(args), choice);+ }+ case "computer_timeline": return heading(window, `${array(args.events).length} events`);+ case "computer_display": return heading(String(args.display ?? "display"), `${args.width}×${args.height}`);+ default: return heading(last(String(args.path ?? args.window ?? args.at ?? "")));+ }+}++const KINDS: Record<string, string> = { w: "window", o: "observation", s: "sequence", a: "app", b: "sandbox", d: "display" };+/** A short computer address in words: `computer:local/w1tt` is "window w1tt" on machine local. */+function addressWords(address: string): { what: string; machine?: string } {+ const short = /^computer:([a-z0-9_-]+)\/([wosabd])(\d+[a-z]{2})(?:\/([a-z0-9_-]+))?$/.exec(address.replace(/\?.*$/, ""));+ if (short) return { what: `${KINDS[short[2]]} ${short[2]}${short[3]}${short[4] ? `/${short[4]}` : ""}`, machine: short[1] };+ const machine = /^computer:machine\/([^/?]+)(?:\/(windows|apps|displays))?/.exec(address);+ if (machine) return { what: machine[2] ? `${machine[1]} ${machine[2]}` : `machine ${machine[1]}`, machine: machine[1] };+ if (address === "computer:session") return { what: "session" };+ return { what: last(address) };+}++/** The heading of an IDE tool call on a computer address (Read/Search/Write/Insert/Delete and the standalone twins). */+export function resourceHeading(tool: string, rawArgs: unknown, describe?: DescribeAddress): Heading {+ const args = record(rawArgs), path = str(args.path) ?? "";+ const views = array(args.views).filter((v): v is string => typeof v === "string");+ const { what, machine } = addressWords(path);+ const title = path.startsWith("computer:") ? describe?.(path)?.title : undefined;+ const qualifiers: string[] = [];+ if (typeof args.query === "string") qualifiers.push(`“${oneLine(args.query, 60)}”`);+ if (views.length) qualifiers.push(`views ${views.join(",")}`);+ if (machine && !/^machine /.test(what) && !/^[a-z0-9_-]+ (windows|apps|displays)$/.test(what)) qualifiers.push(machine);+ return { tool, primary: { text: title ? `“${oneLine(title, 60)}”` : what }, qualifiers, details: [["path", path], ...argumentDetails(args).filter(([key]) => key !== "path")] };+}++const sizeFact = (bounds: unknown) => { const b = record(bounds); return size(b.width, b.height); };++/** Facts, notes and rows of a window-management receipt. */+function managementFacts(state: Record<string, unknown>): { facts: string[]; notes: Note[] } {+ const operation = str(state.operation), observed = record(record(state.observed).bounds), requested = record(state.requested_bounds);+ const facts: string[] = [];+ if (operation) facts.push(operation);+ const now = size(observed.width, observed.height), want = size(requested.width, requested.height);+ if (now) facts.push(now);+ const notes: Note[] = [];+ if (now && want && now !== want && state.postcondition_confirmed === true) notes.push({ tone: "muted", text: `Windows settled at ${now} for the requested ${want}.` });+ return { facts, notes };+}++/** The outcome of one typed computer_* action, from the semantic data its result carries. */+export function actionOutcome(result: ToolResult, isError: boolean, describe?: DescribeAddress): Outcome {+ const data = record(record(record(result.details).metadata).semanticAction);+ if (data.kind !== "computer-action") {+ // A result that is not a computer receipt (a schema or runtime failure): say that it failed, and why in one line.+ const text = result.content?.find(block => block.type === "text")?.text;+ return { state: isError ? "failed" : "delivered", notes: isError && firstReason(text) ? [{ tone: "error", text: firstReason(text)! }] : [] };+ }+ const script = record(data.script), state = actionState(data, isError);+ const windowAddress = str(script.window) ?? str(data.source);+ const named = targetNames(describe, windowAddress);+ // The heading already names the window by its title; the status line adds only where it ran.+ const facts = [str(data.machine) ?? named.machine ?? machineOf(describe, windowAddress)].filter((f): f is string => !!f);+ const notes = problemNotes(state, data);+ const rows: string[] = [];+ const details: [string, string][] = [];+ const stateRecord = record(data.state);+ if (typeof stateRecord.mutation_attempted === "boolean") {+ const managed = managementFacts(stateRecord);+ facts.push(...managed.facts); notes.push(...managed.notes);+ }+ if (state === "delivered" || state === "observed") {+ const ui = record(data.ui), image = record(record(data.agent_image).output);+ const dims = size(image.width, image.height);+ if (dims) facts.push(`image ${dims}`);+ if (num(ui.collected) !== undefined) { facts.push(`UI ${ui.collected} element${ui.collected === 1 ? "" : "s"}${ui.truncated === true ? " (truncated)" : ""}`); rows.push(...elementRows(ui.elements)); }+ const successor = successorNotes(data.successor);+ notes.push(...successor.notes); rows.push(...successor.rows);+ }+ const code = str(script.code) ?? str(record(data.error).code);+ if (code) details.push(["code", code]);+ if (str(script.stage)) details.push(["stage", String(script.stage)]);+ if (str(script.window)) details.push(["window", String(script.window)]);+ if (str(script.observation)) details.push(["observation", String(script.observation)]);+ if (str(data.mode)) details.push(["mode", String(data.mode)]);+ const delivery = str(stateRecord.delivery_mode);+ if (delivery) details.push(["delivery", delivery]);+ if (str(script.application_effect)) details.push(["application effect", String(script.application_effect)]);+ return { state, facts, notes, rows, details };+}++/** The human-readable lines of a result's text, without the model-only envelope and attached guides. */+function contentRows(result: ToolResult): string[] {+ const text = (result.content ?? []).filter(block => block.type === "text" && typeof block.text === "string" && !block.text.startsWith("\n\n---\n\n# Guide:"))+ .map(block => block.text!.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "")).join("\n");+ return text.split(/\r?\n/).map(line => line.trimEnd()).filter(Boolean);+}++const READ_STATES = new Set<ComputerState>(["observed", "listed", "delivered", "closed", "refused", "unknown"]);+const stateOf = (status: unknown, isError: boolean, fallback: ComputerState): ComputerState =>+ READ_STATES.has(status as ComputerState) ? status as ComputerState : isError ? "failed" : fallback;++/** The outcome of a Read of a computer address, whether through the IDE or computer_read. */+export function readOutcome(result: ToolResult, isError: boolean, describe?: DescribeAddress): Outcome {+ const outer = record(result.details), d = record(outer.computer ?? outer.payload ?? result.details);+ const state = stateOf(d.status, isError, "observed");+ const source = str(d.window_ref) ? `computer:window/${d.window_ref}` : str(outer.source);+ const named = targetNames(describe, source);+ const facts = [str(d.machine) ?? named.machine, named.title].filter((f): f is string => !!f);+ const notes: Note[] = [], rows: string[] = [], details: [string, string][] = [];+ const image = record(record(d.agent_image).output), dims = size(image.width, image.height);+ if (dims) facts.push(`image ${dims}`);+ const page = record(d.state);+ if (record(d.agent_image).route === "playwright_screenshot") {+ facts.push(`${oneLine(record(d.agent_image).kind ?? "page", 20)} screenshot`);+ if (str(page.title)) facts.push(`“${oneLine(page.title, 60)}”`);+ if (str(page.url)) facts.push(oneLine(page.url, 80));+ }+ const crop = record(d.screenshot_crop);+ if (size(crop.width, crop.height)) facts.push(`crop ${size(crop.width, crop.height)}`);+ const ui = record(d.ui);+ if (num(ui.collected) !== undefined) {+ facts.push(`UI ${ui.collected}${num(ui.native_total) !== undefined && ui.native_total !== ui.collected ? `/${ui.native_total}` : ""} element${ui.collected === 1 ? "" : "s"}${ui.truncated === true ? " (truncated)" : ""}`);+ rows.push(...elementRows(ui.elements));+ }+ const dom = record(d.dom);+ if (str(dom.snapshot)) {+ facts.push(`page ${oneLine(dom.title ?? dom.url ?? "", 50)}`.trim());+ rows.push(...String(dom.snapshot).split("\n").map(line => line.trimEnd()).filter(Boolean));+ if (dom.truncated === true) notes.push({ tone: "warning", text: "The page snapshot is truncated; narrow it with dom:{target,depth}." });+ }+ const lines = array(d.text_lines);+ if (lines.length) { facts.push(`OCR ${lines.length} line${lines.length === 1 ? "" : "s"}`); rows.push(...lines.map(line => oneLine(array(record(line).words).map(w => record(w).text).join(" ")))); }+ const regions = array(d.visual_regions);+ if (regions.length) facts.push(`${regions.length} region${regions.length === 1 ? "" : "s"}`);+ const element = record(d.element_text);+ if (str(element.text)) { facts.push(`element ${oneLine(element.role ?? "", 20)} ${oneLine(element.name ?? "", 40)}`.trim()); rows.push(...String(element.text).split(/\r?\n/)); if (element.cut === true) notes.push({ tone: "muted", text: "The element's text is cut." }); }+ const sequence = record(d.sequence);+ if (num(sequence.frame_count) !== undefined) facts.push(`${sequence.frame_count} frames`);+ const changes = successorNotes(d.changes); notes.push(...changes.notes); rows.push(...changes.rows);+ const stateRecord = record(d.state);+ if (d.window_state_kind === "management" || stateRecord.bounds !== undefined) {+ const sample = d.window_state_kind === "management" ? record(stateRecord.state) : stateRecord, bs = sizeFact(sample.bounds);+ if (bs) facts.push(bs);+ if (sample.minimized === true) facts.push("minimized"); else if (sample.maximized === true) facts.push("maximized");+ }+ const windows = array(record(d.state).windows), apps = array(record(d.state).apps), displays = array(record(d.state).displays);+ if (windows.length || apps.length || displays.length) {+ const [kind, items] = windows.length ? ["window", windows] as const : apps.length ? ["app", apps] as const : ["display", displays] as const;+ facts.push(`${items.length} ${kind}${items.length === 1 ? "" : "s"}`);+ rows.push(...items.map(item => listedRow(kind, record(item), describe)));+ }+ const verification = record(d.verification);+ if (array(verification.conditions).length) {+ const conditions = array(verification.conditions).map(raw => record(raw));+ facts.push(`${conditions.filter(c => c.verdict === "satisfied").length}/${conditions.length} conditions`);+ rows.push(...conditions.map((c, i) => `${i + 1}. ${oneLine(c.verdict ?? "")}${str(c.reason) ? ` — ${oneLine(c.reason, 120)}` : ""}`));+ }+ if (state === "refused" || state === "unknown" || state === "failed") {+ const reason = firstReason(result.content?.find(block => block.type === "text")?.text?.replace(/^<system-result\b[^>]*>[\s\S]*?<\/system-result>\r?\n?/, "").split("\n").slice(1).join("\n"));+ notes.push(...problemNotes(state, {}, reason));+ }+ // A resource with no typed observation fields, such as the session descriptor: its own text is the readable form.+ if (!facts.length && !rows.length && !notes.length && (state === "observed" || state === "listed")) rows.push(...contentRows(result));+ const captured = str(record(d.source).captured_at);+ if (captured) details.push(["captured", captured]);+ const freshness = str(record(record(d.source).visibility).paint_freshness);+ if (freshness) details.push(["paint", freshness]);+ if (str(d.mode)) details.push(["mode", String(d.mode)]);+ return { state, facts, notes, rows, details };+}++function listedRow(kind: "window" | "app" | "display", item: Record<string, unknown>, describe?: DescribeAddress): string {+ void describe;+ if (kind === "window") return [str(item.app_name), str(item.title) ? `“${oneLine(item.title, 70)}”` : undefined, sizeFact(item.bounds), item.owned === true ? "yours" : undefined].filter(Boolean).join(" ");+ if (kind === "app") return oneLine(item.name ?? item.launch_path ?? "");+ return [str(item.display_id) ?? str(item.name), sizeFact(item.bounds), item.primary === true ? "primary" : undefined].filter(Boolean).join(" ");+}++// --- Components Pi mounts ---++const reuse = <T extends Component>(previous: Component | undefined, type: new (...args: never[]) => T): T | undefined => previous instanceof type ? previous : undefined;++/** A heading component, reusing the previous one so a redraw does not flicker. */+export function headingComponent(heading: Heading, theme: Theme, expanded: boolean, previous?: Component): Component {+ const component = reuse(previous, HeadingLine) as HeadingLine | undefined;+ if (component) { component.update(heading, theme, expanded); return component; }+ return new HeadingLine(heading, theme, expanded);+}+/** An outcome component, reusing the previous one. */+export function outcomeComponent(outcome: Outcome, theme: Theme, expanded: boolean, previous?: Component): Component {+ const component = reuse(previous, OutcomeCard) as OutcomeCard | undefined;+ if (component) { component.update(outcome, theme, expanded); return component; }+ return new OutcomeCard(outcome, theme, expanded);+}++/** renderCall and renderResult of one typed computer_* tool. */+export function typedRenderers(tool: string, describe?: DescribeAddress) {+ return {+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>+ headingComponent(typedHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>+ outcomeComponent(options.isPartial ? { state: "running" } : actionOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent),+ };+}++/** Pi Agent IDE's Read result renderer for computer addresses. */+export function computerReadRenderer(describe?: DescribeAddress) {+ return (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>+ outcomeComponent(options.isPartial ? { state: "running" } : readOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent);+}++/** Pi Agent IDE's Search result renderer for computer addresses. Search results carry the listing under `payload`. */+export const computerSearchRenderer = computerReadRenderer;++/** A Write, Insert or Delete on a computer address: refused or closed, with a heading that names the target. */+export function computerActionRenderer(tool: "write" | "insert" | "delete", describe?: DescribeAddress) {+ const owns = (value: unknown) => typeof record(value).path === "string" && String(record(value).path).startsWith("computer:");+ return {+ tool,+ matches: (value: unknown, phase: "call" | "result") => phase === "call" ? owns(value)+ : record(record(record(value).details).metadata).semanticAction !== undefined && record(record(record(record(value).details).metadata).semanticAction).kind === "computer-action",+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>+ headingComponent(resourceHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>+ outcomeComponent(options.isPartial ? { state: "running" } : actionOutcome(result, context.isError === true, describe), theme, options.expanded, context.lastComponent),+ };+}++/** renderCall and renderResult of the standalone computer_read / computer_search / computer_delete tools. */+export function resourceRenderers(tool: "computer_read" | "computer_search" | "computer_delete", describe?: DescribeAddress) {+ return {+ renderCall: (args: unknown, theme: Theme, context: RenderContext & { expanded?: boolean }): Component =>+ headingComponent(resourceHeading(tool, args, describe), theme, context.expanded === true, context.lastComponent),+ renderResult: (result: ToolResult, options: RenderOptions, theme: Theme, context: RenderContext): Component =>+ outcomeComponent(options.isPartial ? { state: "running" } : tool === "computer_delete" ? actionOutcome(result, context.isError === true, describe) : readOutcome(result, context.isError === true, describe),+ theme, options.expanded, context.lastComponent),+ };+}diff --git a/src/computer-resources.ts b/src/computer-resources.tsindex 2a4e4b8..6b3c8d4 100644--- a/src/computer-resources.ts+++ b/src/computer-resources.ts@@ -424,6 +424,19 @@ export class ComputerResources {shorten(text: string): string {return this.shorts.shorten(text, (kind, ref) => this.place(requireComputerAddress(computerSource(kind, ref)))?.machine);}+ /**+ * What a person calls the thing an address points at: its machine and, for a window, the title its last listing+ * gave. For display only: it never throws, issues no alias and starts no native work, so an address from an+ * earlier run or a window that is gone simply has no name.+ */+ describe(source: string): { machine?: string; title?: string } | undefined {+ try {+ const address = this.address(source);+ const place = this.place(address);+ const title = address.kind === "window" || address.kind === "viewport" ? this.titles.get(address.ref) : undefined;+ return place || title ? { ...(place ? { machine: place.machine } : {}), ...(title ? { title } : {}) } : undefined;+ } catch { return undefined; }+ }/** The owner of a window, or of one machine's display. */private async route(cwd: string, target: ObservationTarget, signal?: AbortSignal): Promise<Owner> {if (target.window_ref !== undefined)diff --git a/src/computer-tools.ts b/src/computer-tools.tsindex 30213ce..b2a2aa1 100644--- a/src/computer-tools.ts+++ b/src/computer-tools.ts@@ -1,13 +1,11 @@import { Type, type JsonValue, type Static } from "@earendil-works/pi-ai";import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";-import { Text } from "@earendil-works/pi-tui";import { ComputerRequestError, refKind, type ComputerResources, type InputPlan, type InputPoint } from "./computer-resources.ts";import type { ComputerAddress } from "./computer-address.ts";-import { computerActionRenderer } from "./computer-action-renderer.ts";+import { resourceRenderers, typedRenderers } from "./computer-renderers.ts";import { parseComputerViews } from "./computer-views.ts";import { COMPUTER_DELETE_HELP, computerAction, deleteComputer } from "./provider.ts";-const plain = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, " ");type Action = Awaited<ReturnType<typeof computerAction>>;/**@@ -492,16 +490,14 @@ export function timelinePlan(params: Static<typeof timelineParameters>): InputPlexport function registerComputerTools(pi: ExtensionAPI, resources: ComputerResources): void {const namespace = computerNamespace();const common = { exposure: "deferred" as const, namespace, outputSchema: receiptSchema };- const rendered = (name: string, call: (args: Record<string, unknown>) => string) => ({- renderCall: (args: unknown) => new Text(`${name} ${plain(call(args as Record<string, unknown>))}`, 0, 0),- renderResult: computerActionRenderer(name).renderResult,- });+ const describe = (source: string) => resources.describe(source);+ const rendered = (name: string) => typedRenderers(name, describe);pi.registerTool({...common, name: "computer_open", label: "Computer open",description: "Use computer_open to start an app, a Playwright browser, a Windows Sandbox or an isolated session on a machine, or to attach Playwright to a running browser window. The result gives the new window or sandbox address.",parameters: openParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_open", args => [args.at, args.app ?? args.path ?? args.url ?? args.browser ?? (args.sandbox ? "sandbox" : args.isolated_session ? "isolated session" : "")].filter(Boolean).join(" ")),+ ...rendered("computer_open"),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.at, "open", () => {@@ -516,7 +512,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_window to move, resize, maximize, minimize, focus or restore an app window by its address without keys or clicks, or to give it a short name.",parameters: windowParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_window", args => `${args.window} ${args.action}${args.action === "move" ? ` ${args.x},${args.y}` : args.action === "resize" ? ` ${args.width}x${args.height}` : args.action === "name" ? ` ${args.as}` : ""}`),+ ...rendered("computer_window"),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.window, params.action, () => {@@ -530,7 +526,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_view to make, move or resize a viewport: a named rectangle of a window that follows the window. The result shows the viewport at the window's own resolution. Give its address to the computer_* input tools to act at points of that image and see what changed only there; Read it to look again.",parameters: viewParameters,annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },- ...rendered("computer_view", args => `${args.viewport}${args.x !== undefined || args.y !== undefined ? ` at ${args.x ?? "_"},${args.y ?? "_"}` : ""}${args.width !== undefined || args.height !== undefined ? ` ${args.width ?? "_"}x${args.height ?? "_"}` : ""}`),+ ...rendered("computer_view"),async execute(_id, params, signal, _update, ctx) {const { viewport, ...rect } = params;return actionResult(await computerAction(resources, viewport, "view", () => resources.address(viewport),@@ -542,7 +538,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_wait to wait until a window's UI or a browser page meets conditions, or to check them once, without sending input. The last sample becomes the window's current observation.",parameters: waitParameters,annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },- ...rendered("computer_wait", args => `${args.window} ${args.mode ?? "wait"} ${(args.conditions as unknown[] | undefined)?.length ?? 0} condition(s)`),+ ...rendered("computer_wait"),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.window, params.mode === "check" ? "verify" : "wait_for", () => {@@ -558,7 +554,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_browser to navigate, manage tabs, answer JavaScript dialogs, upload or download files, and run page JavaScript in a browser window. The result shows the page's fresh snapshot.",parameters: browserParameters,annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true },- ...rendered("computer_browser", args => `${args.window} ${args.action}${args.url ? ` ${args.url}` : args.tab ? ` ${args.tab}` : ""}`),+ ...rendered("computer_browser"),async execute(_id, params, signal, _update, ctx) {const { window, ...command } = params;return actionResult(await computerAction(resources, window, params.action, () => resources.address(window),@@ -570,7 +566,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_files to copy files between the workspace and this session's own folder on a machine, or to list that folder, so apps there can open or save them.",parameters: filesParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_files", args => `${args.machine} ${args.action}${args.from ? ` ${args.from}` : args.name ? ` ${args.name}` : ""}`),+ ...rendered("computer_files"),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.machine, params.action, () => {@@ -584,7 +580,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_clipboard to read or replace a machine's clipboard in YOLO mode, with text or an image.",parameters: clipboardParameters,annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },- ...rendered("computer_clipboard", args => `${args.machine} ${args.action}${args.image ? ` ${args.image}` : ""}`),+ ...rendered("computer_clipboard"),async execute(_id, params, signal, _update, ctx) {let command: Record<string, unknown> = {};return actionResult(await computerAction(resources, params.machine, `clipboard_${params.action}`, () => {@@ -598,7 +594,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_record to record a machine's display to video or its playback audio, to capture frames of a window or display over time for motion and animation, or to watch a window or viewport in the background while acting on it, such as panning a panorama or dragging a 3D camera.",parameters: recordParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_record", args => `${args.at} ${args.action}${args.seconds ? ` ${args.seconds} s` : args.duration_ms ? ` ${args.duration_ms} ms` : ""}`),+ ...rendered("computer_record"),async execute(_id, params, signal, _update, ctx) {let run: RecordRun = { command: {} };return actionResult(await computerAction(resources, params.at, params.action, () => {@@ -628,56 +624,56 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResou...common, name: "computer_click", label: "Computer click",description: "Use computer_click to click an element or a point of a window: any button, double or triple clicks, with modifier keys. The result lists what changed.",parameters: clickParameters, annotations: inputAnnotations,- ...rendered("computer_click", args => `${args.window} ${aimed(args)}${args.count ? ` x${args.count}` : ""}${args.button && args.button !== "left" ? ` ${args.button}` : ""}`),+ ...rendered("computer_click"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "click", () => clickPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_hover", label: "Computer hover",description: "Use computer_hover to rest the pointer on an element or a point without clicking, for tooltips, hover menus and hover states.",parameters: hoverParameters, annotations: inputAnnotations,- ...rendered("computer_hover", args => `${args.window} ${aimed(args)}`),+ ...rendered("computer_hover"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "hover", () => hoverPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_drag", label: "Computer drag",description: "Use computer_drag to press a button at one element or point, move to another, optionally through a path, and release there, for sliders, drag and drop and drawing.",parameters: dragParameters, annotations: inputAnnotations,- ...rendered("computer_drag", args => `${args.window} ${aimed(args.from as Record<string, unknown>)} → ${aimed(args.to as Record<string, unknown>)}`),+ ...rendered("computer_drag"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "drag", () => dragPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_scroll", label: "Computer scroll",description: "Use computer_scroll to turn the mouse wheel over an element, a point or the middle of a window or page.",parameters: scrollParameters, annotations: inputAnnotations,- ...rendered("computer_scroll", args => `${args.window} ${aimed(args)} ${args.delta_x ?? 0},${args.delta_y ?? 0}`),+ ...rendered("computer_scroll"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "scroll", () => scrollPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_type", label: "Computer type",description: "Use computer_type to type text into a field or the focused element, or to replace a field's value, optionally pressing Enter after it.",parameters: typeParameters, annotations: inputAnnotations,- ...rendered("computer_type", args => `${args.window} ${aimed(args)} ${String(args.text ?? "").length} chars${args.submit ? " + Enter" : ""}`),+ ...rendered("computer_type"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "type", () => typePlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_press", label: "Computer press",description: "Use computer_press to press a key or shortcut such as Enter or ctrl+s, or to hold keys down for a time.",parameters: pressParameters, annotations: inputAnnotations,- ...rendered("computer_press", args => `${args.window} ${args.key}${args.hold_ms ? ` for ${args.hold_ms} ms` : ""}`),+ ...rendered("computer_press"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "press", () => pressPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_select", label: "Computer select",description: "Use computer_select to choose options in a list or select element, set a checkbox, radio or toggle, or open an app's menu item.",parameters: selectParameters, annotations: inputAnnotations,- ...rendered("computer_select", args => `${args.window} ${args.menu ? (args.menu as string[]).join(" > ") : `${args.target ?? ""} ${args.values ? (args.values as string[]).join(", ") : args.checked !== undefined ? (args.checked ? "checked" : "unchecked") : ""}`}`),+ ...rendered("computer_select"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "select", () => selectPlan(params), ctx.cwd, signal); },});pi.registerTool({...common, name: "computer_timeline", label: "Computer timeline",description: "Use computer_timeline to play timed key, button, pointer, wheel and pen events in one call, for games, held keys across moves and pressure-sensitive strokes.",parameters: timelineParameters, annotations: inputAnnotations,- ...rendered("computer_timeline", args => `${args.window} ${(args.events as unknown[] | undefined)?.length ?? 0} events`),+ ...rendered("computer_timeline"),async execute(_id, params, signal, _update, ctx) { return input(params.window, "timeline", () => timelinePlan(params), ctx.cwd, signal); },});pi.registerTool({@@ -685,7 +681,7 @@ export function registerComputerTools(pi: ExtensionAPI, resources: ComputerResoudescription: "Use computer_display to change a display's resolution in YOLO mode, for example to test an app at another size; closing computer use restores it.",parameters: displayParameters,annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false },- ...rendered("computer_display", args => `${args.display} ${args.width}x${args.height}`),+ ...rendered("computer_display"),async execute(_id, params, signal, _update, ctx) {return actionResult(await computerAction(resources, params.display, "resize", () => resources.address(params.display),address => resources.perform(address, ctx.cwd, JSON.stringify({ action: "resize", width: params.width, height: params.height }), signal)));@@ -700,8 +696,7 @@ export function registerComputerDelete(pi: ExtensionAPI, resources: ComputerResodescription: "Use computer_delete to close an app window, or to release a sandbox, isolated session, machine or all of computer use.",parameters: Type.Object({ path: Type.String({ description: COMPUTER_DELETE_HELP }) }, { additionalProperties: false }),annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },- renderCall: args => new Text(`computer_delete ${plain(args.path)}`, 0, 0),- renderResult: computerActionRenderer("computer_delete").renderResult,+ ...resourceRenderers("computer_delete", resource => resources.describe(resource)),async execute(_id, params, signal, _update, ctx) {return actionResult(await deleteComputer(resources, params.path, params, ctx.cwd, signal));},diff --git a/src/computer-tui.ts b/src/computer-tui.tsnew file mode 100644index 0000000..f435829--- /dev/null+++ b/src/computer-tui.ts@@ -0,0 +1,208 @@+import type { Theme, ThemeColor } from "@earendil-works/pi-coding-agent";+import { keyText } from "@earendil-works/pi-coding-agent";+import { type Component, truncateToWidth, visibleWidth, wrapTextWithAnsi } from "@earendil-works/pi-tui";++/**+ * What a person sees of a Computer Use call and its outcome in the native Pi TUI.+ *+ * Everything here is presentation: it reads typed result fields (never the agent's receipt text) and draws them in the+ * conventions of Pi Agent IDE's tool cards, a bold tool title with a primary value and muted qualifiers, and a status+ * line with an icon, so the two kinds of tool read alike. Nothing here changes what the agent receives.+ */++/** The state of a call, as a person reads it. Refused means nothing was sent; unknown means it may have been. */+export type ComputerState = "running" | "observed" | "listed" | "delivered" | "closed" | "refused" | "unknown" | "failed";++const STATES: Record<ComputerState, { icon: string; tone: ThemeColor }> = {+ running: { icon: "●", tone: "accent" },+ observed: { icon: "✓", tone: "success" },+ listed: { icon: "✓", tone: "success" },+ delivered: { icon: "✓", tone: "success" },+ closed: { icon: "✓", tone: "success" },+ refused: { icon: "✗", tone: "warning" },+ unknown: { icon: "?", tone: "warning" },+ failed: { icon: "✗", tone: "error" },+};++/** One styled value of a call heading. */+export interface HeadingPart { text: string; color?: ThemeColor }++/** A call heading: the tool, what it acts on, and the options that matter, most important first. */+export interface Heading {+ tool: string;+ primary?: HeadingPart;+ qualifiers?: readonly string[];+ /** Exact arguments, shown only when the card is expanded. */+ details?: readonly (readonly [label: string, value: string])[];+}++/** A note under the status line. */+export interface Note { tone: ThemeColor; text: string }++/** The outcome of a call, ready to draw. */+export interface Outcome {+ state: ComputerState;+ /** What the status line says after the state: where, on what, how big. */+ facts?: readonly string[];+ /** Why, or what to know, always shown (wrapped). */+ notes?: readonly Note[];+ /** Short rows such as UI elements or listed windows. Compact shows the first few. */+ rows?: readonly string[];+ /** More rows than those given, when the source was already bounded. */+ moreRows?: number;+ /** Labelled facts shown only when expanded. */+ details?: readonly (readonly [label: string, value: string])[];+}++/** Rows and note lines a compact card shows before it offers to expand. */+export const COMPACT_ROWS = 6;+const EXPANDED_ROWS = 40;++/** App text is data, never terminal control sequences. One line, no layout characters. */+export function oneLine(value: unknown, max = 240): string {+ const text = String(value ?? "").replace(/[\x00-\x1f\x7f-\x9f]+/g, " ").replace(/\s+/g, " ").trim();+ return text.length > max ? `${text.slice(0, max - 1)}…` : text;+}++const separator = (theme: Theme) => theme.fg("dim", " · ");++/** A width-safe tool-call heading: `tool primary · qualifier · qualifier`, dropping qualifiers before it clips the primary. */+export class HeadingLine implements Component {+ constructor(private heading: Heading, private theme: Theme, private expanded: boolean) {}+ update(heading: Heading, theme: Theme, expanded: boolean): void { this.heading = heading; this.theme = theme; this.expanded = expanded; }+ invalidate(): void {}+ render(width: number): string[] {+ const w = Math.max(1, width), theme = this.theme, { tool, primary, details } = this.heading;+ const title = theme.fg("toolTitle", theme.bold(oneLine(tool)));+ let line = title;+ if (primary && visibleWidth(title) < w) {+ const qualifiers = (this.heading.qualifiers ?? []).map(q => oneLine(q)).filter(Boolean);+ const sep = separator(theme), sepWidth = visibleWidth(sep);+ let kept = qualifiers.length;+ const total = () => visibleWidth(title) + 1 + visibleWidth(oneLine(primary.text)) ++ qualifiers.slice(0, kept).reduce((sum, q) => sum + sepWidth + visibleWidth(q), 0) + (kept < qualifiers.length ? sepWidth + 4 : 0);+ while (kept > 0 && total() > w) kept--;+ const shown = qualifiers.slice(0, kept).map(q => `${sep}${theme.fg("muted", q)}`).join("");+ // The "… +N" marker is worth its room only when the target fits whole beside it.+ const more = kept < qualifiers.length && total() <= w ? `${sep}${theme.fg("dim", `… +${qualifiers.length - kept}`)}` : "";+ const room = Math.max(1, w - visibleWidth(title) - 1 - visibleWidth(shown) - visibleWidth(more));+ line = `${title} ${theme.fg(primary.color ?? "accent", truncateToWidth(oneLine(primary.text), room))}${shown}${more}`;+ }+ const lines = [truncateToWidth(line, w)];+ if (this.expanded && details?.length) {+ const text = [theme.fg("dim", "args ")].concat(details.map(([label, value], i) =>+ `${i === 0 ? "" : theme.fg("dim", " · ")}${theme.fg("muted", `${oneLine(label)}=`)}${theme.fg("toolOutput", oneLine(value))}`)).join("");+ lines.push(...wrapTextWithAnsi(text, w).map(l => truncateToWidth(l, w)));+ }+ return lines;+ }+}++/** The outcome of a call below its heading: a status line, notes, and a few rows that expand on demand. */+export class OutcomeCard implements Component {+ constructor(private outcome: Outcome, private theme: Theme, private expanded: boolean) {}+ update(outcome: Outcome, theme: Theme, expanded: boolean): void { this.outcome = outcome; this.theme = theme; this.expanded = expanded; }+ invalidate(): void {}+ render(width: number): string[] {+ const w = Math.max(1, width), theme = this.theme, { state, facts = [], notes = [], rows = [], moreRows = 0, details = [] } = this.outcome;+ const { icon, tone } = STATES[state];+ const status = theme.fg(tone, `${icon} ${state}`);+ const line = [status, ...facts.filter(Boolean).map(f => theme.fg("muted", oneLine(f)))].join(separator(theme));+ const lines = [truncateToWidth(line, w)];+ const indent = " ", inner = Math.max(1, w - indent.length);+ for (const note of notes) for (const part of wrapTextWithAnsi(theme.fg(note.tone, oneLine(note.text, 600)), inner)) lines.push(indent + truncateToWidth(part, inner));+ const limit = this.expanded ? EXPANDED_ROWS : COMPACT_ROWS;+ for (const row of rows.slice(0, limit)) lines.push(indent + truncateToWidth(theme.fg("dim", oneLine(row, 400)), inner));+ const hidden = Math.max(0, rows.length - limit) + moreRows;+ if (this.expanded) {+ for (const [label, value] of details) lines.push(indent + truncateToWidth(`${theme.fg("muted", `${oneLine(label)} `)}${theme.fg("toolOutput", oneLine(value, 400))}`, inner));+ if (hidden > 0) lines.push(indent + truncateToWidth(theme.fg("muted", `… ${hidden} more`), inner));+ } else if (hidden > 0) {+ lines.push(indent + truncateToWidth(theme.fg("muted", `… ${hidden} more · ${keyText("app.tools.expand")} to expand`), inner));+ }+ return lines;+ }+}++// --- Reading typed result fields. Anything absent or of another type is simply not shown. ---++export const record = (value: unknown): Record<string, unknown> =>+ value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};+const array = (value: unknown): unknown[] => Array.isArray(value) ? value : [];+const str = (value: unknown): string | undefined => typeof value === "string" && value.trim() ? value : undefined;+const num = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? value : undefined;++/** The opening of a message for a status note: its first line, up to two sentences. */+export function firstReason(message: unknown, max = 220): string | undefined {+ const text = str(message);+ if (!text) return undefined;+ const meaningful = text.split(/\r?\n/).map(l => l.trim()).filter(l => l && !l.startsWith("{") && !l.startsWith("}"));+ // A header that ends in a colon ("Validation failed for tool x:") is only worth showing with the line it introduces.+ const first = meaningful[0] ?? text;+ const line = oneLine(first.endsWith(":") && meaningful[1] ? `${first} ${meaningful[1].replace(/^-\s*/, "")}` : first, 2000);+ const sentences = line.split(/(?<=[.!?])\s+/);+ let reason = sentences[0];+ if (sentences[1] && reason.length + sentences[1].length < max) reason += ` ${sentences[1]}`;+ return oneLine(reason, max);+}++/** Names for what an address points at, supplied by the live session; absent for an address from an earlier run. */+export type DescribeAddress = (source: string) => { machine?: string; title?: string } | undefined;++/** The machine and window title an address names in this run, for display. */+export function targetNames(describe: DescribeAddress | undefined, source: unknown): { machine?: string; title?: string } {+ const address = str(source);+ const named = address ? describe?.(address) : undefined;+ return { ...(named?.machine ? { machine: named.machine } : {}), ...(named?.title ? { title: `“${oneLine(named.title, 60)}”` } : {}) };+}++/** The state of a typed action from its semantic data. */+export function actionState(data: Record<string, unknown>, isError: boolean): ComputerState {+ const status = str(data.status);+ if (status === "delivered" || status === "observed" || status === "listed" || status === "closed" || status === "unknown" || status === "refused") {+ // A delivered receipt that the adapter still marked not ok (a failed postcondition) is not a success.+ return data.ok === false && status !== "unknown" && status !== "refused" ? "unknown" : status;+ }+ if (data.ok === true && !isError) return "delivered";+ if (data.effect === "not-applied") return "refused";+ if (data.effect === "unknown") return "unknown";+ return isError ? "failed" : "delivered";+}++/** Notes for a refused, unknown or failed action: the reason in a person's terms, and what to do not do. */+export function problemNotes(state: ComputerState, data: Record<string, unknown>, fallback?: string): Note[] {+ if (state !== "refused" && state !== "unknown" && state !== "failed") return [];+ const script = record(data.script), error = record(data.error);+ const reason = firstReason(script.reason) ?? firstReason(error.message) ?? fallback;+ const notes: Note[] = [];+ if (reason) notes.push({ tone: state === "failed" ? "error" : "warning", text: reason });+ if (state === "unknown") notes.push({ tone: "muted", text: "It may have been applied. Read the window before acting again; do not repeat it blindly." });+ return notes;+}++/** What changed in the UI after an action, from the successor observation's diff. */+export function successorNotes(successor: unknown): { notes: Note[]; rows: string[] } {+ const diff = record(record(successor).diff);+ if (!Object.keys(diff).length) return { notes: [], rows: [] };+ const counts = record(diff.counts), changed = num(counts.changed) ?? 0, added = num(counts.added) ?? 0, removed = num(counts.removed) ?? 0;+ const waited = num(diff.waited_ms);+ const what = diff.kind === "dom" ? "page" : diff.kind === "pixels" ? "pixels" : "UI";+ if (changed + added + removed === 0) {+ const pixels = record(diff.pixels);+ return { notes: [{ tone: "warning", text: `Nothing changed in the ${what}${waited !== undefined ? ` within ${waited} ms` : ""}. Check the target before acting again.${pixels.changed === true ? "" : ""}` }], rows: [] };+ }+ const parts = [changed && `${changed} changed`, added && `${added} added`, removed && `${removed} removed`].filter(Boolean).join(", ");+ const rows = [...array(diff.changed).map(l => `~ ${oneLine(l, 160)}`), ...array(diff.added).map(l => `+ ${oneLine(l, 160)}`), ...array(diff.removed).map(l => `- ${oneLine(l, 160)}`)];+ return { notes: [{ tone: "muted", text: `${what} ${parts}${waited !== undefined ? ` within ${waited} ms` : ""}` }], rows };+}++/** `500×350` from a width and height. */+export const size = (width: unknown, height: unknown): string | undefined => num(width) !== undefined && num(height) !== undefined ? `${width}×${height}` : undefined;++/** UI element rows `hint role name`; values are left out, they can be typed text or secrets. */+export function elementRows(elements: unknown): string[] {+ return array(elements).map(raw => {+ const e = record(raw);+ return [str(e.hint), str(e.role), str(e.name) ? `“${oneLine(e.name, 80)}”` : undefined, e.enabled === false ? "(disabled)" : undefined].filter(Boolean).join(" ");+ }).filter(Boolean);+}diff --git a/src/provider.ts b/src/provider.tsindex 30972fd..3913544 100644--- a/src/provider.ts+++ b/src/provider.ts@@ -11,8 +11,7 @@ import { REMOTE_DESKTOP_DISPLAY } from "./display-mode.ts";import type { WebLogEntry } from "./web-browser.ts";import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";import type { PixelBox } from "./pixel-diff.ts";-import { computerActionRenderer } from "./computer-action-renderer.ts";-import { computerReadRenderer } from "./computer-read-renderer.ts";+import { computerActionRenderer, computerReadRenderer, computerSearchRenderer } from "./computer-renderers.ts";import type { ComputerSession, ComputerResult, ComputerMode, Content } from "./session.ts";const owns = (source: unknown): source is string => typeof source === "string" && source.startsWith("computer:");@@ -452,6 +451,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwmarkdown: guide,triggers: ["read", "search"].map(tool => ({ tool, resourcePrefixes: ["computer:"] })) }]);const resources = new ComputerResources(getSession, closeSession, openSandbox, connectMachine, getMode);+ const describe = (source: string) => resources.describe(source);// Pi shutdown releases every machine, sandbox and session this computer use started.pi.on("session_shutdown", () => resources.closeAll().then(() => undefined));const resolver: ResourceResolver = {@@ -475,7 +475,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwawait Promise.all([optional(connectReadPlugin(pi, { protocol: READ_PROTOCOL, apiVersion: READ_API_VERSION, id: "computer", setup(api) {accepted.add("read");- api.addResolver({ resolver, priority: -250, renderResult: computerReadRenderer });+ api.addResolver({ resolver, priority: -250, renderResult: computerReadRenderer(describe) as never });for (const view of ["image", "ui", "full", "state", "dom", "region", "sequence", "frames", "capture", "element", "ocr", "changes", "regions", "aria"])api.addView({ view, contentKind: "any", presenter: { id: `computer-${view}`, present: document => document } });api.addHandler({ stage: "read", when: { resolvedBy: resolver.id, contentKind: "any" }, async handler(context) {@@ -489,7 +489,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cw} })),optional(connectSearchPlugin(pi, { protocol: SEARCH_PROTOCOL, apiVersion: SEARCH_API_VERSION, id: "computer", setup(api) {accepted.add("search");- api.addResolver({ priority: -250, resolver: { id: "computer", readResources: request => owns(request.path) ? [request.path] : [],+ api.addResolver({ priority: -250, resolver: { id: "computer", renderResult: computerSearchRenderer(describe) as never, readResources: request => owns(request.path) ? [request.path] : [],async tryResolve(request, context) {if (!owns(request.path)) return { kind: "not-handled" };try {@@ -511,7 +511,7 @@ export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwlet renderersRegistered = false;pi.on("session_start", () => {if (renderersRegistered) return;- for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool));+ for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool, describe) as never);renderersRegistered = true;});// Write and Insert do not act on computer addresses; they refuse instead of treating the address as a file.diff --git a/src/standalone-tools.ts b/src/standalone-tools.tsindex c59bc55..f3b18e6 100644--- a/src/standalone-tools.ts+++ b/src/standalone-tools.ts@@ -1,20 +1,19 @@import { Type, type JsonValue } from "@earendil-works/pi-ai";import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";-import { Text } from "@earendil-works/pi-tui";import type { ComputerResources } from "./computer-resources.ts";-import { computerReadRenderer } from "./computer-read-renderer.ts";+import { resourceRenderers } from "./computer-renderers.ts";import { computerNamespace, registerComputerDelete } from "./computer-tools.ts";import { COMPUTER_PATH_HELP, COMPUTER_SEARCH_HELP, COMPUTER_VIEWS_HELP, presentComputerResult, readComputer } from "./provider.ts";const GUIDE = "docs:computer";const image = Type.Object({ type: Type.Literal("image"), data: Type.String(), mimeType: Type.String() });-const plain = (value: unknown) => String(value).replace(/[\x00-\x1f\x7f-\x9f]/g, " ");/*** Offer computer use without Pi Agent IDE: computer_read, computer_search and computer_delete stand in for the IDE's Read,* Search and Delete. Like the other computer_* tools they are deferred, off the model's tool list until tool_search loads them.*/export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerResources, guide: string): void {+ const describe = (source: string) => resources.describe(source);const namespace = computerNamespace();pi.registerTool({name: "computer_read", label: "Computer read", exposure: "deferred", namespace,@@ -31,8 +30,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerResType.Object({ type: Type.Literal("images"), note: Type.String(), images: Type.Array(image) }),]),annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },- renderCall: args => new Text(`computer_read ${plain(args.path)}${args.views?.length ? ` [${plain(args.views.join(", "))}]` : ""}`, 0, 0),- renderResult: computerReadRenderer,+ ...resourceRenderers("computer_read", describe),async execute(_id, params, signal, _update, ctx) {if (params.path === GUIDE) {const lines = guide.replace(/\n$/, "").split("\n"), offset = params.offset ?? 1;@@ -59,7 +57,7 @@ export function registerStandaloneTools(pi: ExtensionAPI, resources: ComputerReslimit: Type.Optional(Type.Integer({ minimum: 1, description: "Maximum results." })),}, { additionalProperties: false }),annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },- renderCall: args => new Text(`computer_search ${plain(args.path)} ${plain(JSON.stringify(args.query))}`, 0, 0),+ ...resourceRenderers("computer_search", describe),async execute(_id, params, signal, _update, ctx) {const result = await resources.search(resources.address(params.path), ctx.cwd, params.query, params.limit, signal);return { content: [{ type: "text", text: resources.shorten(presentComputerResult(result)) }], details: result.details, ...(result.isError ? { isError: true } : {}) };diff --git a/test/computer-action-renderer.test.ts b/test/computer-action-renderer.test.tsdeleted file mode 100644index 3f93c1f..0000000--- a/test/computer-action-renderer.test.ts+++ /dev/null@@ -1,53 +0,0 @@-import { visibleWidth } from "@earendil-works/pi-tui";-import { expect, test } from "vitest";-import { computerActionRenderer } from "../src/computer-action-renderer.ts";--function render(summary: string, width = 80, guide = "") {- const renderer = computerActionRenderer("insert");- return renderer.renderResult!({ content: [{ type: "text", text: summary }, { type: "text", text: guide }], details: {- metadata: { semanticAction: { kind: "computer-action", receipt: "PRIVATE_NATIVE_DIAGNOSTIC", error: { message: "PRIVATE_NATIVE_ERROR" } } },- } } as never, { expanded: false, isPartial: false }, {} as never, {} as never).render(width);-}--test("computer renderers select only computer calls and semantic computer results", () => {- for (const tool of ["write", "insert"] as const) {- const renderer = computerActionRenderer(tool);- expect(renderer.matches?.({ path: "computer:observation/owned" }, "call")).toBe(true);- expect(renderer.matches?.({ path: "notes.txt" }, "call")).toBe(false);- expect(renderer.matches?.({ path: ["computer:session"] }, "call")).toBe(false);- expect(renderer.matches?.({ details: { metadata: { semanticAction: { kind: "computer-action" } } } }, "result")).toBe(true);- for (const result of [undefined, { details: {} }, { details: { metadata: { semanticAction: { kind: "file-action" } } } }])- expect(renderer.matches?.(result, "result")).toBe(false);- }-});--test.each([- "Computer refused\nDOM action: refused; trusted=false; atomic=false; effects=unverified\nAttempted steps: []; acknowledged: []\nIn flight: none; reason=dom_action_trusted_route_unavailable; cleanup=not_started",- 'Computer unknown\nDOM action: refused; effects=unverified\nAttempted steps: ["click","dblclick"]; acknowledged: ["click"]\nIn flight: dblclick; progress=unknown; cleanup=unconfirmed',- 'Computer delivered\nDOM action: dispatched; effects=unverified\nAttempted steps: ["click"]; acknowledged: ["click"]\nIn flight: none; cleanup=confirmed',-])("keeps the sanitized receipt, not private metadata, even when collapsed", summary => {- const rendered = render(summary, 180).join("\n");- for (const line of summary.split("\n")) expect(rendered).toContain(line);- expect(rendered).not.toContain("PRIVATE_NATIVE");- expect(rendered).not.toContain("edit failed");-});--test("wraps wide receipt text and never emits app-provided terminal controls", () => {- const rendered = render("Computer refused\nDOM refusal: 测试 e\u0301 \u001b[2J unsafe", 18);- expect(rendered.every(line => visibleWidth(line) <= 18)).toBe(true);- expect(rendered.join("\n")).not.toContain("\u001b");- expect(rendered.join("\n")).toContain("测试");-});--test("keeps model-only result envelopes and attached guides out of the terminal receipt", () => {- const rendered = render('<system-result note="Internal reference"><uuid>PRIVATE_REFERENCE</uuid></system-result>\nComputer refused', 80, "PRIVATE_AGENT_GUIDE").join("\n");- expect(rendered).toContain("Computer refused");- expect(rendered).not.toContain("PRIVATE_");- expect(rendered).not.toContain("system-result");-});-test("computer call headings omit literal input and native data", () => {- const renderer = computerActionRenderer("write");- const lines = renderer.renderCall!({ path: "computer:observation/owned", content: "PRIVATE_TYPED_TEXT" }, {} as never, {} as never).render(100);- expect(lines.join("\n")).toContain("write computer:observation/owned");- expect(lines.join("\n")).not.toContain("PRIVATE_TYPED_TEXT");-});diff --git a/test/computer-read-renderer.test.ts b/test/computer-read-renderer.test.tsdeleted file mode 100644index c017308..0000000--- a/test/computer-read-renderer.test.ts+++ /dev/null@@ -1,28 +0,0 @@-import { visibleWidth } from "@earendil-works/pi-tui";-import { expect, test } from "vitest";-import { computerReadRenderer } from "../src/computer-read-renderer.ts";--const theme = { fg: (_: unknown, text: string) => text, bold: (text: string) => text };-const context = { args: { path: "computer:observation/current" }, cwd: "/", isError: false };-const render = (text: string, computer: unknown, width = 79) =>- computerReadRenderer({ content: [{ type: "text", text }, { type: "text", text: "PRIVATE_AGENT_GUIDE" }],- details: { source: context.args.path, resolvedBy: "computer", computer } } as never,- { expanded: false, isPartial: false }, theme as never, context as never).render(width);--test("a Playwright screenshot shows a short sanitized provenance line above Pi's enlarged image", () => {- const lines = render("Computer observed (yolo)\nImage: 36x36 image/png", {- status: "observed", state: { title: "Form\u001b[2J", url: "http://127.0.0.1:8123/form" },- agent_image: { route: "playwright_screenshot", kind: "crop", output: { width: 36, height: 36 } },- });- expect(lines).toHaveLength(2);- const output = lines.join("\n");- for (const value of ["Playwright crop screenshot", "36x36 PNG", "page pixels, input=none", "http://127.0.0.1:8123/form"]) expect(output).toContain(value);- expect(output).not.toContain("\u001b");- expect(output).not.toContain("PRIVATE_");- expect(lines.every(line => visibleWidth(line) <= 79)).toBe(true);-});-test("ordinary computer reads retain the standard source renderer", () => {- const lines = render("ordinary window overview", { status: "observed", agent_image: { route: "cua_native" } });- expect(lines.join("\n")).toContain("ordinary window overview");- expect(lines.join("\n")).toMatch(/[╭┌]/);-});