Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

48.1 KB
Raw Blame
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { connectReadPlugin, READ_PROTOCOL, READ_API_VERSION } from "pi-agent-ide/api/read";
import { connectSearchPlugin, SEARCH_PROTOCOL, SEARCH_API_VERSION } from "pi-agent-ide/api/search";
import { connectTextEditorPlugin, TEXT_EDITOR_PROTOCOL, TEXT_EDITOR_API_VERSION } from "pi-agent-ide/api/text-editor";
import type { ResourceResolver } from "pi-agent-ide/api/resource";
import { connectAgentDocumentation } from "pi-agent-ide/api/documentation";
import { registerIdeCapability } from "pi-agent-ide/api/capabilities";
import { readFile } from "node:fs/promises";
import { ComputerResources, computerResultText, ComputerRequestError, FORCE, LOCAL_MACHINE, type SandboxFactory, type MachineFactory } from "./computer-resources.ts";
import { computerSource, displaySource, type ComputerAddress } from "./computer-address.ts";
import { REMOTE_DESKTOP_DISPLAY } from "./display-mode.ts";
import type { WebLogEntry } from "./web-browser.ts";
import { diffIsEmpty, type PixelReport, type StateDiff } from "./successor.ts";
import type { PixelBox } from "./pixel-diff.ts";
import { computerActionRenderer, computerReadRenderer, computerSearchRenderer } from "./computer-renderers.ts";
import type { ComputerSession, ComputerResult, ComputerMode, Content } from "./session.ts";
import { fillToolNames, IDE_TOOL_NAMES, type ComputerToolNames } from "./tool-names.ts";

const owns = (source: unknown): source is string => typeof source === "string" && source.startsWith("computer:");
const record = (value: unknown): Record<string, unknown> => value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
const label = (value: unknown) => typeof value === "string" ? value.replace(/[\r\n\t]/g, " ").slice(0, 1024) : "";

/** Agent guidance shared by the IDE plugin and the standalone tools, one form per line like the IDE's own field lines. */
export const computerPathHelp = (names: ComputerToolNames) => [
  `computer:machine/<name> — a machine, local or configured; /windows and /apps for ${names.search}, /displays, /display/primary and /display/<ref> for display pixels.`,
  `computer:window/… — a window from ${names.search} or computer_open: UI tree and overview image; computer:<name> after computer_window name.`,
  `computer:window/…/<viewport> — a viewport from computer_view: its image at full resolution.`,
  `computer:observation/… — the result of an earlier ${names.read}; computer:sequence/… — frames from computer_record.`,
  `computer:session — machines, sandboxes, windows and names so far.`,
  `Results show short forms such as computer:lab/w3qk; docs:computer has the rest.`,
].join("\n");
export const computerViewsHelp = (names: ComputerToolNames) => [
  `On a computer window: full (default), image, ui — pixels and UI, pixels only, UI only.`,
  `aria — UI as a Playwright aria snapshot with [ref=…].`,
  `dom[:{target,depth,boxes,log}] — aria snapshot of a browser page opened with computer_open; use alone.`,
  `state — minimized, maximized, foreground, bounds, hung; use alone.`,
  `changes — only what changed since the window's previous observation.`,
  `ocr — Windows OCR lines with boxes; regions — icons and text areas from Cua perception.`,
  `capture:{maxLongEdge,scale,max_elements,max_depth,hints} — limits of one ${names.read}.`,
  `element:inspect|scope|highlight|crop|text,<ref> — one element of an observation.`,
  `region:x,y,w,h — part of a display observation.`,
  `frames:first|last|all|endpoints|changed or frames:{indexes|from,to,as:sheet|max|min} — frames of a sequence.`,
].join("\n");
export const COMPUTER_SEARCH_HELP = "computer:machine/<name>/windows or /apps, or an observation address.";
export const COMPUTER_DELETE_HELP = "computer:window/… closes that window (?force=true kills its hung app if this session opened it); computer:sandbox/…, computer:machine/<name> and computer:session release what they name.";
/** Each Insert command the typed tools replaced, by the tool that does it now. */
const REPLACED: [RegExp, string][] = [
  [/^(click|double_click|right_click|invoke|check|uncheck)$/, "computer_click"], [/^(type_text|type|fill|set_value)$/, "computer_type"],
  [/^(key|hotkey|press)$/, "computer_press"], [/^hold$/, "computer_press with hold_ms"], [/^(scroll|wheel)$/, "computer_scroll"], [/^drag$/, "computer_drag"], [/^(hover|move_pointer)$/, "computer_hover"],
  [/^(select|select_option|menu)$/, "computer_select"], [/^(timeline|sequence)$/, "computer_timeline"],
  [/^(open|open_browser|open_sandbox|open_session|attach_browser|webview2_debugging)$/, "computer_open"],
  [/^(move|resize|minimize|maximize|restore|name)$/, "computer_window (a pointer move: computer_hover; resize of a display: computer_display)"], [/^(wait_for|verify)$/, "computer_wait"],
  [/^(navigate|back|forward|reload|wait_load|new_tab|switch_tab|close_tab|dialog|upload|wait_download|cancel_download|evaluate)$/, "computer_browser"],
  [/^(put|get|files)$/, "computer_files"], [/^clipboard_/, "computer_clipboard"], [/^(record_start|record_stop|audio_record)$/, "computer_record"],
  [/^(close|kill)$/, "Delete (kill: Delete with ?force=true)"], [/^setup$/, "/computer-setup, which only the user runs (ask them)"], [/^batch$/, "Codemode: a script that calls the computer_* tools in order"],
];
/** Why Write or Insert refuses a computer address, naming the tool that does what it was sent. */
export function computerEditRefusal(tool: "write" | "insert", input: Record<string, unknown>): string {
  let action: unknown = tool === "write" ? "type_text" : undefined;
  if (tool === "insert" && typeof input.text === "string") try { action = (JSON.parse(input.text) as { action?: unknown })?.action; } catch { /* no action to map */ }
  const replacement = typeof action === "string" ? REPLACED.find(([pattern]) => pattern.test(action as string))?.[1] : undefined;
  return `${tool === "write" ? "Write" : "Insert"} does not act on computer resources; nothing was sent. ${replacement ? `Use ${replacement}. ` : ""}Each operation has its own computer_* tool: tool_search loads them with "computer open window", "computer click hover drag scroll", "computer type press select timeline", "computer wait browser", "computer files clipboard record" or "computer view viewport".`;
}

/** Present native resources without dumping provenance; full receipts remain in structured data. */
export function presentComputerResult(result: ComputerResult, full = false): string {
  const d = result.details, state = d.state ?? {};
  // Every routed receipt names the machine (and sandbox) it ran on.
  const lines = [`Computer ${d.status} (${d.mode})${d.machine ? ` on ${d.machine}${d.sandbox ? `, sandbox ${d.sandbox}` : ""}` : ""}`];
  if (d.window_ref) lines.push(`Window: ${computerSource("window", d.window_ref)}`);
  if (d.viewport) lines.push(`Viewport: ${d.viewport.source}; points on it are pixels of its ${d.viewport.image.width}x${d.viewport.image.height} image`);
  if (d.window_state_id && d.status === "observed") {
    const sample = d.window_state_kind === "management" ? record(state.state) : state;
    lines.push(`Window state: minimized=${sample.minimized}, visible=${sample.visible}; foreground=${sample.foreground_at_target}, keyboard_focus=${sample.keyboard_focus_at_target}${d.window_state_kind === "management" ? `; maximized=${sample.maximized}` : ""}`);
    if (d.window_state_kind === "management") {
      lines.push(`Bounds: ${JSON.stringify(sample.bounds)}; source=${sample.bounds_source}; unit=${sample.coordinate_space}`,
        `Current virtual desktop: ${record(sample.virtual_desktop).on_current_desktop}`);
    }
  }
  const invalidManagement = state.code === "tool_output_invalid" ? record(state.invalid_output) : {};
  const management = typeof state.mutation_attempted === "boolean" ? state : invalidManagement;
  if (typeof management.mutation_attempted === "boolean") {
    const measured = record(management.observed), unvalidated = management === invalidManagement;
    lines.push(`${unvalidated ? "Unvalidated" : "Native"} management: ${management.operation}; mutation_attempted=${management.mutation_attempted}, mutation_accepted=${management.mutation_accepted}, postcondition_confirmed=${management.postcondition_confirmed}`,
      `Requested bounds: ${JSON.stringify(management.requested_bounds)}; observed bounds: ${JSON.stringify(measured.bounds ?? null)}; source=${measured.bounds_source ?? "unavailable"}`,
      `${unvalidated ? "Reported" : "Measured"} state: minimized=${measured.minimized}, maximized=${measured.maximized}, visible=${measured.visible}; foreground=${measured.foreground_at_target}, keyboard_focus=${measured.keyboard_focus_at_target}`);
    if (unvalidated) lines.push("The native schema rejected this payload. Keep outcome unknown; do not treat its reported completion as validated success.");
  }
  if (typeof state.restore_attempted === "boolean") {
    lines.push(`Native restore prefix: restore_attempted=${state.restore_attempted}, restore_requested=${state.restore_requested}, restored=${state.restored}`,
      `Activation: activation_attempted=${state.activation_attempted}, activation_succeeded=${state.activation_succeeded}; foreground_changed=${state.foreground_changed}`,
      `Measured state: visible=${state.visible}; foreground=${state.foreground_at_target}, keyboard_focus=${state.keyboard_focus_at_target}`);
  }
  const id = d.observation_id ?? d.agent_image?.image_id;
  if (typeof id === "string" && (d.status === "observed" || d.dom && d.status === "delivered")) lines.push(`Observation: ${computerSource("observation", id)}`);
  if (d.agent_image) {
    const image = d.agent_image, output = record(image.output), configuration = record(image.configuration);
    lines.push(`Image: ${output.width}x${output.height} ${label(output.mimeType)}; ${label(image.route)}; ${label(image.coordinate_space)}`);
    if (configuration.maxLongEdge !== undefined) lines.push(`Capture: maxLongEdge=${configuration.maxLongEdge}, scale=${configuration.scale}`);
  }
  if (d.source) lines.push(`Source: ${d.source.captured_at}; input=${d.source.capabilities.input}; paint freshness=${d.source.visibility.paint_freshness}`);
  if (d.element_text) {
    const e = d.element_text;
    lines.push(computerResultText(result).split("\n{", 1)[0], `Element: ${e.hint} | ${label(e.role)} | ${label(e.name)}`, ...e.text.split(/\r?\n/));
  } else if (d.verification) {
    const v = d.verification;
    lines.push(computerResultText(result).split("\n{", 1)[0]);
    if (d.dom) lines.push(`Page: ${label(d.dom.title)} | ${label(d.dom.url)} | tab ${d.dom.active ?? "none"}`);
    v.conditions.forEach((condition, i) => {
      const spec = v.spec.conditions[i];
      const check = spec.is ? (spec.is === "exists" || spec.is === "absent" ? spec.is : `is ${spec.is}`) : spec.value !== undefined ? `value = ${JSON.stringify(spec.value)}` : spec.contains !== undefined ? `contains ${JSON.stringify(spec.contains)}`
        : spec.count !== undefined ? `count = ${spec.count}` : spec.aria !== undefined ? "matches the aria snapshot"
        : spec.title !== undefined ? `title = ${JSON.stringify(spec.title)}` : `url contains ${JSON.stringify(spec.url_contains)}`;
      const target = spec.target === undefined ? "" : typeof spec.target === "string" ? spec.target + " " : JSON.stringify(spec.target) + " ";
      // A missed aria snapshot shows the received tree as YAML lines, the way Playwright's diff shows it.
      const tree = spec.aria !== undefined && condition.observed != null && condition.verdict !== "satisfied";
      const seen = condition.observed == null || tree ? "" : label(condition.observed).length > 200 ? label(condition.observed).slice(0, 200) + "…" : label(condition.observed);
      lines.push(`Condition ${i + 1}: ${target}${check} → ${condition.verdict}${condition.observed != null && !tree ? ` (observed ${JSON.stringify(seen)})` : ""}${condition.reason ? `; ${label(condition.reason)}` : ""}`);
      if (tree) lines.push("Received aria snapshot:", ...condition.observed!.split("\n"));
    });
    if (d.ui) lines.push(`UI: collected=${d.ui.collected}, complete=${d.ui.complete}, truncated=${d.ui.truncated}. Read or Search the observation for elements and hints.`);
    // A browser page's last sample is a fresh snapshot: show it like a dom Read, so the next step needs no Read.
    if (d.dom) {
      if (d.dom.truncated) lines.push("Snapshot truncated; Read the window with dom:{\"target\":...,\"depth\":...} to narrow it.");
      lines.push(...d.dom.snapshot.split("\n"));
    }
  } else if (d.dom) {
    if (d.status !== "observed") {
      const message = computerResultText(result).split("\n{", 1)[0];
      if (message) lines.push(message);
    }
    const dom = d.dom;
    lines.push(dom.active ? `Page: ${label(dom.title)} | ${label(dom.url)} | tab ${dom.active}` : "Page: none; no tab is active. Use computer_browser switch_tab.");
    for (const tab of dom.tabs ?? [])
      lines.push(`Tab ${tab.id}${tab.active ? " (active)" : ""}: ${label(tab.title)} | ${label(tab.url)} | ${tab.load}${tab.opener ? ` | opened by ${tab.opener}` : ""}${tab.dialog ? ` | ${tab.dialog} dialog open` : ""}`);
    if (dom.dialog) lines.push(`Dialog: ${dom.dialog.type} ${JSON.stringify(dom.dialog.message)}${dom.dialog.default_value !== undefined ? ` (default ${JSON.stringify(dom.dialog.default_value)})` : ""}. ` +
      "The page waits for it: answer with computer_browser dialog accept true or false; an accepted prompt takes text. Other page actions refuse until then.");
    for (const item of dom.downloads ?? [])
      lines.push(`Download ${item.id}: ${label(item.filename)} | ${item.state} | tab ${item.tab}${item.bytes !== undefined ? ` | ${item.bytes} bytes` : ""}${item.path ? ` | ${label(item.path)}` : ""}${item.error ? ` | ${label(item.error)}` : ""}`);
    for (const event of dom.events ?? []) lines.push(`Event: ${label(event)}`);
    const entry = (item: WebLogEntry) => `#${item.seq} ${item.at.slice(11, 23)} ${item.tab} ${item.kind}${item.kind === "console" ? `.${item.level}` : ` ${item.level}`}: ${label(item.text)}${item.url ? ` | ${label(item.url)}` : ""}`;
    if (dom.evaluation) lines.push(dom.evaluation.error !== undefined ? `Evaluation error: ${label(dom.evaluation.error)}`
      : `Evaluation result${dom.evaluation.truncated ? " (truncated to 20000 characters)" : ""}: ${dom.evaluation.value}`);
    for (const item of dom.log?.entries ?? []) lines.push(`Log ${entry(item)}`);
    if (dom.log?.omitted) lines.push(`Log: ${dom.log.omitted} more new entries; Read dom:{"log":true} for the retained history.`);
    if (dom.history) {
      lines.push(`Log history: ${dom.history.entries.length} retained entries from #${dom.history.first_seq}${dom.history.dropped ? `; ${dom.history.dropped} older entries were dropped, so earlier history is incomplete` : ""}.`);
      for (const item of dom.history.entries) lines.push(`History ${entry(item)}`);
    }
    if (d.dom.matches !== undefined) lines.push(`Matching snapshot lines: ${d.dom.matches}`);
    if (d.dom.truncated) lines.push("Snapshot truncated; narrow it with dom:{\"target\":...,\"depth\":...}.");
    // With a successor, the compact change list replaces the full snapshot; Read the observation for all of it.
    if (!d.successor) lines.push(...d.dom.snapshot.split("\n"));
  } else if (d.ui) {
    lines.push(`UI: collected=${d.ui.collected}, complete=${d.ui.complete}, truncated=${d.ui.truncated || d.ui.projection_truncated}`);
    // A changes Read lists only the difference; the observation still holds, and Search finds, the whole UI.
    if (d.changes?.diff) lines.push(`Since ${computerSource("observation", d.changes.since!)}:`, ...diffLines(d.changes.diff));
    else if (d.changes?.reason) lines.push(d.changes.reason);
    // An aria Read shows the tree as Playwright does; refs are this observation's hints.
    if (d.aria !== undefined) lines.push("Playwright aria snapshot; [ref=…] values are this observation's hints. Act with a target such as getByRole('button', { name: 'Save' }) on the window address, or a hint on this observation.", d.aria);
    else if (!d.changes?.diff) for (const e of d.ui.elements) lines.push(`${e.hint} | ${label(e.role)} | ${label(e.name)} | ${label(e.value)} | enabled=${e.enabled} | image_bounds=${JSON.stringify(e.bounds.window_image ?? null)}`);
  } else if (d.status === "listed") {
    if (state.remote_session === true) lines.push(`These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}`);
    for (const kind of ["windows", "apps", "displays"] as const) {
      const rows = state[kind];
      if (!Array.isArray(rows)) continue;
      for (const raw of rows) {
        const row = record(raw);
        const ref = kind === "windows" ? row.window_ref : kind === "displays" ? row.display_ref : undefined;
        const source = typeof ref !== "string" ? undefined : kind === "windows" ? computerSource("window", ref) : displaySource(d.machine ?? LOCAL_MACHINE, ref);
        const shown = source !== undefined ? source
          : kind === "apps" && typeof row.app_ref === "string" ? computerSource("app", row.app_ref) : "(no controllable reference)";
        const metadata = Object.fromEntries(["pid", "window_id", "bounds", "minimized", "owned", "width", "height", "primary"]
          .filter(key => row[key] !== undefined).map(key => [key, row[key]]));
        lines.push(`${shown} | ${label(row.name ?? row.title ?? row.app)} | ${JSON.stringify(metadata)}`);
      }
    }
  } else {
    // ComputerSession puts its complete dispatch summary before the native JSON projection.
    const message = computerResultText(result).split("\n{", 1)[0];
    if (message) lines.push(message);
    for (const key of ["code", "native_message", "delivery_mode", "path", "effect"])
      if (typeof state[key] === "string") lines.push(`${key}: ${state[key]}`);
    if (state.unit === "wheel_detents") {
      lines.push(`Scroll: delta_x=${state.delta_x}, delta_y=${state.delta_y}; unit=wheel_detents`,
        `Native prefix: pointer_attempted=${state.pointer_attempted}, pointer_delivered=${state.pointer_delivered}; attempted_axes=${JSON.stringify(state.attempted_axes)}, delivered_axes=${JSON.stringify(state.delivered_axes)}`);
    }
  }
  if (d.sequence) {
    const sequence = d.sequence;
    lines.push(`Sequence: ${computerSource("sequence", sequence.sequence_id)}`,
      // A running watch has no stop reason or slots yet; a watch the agent stopped left no slots pending.
      "running" in sequence
        ? `Frames so far: ${sequence.frames.length}; running; elapsed_ms=${Math.round(sequence.elapsed_ms)}${sequence.actual_fps === null ? "" : `; actual_fps=${sequence.actual_fps}`}`
        : `Frames: ${sequence.frames.length}; stop=${sequence.stop_reason}; elapsed_ms=${Math.round(sequence.elapsed_ms)}; actual_fps=${sequence.actual_fps}; last_frame_age_ms=${sequence.last_frame_age_ms === null ? null : Math.round(sequence.last_frame_age_ms)}; skipped_slots=${sequence.skipped_slots.length}${sequence.stop_reason === "stopped" ? "" : `; pending_slots=${sequence.pending_slots}`}`);
  }
  if (d.watch) lines.push(d.watch.running
    ? `Watch: ${d.watch.watching} runs in the background as ${computerSource("sequence", d.watch.sequence_id)}; computer_record watch_stop at that address stops it`
    : `Watch: ${d.watch.watching} stopped; ${d.watch.changed_frames ?? 0} frames changed${d.watch.files ? `; files in ${d.watch.files.dir}` : ""}`);
  for (const span of d.watch_spans ?? [])
    lines.push(`Watch: ${span.label} ran ${Math.round(span.from_ms)}–${Math.round(span.to_ms)} ms into ${computerSource("sequence", span.sequence_id)}; ${span.frames ? `frames ${span.frames.from}–${span.frames.to}` : "no frame captured during it yet"}`);
  if (d.process) lines.push(d.process.responding
    ? `Process: ${d.process.name} (pid ${d.process.pid}) responds; ${d.process.memory_mb} MB memory, ${d.process.cpu_seconds} s CPU`
    : `Process: ${d.process.name} (pid ${d.process.pid}) is NOT responding; ${d.process.memory_mb} MB memory. Capture and input cannot recover a hung app: wait and read again, or close it.`);
  if (d.visual_regions) {
    lines.push(`Regions: ${d.visual_regions.length} icons and text areas found by Cua perception; boxes in this observation's overview pixels. Click a box center with pixel input on this observation. Detection can miss, split or merge items, and labels can be wrong.`);
    for (const r of d.visual_regions) lines.push(`${r.kind}${r.text || r.label ? ` "${label(r.text ?? r.label)}"` : ""} | x=${r.x} y=${r.y} w=${r.width} h=${r.height} | center ${Math.round(r.x + r.width / 2)},${Math.round(r.y + r.height / 2)} | confidence ${r.confidence}`);
  }
  if (d.text_lines) {
    lines.push(`OCR: ${d.text_lines.length} lines; boxes in ${d.viewport ? "pixels of the viewport's image" : "this observation's overview pixels"}. Recognised text can be wrong; confirm with pixels or UI before acting on it.`);
    for (const line of d.text_lines) lines.push(`"${label(line.text)}" | x=${line.x} y=${line.y} w=${line.width} h=${line.height}`);
  }
  if (d.successor) {
    const s = d.successor, diff = s.diff;
    lines.push(s.observation_id ? `Successor: ${computerSource("observation", s.observation_id)}` : `Successor: none. ${s.reason ?? ""}`);
    if (diff) lines.push(...diffLines(diff, true));
    // Without a successor, the reason already follows "Successor: none".
    else if (s.reason && s.observation_id) lines.push(s.reason);
  }
  if (d.selected_frames) lines.push(`Selected frames: ${d.selected_frames.map(frame => frame.index).join(", ")}; readonly, input=none`);
  lines.push(d.successor?.observation_id ? "Refs in this change list are current for the next input; Read the window for its full state."
    : d.verification ? (d.verification.verdict === "satisfied" ? "Conditions held; refs of this sample are current for input." : "Not verified: neither proof of success nor of failure beyond the listed verdicts. Never replay uncertain input blindly.")
    : d.dom ? "Act on the window with the computer_* input tools, a ref such as e2 as target. Read element:crop|highlight,<ref> or image for page pixels. Refs belong to the latest snapshot."
    : d.window_state_id ? "This Read only looks; computer_window changes the window's state."
    : d.watch && d.status === "observed" ? "A watch only looks: the window's refs and points stay as they were for input."
    : d.viewport && d.status === "observed" ? "Act in this viewport with the computer_* input tools at its address; a viewport Read gives no refs."
    : (d.selected_frames || d.sequence) && d.status === "observed" ? "Frames only look: a sequence grants no input. Read the window for refs and points."
    // A window overview without accessibility has points but no refs.
    : d.granularity && !d.ui?.elements.length && d.captured?.image && d.status === "observed" ? "Points of this image are current for the next input to its window; it has no refs."
    : d.status === "observed" ? "Refs of this Read are current for the next input to its window; detail and display Reads give none."
    : "Native delivery is not verified application state. Never replay unknown input; Read a fresh window overview/state before another action.");
  const text = lines.join("\n");
  if (full || !d.dom || Buffer.byteLength(text) <= 24_000) return text;
  // Read and Search share the same byte budget; retain whole lines and never split Unicode.
  const preview = fittingLines(lines, 24_000 - 300);
  // A Read of the observation renders slightly different header lines, so page it there rather than by these numbers.
  return `${preview.join("\n")}\nDOM text preview reached its byte limit after ${preview.length} of ${lines.length} lines. Read this observation with offset and limit to page through it (each page names the next offset), or Search it; omitted lines are not absence.`;
}


/** Lines of a compact UI, DOM or pixel change list; changed and added rows carry the new observation's hints. */
function diffLines(diff: StateDiff, afterAction = false): string[] {
  const page = diff.page ?? {};
  const total = diff.counts.changed + diff.counts.added + diff.counts.removed;
  const omitted = total - diff.changed.length - diff.added.length - diff.removed.length;
  const pixels = diff.pixels;
  const subject = diff.kind === "dom" ? "on the page" : diff.kind === "pixels" ? "in the window's pixels" : pixels ? "in the UI or the window's pixels" : "in the UI";
  const nothing = !afterAction ? `Nothing changed ${subject} since that observation.`
    : `Nothing changed ${subject}${diff.waited_ms !== undefined ? ` within ${diff.waited_ms} ms` : ""} after the action. Check that the target was right before acting again; do not replay blindly.`;
  const uiChanged = total > 0 || Object.keys(page).length > 0;
  return [...Object.entries(page).map(([key, change]) => `Changed ${key}: ${JSON.stringify(change![0])} → ${JSON.stringify(change![1])}`),
    ...(diffIsEmpty(diff) ? [nothing] : diff.kind === "pixels" ? [] : [`Changes: ${diff.counts.changed} changed, ${diff.counts.added} added, ${diff.counts.removed} removed`]),
    ...diff.changed.map(line => `~ ${label(line)}`), ...diff.added.map(line => `+ ${label(line)}`), ...diff.removed.map(line => `- ${label(line)}`),
    ...(omitted > 0 ? [`${omitted} more changes are not listed; Search or Read the new observation for them.`] : []),
    ...(pixels && !diffIsEmpty(diff) ? pixelLines(pixels, uiChanged) : []),
    ...(afterAction && !pixels && diff.kind === "ui" ? ["Pixels were not compared: the input observation had no pixels."] : []),
    ...(diff.unsettled ? [`The window was still changing ${diff.waited_ms} ms after the action; the successor shows it at that moment.`] : []),
    ...(diff.note ? [diff.note] : [])];
}

/** How the window's pixels changed, in overview pixels, and what the attached crops show. */
function pixelLines(pixels: PixelReport, uiChanged: boolean): string[] {
  const since = pixels.since ? ` since ${computerSource("observation", pixels.since)}` : "";
  if (pixels.size_changed) return [`The window's overview size changed${since} from ${pixels.size_changed.width}x${pixels.size_changed.height} to ${pixels.width}x${pixels.height}; pixels were not compared.`];
  if (!pixels.box) return uiChanged ? [`No visible pixel change${since}.`] : [];
  const share = pixels.share * 100, percent = share < 0.1 ? share.toFixed(2) : share.toFixed(1), crops = pixels.crops ?? [];
  const rect = (b: PixelBox) => `x=${b.x} y=${b.y} w=${b.width} h=${b.height}`;
  const chrome = pixels.chrome ? [`Browser chrome outside the page also changed (${rect(pixels.chrome.box)}, ${pixels.chrome.regions} ${pixels.chrome.regions === 1 ? "area" : "areas"}): tab strip, address bar or loading state; not listed.`] : [];
  const page = pixels.space === "page", viewport = pixels.space === "viewport";
  const whole = page ? "page" : viewport ? "viewport" : "window", unit = page ? "CSS pixels of the page" : viewport ? "pixels of the viewport's image" : "overview pixels";
  if (pixels.full) return [pixels.navigated ? `The page navigated (${percent}% of its pixels changed): image 1 shows the whole page after the action.`
    : `Most of the ${whole} changed${since} (${percent}% of its pixels): image 1 shows the whole ${whole} after the action.`, ...chrome];
  if (!pixels.areas.length) return chrome;
  // The overview, when attached, is image 1 and numbers the areas as listed; each crop pair follows it.
  const first = pixels.overview ? 2 : 1;
  const lines = [`Pixels changed${since} in ${pixels.regions} ${pixels.regions === 1 ? "area" : "areas"} (${unit}; ${percent}% of the ${whole})${pixels.overview ? `; image 1 outlines and numbers them on the ${whole}` : ""}:`,
    ...pixels.areas.map((area, i) => {
      const crop = crops[i], zoom = crop && crop.scale > 1 ? `, enlarged ${crop.scale}x` : "";
      return `  ${i + 1}. ${rect(area)}${area.at_input ? " (at the input point)" : ""}${crop ? `; images ${first + 2 * i} and ${first + 2 * i + 1} show ${rect(crop)} before and after${zoom}` : ""}`;
    })];
  if (pixels.regions > pixels.areas.length) lines.push(`  …and ${pixels.regions - pixels.areas.length} more areas, not listed; Read the window for all of it.`);
  return [...lines, ...chrome];
}

/** Whole lines from the start that fit a byte budget, never splitting Unicode. */
function fittingLines(lines: readonly string[], budget: number): string[] {
  const kept: string[] = [];
  let bytes = 0;
  for (const line of lines) {
    const size = Buffer.byteLength(line) + 1;
    if (bytes + size > budget) break;
    kept.push(line); bytes += size;
  }
  return kept;
}

/** Preserve uncertainty when a later refusal follows an already-delivered action prefix. */
export function computerEffect(result: ComputerResult): "applied" | "not-applied" | "unknown" {
  if (!result.isError && ["delivered", "observed", "closed"].includes(result.details.status)) return "applied";
  const sequence = record(result.details.state?.sequence);
  const delivered = typeof sequence.completed === "number" && sequence.completed > 0;
  return result.details.status === "refused" && !delivered ? "not-applied" : "unknown";
}

/**
 * The value a Codemode script receives from a computer_* tool. Every outcome resolves with this
 * shape, so scripts branch on `outcome` instead of catching errors. Delivery never verifies application state.
 */
export interface ComputerReceipt {
  /** `refused` sent nothing; `unknown` may have reached the target, so never replay it blindly. */
  outcome: "delivered" | "refused" | "unknown" | "observed" | "closed";
  /** Input effect: `not-applied` only when nothing was delivered, including no earlier sequence step. */
  effect: "applied" | "not-applied" | "unknown";
  /** `validation`: rejected before reaching the target; `target`: the session, browser or native driver answered. */
  stage: "validation" | "target";
  action: string | null;
  source: string;
  window: string | null;
  /** The viewport a Read showed or an input went to. */
  viewport?: string;
  /** The sandbox address computer_open started. */
  sandbox?: string;
  /** The sequence address of a watch this call started or stopped. */
  watch?: string;
  /** The action's span in each running watch of its window, in that watch's milliseconds, with the frames captured during it. */
  watch_spans?: { sequence: string; from_ms: number; to_ms: number; frames: { from: number; to: number } | null }[];
  /** The new current observation issued by this call, such as a DOM action's next snapshot. */
  observation: string | null;
  /** Whether the addressed observation can no longer be used for input. */
  input_consumed: boolean;
  application_effect: "unverified";
  reason: string | null;
  code: string | null;
  /** Page state after a browser action. */
  page?: { url: string; title: string; tab: string | null; dialog: string | null };
  evaluation?: { value?: string; error?: string; truncated?: boolean };
  /** Verdict of a wait_for/verify command, with one entry per requested condition. */
  verification?: { verdict: "satisfied" | "unsatisfied" | "unknown"; timed_out: boolean; samples: number; elapsed_ms: number; reason?: string;
    conditions: { index: number; verdict: "satisfied" | "unsatisfied" | "unknown"; observed?: string | null; reason?: string }[] };
  /** Per-step results of a native sequence. */
  steps?: { index: number; action: unknown; status: unknown; verdict?: string; reason?: string }[];
  /** Compact change list from the action's input observation to `observation`. */
  changes?: { kind: string; changed: string[]; added: string[]; removed: string[]; counts: { changed: number; added: number; removed: number };
    page?: Record<string, unknown>; note?: string };
  /** The same receipt the model sees. */
  text: string;
}
/** Build the script receipt for one computer action, or for a request rejected before dispatch. */
export function computerReceipt(source: string, action: string | null, outcome: { result: ComputerResult; inputConsumed: boolean; text: string }
  | { error: unknown; inputConsumed: boolean }): ComputerReceipt {
  if ("error" in outcome) {
    const message = outcome.error instanceof Error ? outcome.error.message : String(outcome.error);
    const validation = outcome.error instanceof ComputerRequestError;
    return { outcome: validation ? "refused" : "unknown", effect: validation ? "not-applied" : "unknown", stage: validation ? "validation" : "target",
      action, source, window: null, observation: null, input_consumed: outcome.inputConsumed, application_effect: "unverified",
      reason: message, code: validation ? "COMPUTER_INVALID_REQUEST" : "COMPUTER_UNKNOWN", text: message };
  }
  const d = outcome.result.details, state = d.state ?? {};
  const id = d.observation_id ?? d.agent_image?.image_id;
  const issued = typeof id === "string" && d.status !== "refused" && d.status !== "unknown" && (d.status === "observed" || d.dom !== undefined || d.ui !== undefined);
  const next = d.successor?.observation_id ?? (issued ? id : undefined);
  const sequence = record(state.sequence);
  const failed = d.status === "refused" || d.status === "unknown";
  return {
    outcome: d.status === "listed" ? "observed" : d.status, effect: computerEffect(outcome.result), stage: "target", action, source,
    window: typeof d.window_ref === "string" ? computerSource("window", d.window_ref) : null,
    ...(d.viewport ? { viewport: d.viewport.source } : {}),
    ...(typeof d.sandbox_ref === "string" ? { sandbox: computerSource("sandbox", d.sandbox_ref) } : {}),
    ...(d.watch ? { watch: computerSource("sequence", d.watch.sequence_id) } : {}),
    ...(d.watch_spans ? { watch_spans: d.watch_spans.map(span => ({ sequence: computerSource("sequence", span.sequence_id), from_ms: span.from_ms, to_ms: span.to_ms, frames: span.frames })) } : {}),
    observation: next ? computerSource("observation", next) : null,
    input_consumed: outcome.inputConsumed, application_effect: "unverified",
    reason: failed ? computerResultText(outcome.result).split("\n{", 1)[0] : null,
    code: typeof state.code === "string" ? state.code : null,
    ...(d.dom ? { page: { url: d.dom.url, title: d.dom.title, tab: d.dom.active ?? null, dialog: d.dom.dialog?.type ?? null } } : {}),
    ...(d.dom?.evaluation ? { evaluation: d.dom.evaluation } : {}),
    ...(d.verification ? { verification: { verdict: d.verification.verdict, timed_out: d.verification.timed_out, samples: d.verification.samples,
      elapsed_ms: d.verification.elapsed_ms, ...(d.verification.reason ? { reason: d.verification.reason } : {}), conditions: d.verification.conditions } } : {}),
    ...(Array.isArray(sequence.steps) ? { steps: (sequence.steps as Record<string, unknown>[]).map(step => ({ index: step.index as number, action: step.action, status: step.status })) } : {}),
    ...(d.successor?.diff ? { changes: d.successor.diff } : {}),
    text: outcome.text,
  };
}
/** One computer Read: native blocks for the model, canonical blocks for scripts and the details renderers use. */
export async function readComputer(resources: ComputerResources, source: string, cwd: string,
  request: { readonly views?: readonly string[]; readonly offset?: number; readonly limit?: number }, script: boolean, signal?: AbortSignal) {
  const address = resources.address(source);
  const result = await resources.read(address, cwd, request.views, signal);
  const paged = request.offset !== undefined || request.limit !== undefined;
  // The agent sees this run's short addresses; the long ones stay in structured details.
  let body = resources.shorten(result ? presentComputerResult(result, paged) : resources.descriptor(address));
  const lines = body.split("\n"), offset = request.offset ?? 1;
  const start = offset < 0 ? Math.max(0, lines.length + offset) : Math.max(0, offset - 1);
  if (request.limit !== undefined && (!Number.isSafeInteger(request.limit) || request.limit < 0)) throw new ComputerRequestError("limit must be a nonnegative text line count.");
  if (!Number.isSafeInteger(offset)) throw new ComputerRequestError("offset must be an integer text line offset.");
  const end = request.limit === undefined ? lines.length : Math.min(lines.length, start + request.limit);
  body = lines.slice(start, end).join("\n");
  // Paging reads cached text only; the footer gives a deterministic next offset or end.
  if (paged)
    body += `\n[Lines ${start + 1}-${end} of ${lines.length}. ${end < lines.length ? `Continue with offset=${end + 1}.` : "End of this observation's text."} The text belongs to this observation; Read the window again for a fresh one.]`;
  const uiOnly = !!result?.details.dom && !result.details.agent_image || (request.views?.includes("ui") ?? false);
  const canonical = nativeBlocks(body, result, uiOnly);
  const truncated = !script && Buffer.byteLength(body) > 24_000;
  const kept = truncated ? fittingLines(body.split("\n"), 24_000 - 300) : [];
  const preview = truncated ? `${kept.join("\n")}\n[Presentation truncated after ${kept.length} lines. Read again with offset=${start + kept.length + 1} to continue, or narrow with Search.]` : body;
  return { content: nativeBlocks(preview, result, uiOnly), canonical, isError: result?.isError,
    details: { source: address.source, resolvedBy: "computer", presentationTruncated: truncated, ...(result ? { computer: result.details } : {}) } };
}

/** Write or Insert on a computer address: a refused receipt that names the computer_* tool to use instead. */
export function refuseComputerEdit(resources: ComputerResources, tool: "write" | "insert", source: string, input: Record<string, unknown>) {
  return computerAction(resources, source, null, () => { throw new ComputerRequestError(computerEditRefusal(tool, input)); }, () => Promise.reject(new Error("unreachable")));
}
/**
 * Run one computer operation and present its outcome as an action result: the shortened summary, any images, and the
 * receipt that scripts branch on. `resolve` validates the request; its refusal is a not-applied receipt.
 */
export async function computerAction(resources: ComputerResources, source: string, action: string | null,
  resolve: () => ComputerAddress, run: (address: ComputerAddress) => Promise<ComputerResult>) {
  let address: ComputerAddress | undefined;
  try {
    address = resolve();
    const result = await run(address);
    const effect = computerEffect(result);
    // Completion of the native request is separate from application-effect certainty.
    const ok = !result.isError && ["delivered", "observed", "closed"].includes(result.details.status);
    const summary = resources.shorten(presentComputerResult(result));
    // Images of an action result, such as before/after crops of its pixel change, follow the summary for the model.
    const images = result.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
    return { source, summary, ...(images.length ? { images } : {}), data: { kind: "computer-action", ok,
      effect, ...result.details,
      receipt: computerResultText(result),
      ...(!ok ? { error: { code: typeof result.details.state?.code === "string" ? result.details.state.code : `COMPUTER_${effect.toUpperCase().replace("-", "_")}`,
        message: computerResultText(result) } } : {}) },
      script: { ...shortReceipt(resources, computerReceipt(source, action, { result, inputConsumed: !resources.grantsAuthority(address), text: summary })) } };
  } catch (error) {
    const message = resources.shorten(error instanceof Error ? error.message : String(error));
    return { source, summary: message, data: { kind: "computer-action", ok: false,
      effect: error instanceof ComputerRequestError ? "not-applied" : "unknown", error: { message } },
      script: { ...shortReceipt(resources, computerReceipt(source, action, { error, inputConsumed: address ? !resources.grantsAuthority(address) : false })) } };
  }
}
/** Close what a computer address names through Delete. */
export function deleteComputer(resources: ComputerResources, source: string, input: Record<string, unknown>, cwd: string, signal?: AbortSignal) {
  return computerAction(resources, source, "close", () => {
    if (Object.keys(input).some(key => key !== "path")) throw new ComputerRequestError("Delete of a computer address takes only its path; it closes the whole resource.");
    return resources.address(source.endsWith(FORCE) ? source.slice(0, -FORCE.length) : source);
  }, () => resources.delete(source, cwd, signal));
}
/** A receipt whose window, sandbox and observation addresses are this run's short aliases. */
function shortReceipt(resources: ComputerResources, receipt: ComputerReceipt): ComputerReceipt {
  const short = (value: string | null | undefined) => value ? resources.shorten(value) : value;
  return { ...receipt, window: short(receipt.window) ?? null, observation: short(receipt.observation) ?? null,
    ...(receipt.sandbox ? { sandbox: short(receipt.sandbox)! } : {}), ...(receipt.viewport ? { viewport: short(receipt.viewport)! } : {}), ...(receipt.reason ? { reason: resources.shorten(receipt.reason) } : {}), text: resources.shorten(receipt.text) };
}
function nativeBlocks(body: string, result: ComputerResult | undefined, uiOnly: boolean): Content[] {
  return [{ type: "text", text: body }, ...(uiOnly || !result ? [] : result.details.selected_frames
    ? result.content.slice(1) : result.content.filter(block => block.type === "image"))];
}
/**
 * Extend ordinary IDE tools; CUA and ComputerSession remain the sole native backend and policy owner.
 * `attached()` reports whether a loaded IDE accepted all three plugins. An IDE that is absent or refuses
 * this plugin API version leaves it false, so the caller can offer the standalone tools instead.
 */
export async function registerComputerProvider(pi: ExtensionAPI, getSession: (cwd: string) => Promise<ComputerSession>,
  closeSession: () => Promise<ComputerResult>, openSandbox?: SandboxFactory, connectMachine?: MachineFactory, getMode?: () => ComputerMode,
): Promise<{ readonly resources: ComputerResources; readonly guide: string; readonly attached: () => boolean;
  readonly registerCapabilities: (tools: readonly string[], namespace: { name: string; description: string }) => void }> {
  const accepted = new Set<string>();
  const guide = await readFile(new URL("../docs/computer-resources.md", import.meta.url), "utf8");
  connectAgentDocumentation(pi, [{ id: "computer", description: "Windows app windows, displays and browser pages: look with read and search, act with the computer_* tools",
    markdown: fillToolNames(guide, IDE_TOOL_NAMES),
    triggers: ["read", "search"].map(tool => ({ tool, resourcePrefixes: ["computer:"] })) }]);
  const resources = new ComputerResources(getSession, closeSession, openSandbox, connectMachine, getMode);
  const describe = (source: string) => resources.describe(source);
  // Pi shutdown releases every machine, sandbox and session this computer use started.
  pi.on("session_shutdown", () => resources.closeAll().then(() => undefined));
  const resolver: ResourceResolver = {
    id: "computer",
    async tryResolve(source) {
      if (!owns(source)) return { kind: "not-handled" };
      try {
        // Delete's force option is not part of the address; Delete itself checks where it applies.
        const address = resources.address(source.endsWith(FORCE) ? source.slice(0, -FORCE.length) : source);
        return { kind: "resolved", resource: { source, skipPostEdit: true,
          // An expired reference still resolves, so Delete reaches its handler and returns a refused receipt.
          read: async () => {
            try { return [{ type: "text", text: resources.shorten(resources.descriptor(address)) }]; }
            catch (error) { if (error instanceof ComputerRequestError) return [{ type: "text", text: resources.shorten(`# Computer resource\nSource: ${source}\n${error.message}`) }]; throw error; }
          } } };
      } catch (error) { return { kind: "failed", error }; }
    },
  };
  // An IDE that refuses this plugin API version rejects here; computer use then stays standalone.
  const optional = (registration: void | Promise<void>) => Promise.resolve(registration).catch(() => undefined);
  await Promise.all([
    optional(connectReadPlugin(pi, { protocol: READ_PROTOCOL, apiVersion: READ_API_VERSION, id: "computer", setup(api) {
      accepted.add("read");
      api.addResolver({ resolver, priority: -250, renderResult: computerReadRenderer(describe) as never });
      for (const view of ["image", "ui", "full", "state", "dom", "region", "sequence", "frames", "capture", "element", "ocr", "changes", "regions", "aria"])
        api.addView({ view, contentKind: "any", presenter: { id: `computer-${view}`, present: document => document } });
      api.addHandler({ stage: "read", when: { resolvedBy: resolver.id, contentKind: "any" }, async handler(context) {
        const { request, resolverContext } = context;
        const read = await readComputer(resources, context.state!.source, resolverContext.cwd, request, context.audience === "script", resolverContext.signal);
        return { kind: "return", result: { content: read.content, isError: read.isError, details: read.details,
          script: { kind: "native", source: read.details.source, blocks: read.canonical } } };
      } });
    } })),
    optional(connectSearchPlugin(pi, { protocol: SEARCH_PROTOCOL, apiVersion: SEARCH_API_VERSION, id: "computer", setup(api) {
      accepted.add("search");
      api.addResolver({ priority: -250, resolver: { id: "computer", renderResult: computerSearchRenderer(describe) as never, readResources: request => owns(request.path) ? [request.path] : [],
        async tryResolve(request, context) {
          if (!owns(request.path)) return { kind: "not-handled" };
          try {
            if (request.include !== undefined || request.exclude !== undefined || ("navigation" in request && request.navigation !== undefined) || request.caseSensitive || request.wholeWord)
              throw new ComputerRequestError("Computer Search uses native discovery/UI text predicates, not file globs or symbol navigation.");
            const result = await resources.search(resources.address(request.path), context.cwd, request.query, request.limit, context.signal);
            return { kind: "resolved", payload: result };
          } catch (error) { return { kind: "failed", error }; }
        },
        format(payload) { const result = payload as ComputerResult; return { ...result, content: [{ type: "text", text: resources.shorten(presentComputerResult(result)) }] }; },
        toScriptData(payload) { return { kind: "custom", resolverId: "computer", value: (payload as ComputerResult).details }; },
      } });
    } })),
    optional(connectTextEditorPlugin(pi, { protocol: TEXT_EDITOR_PROTOCOL, apiVersion: TEXT_EDITOR_API_VERSION, id: "computer", setup(api) {
      accepted.add("text-editor");
      api.addResolver({ resolver, priority: -250 });
      // Compose over IDE's default renderers after every plugin has finished loading.
      let renderersRegistered = false;
      pi.on("session_start", () => {
        if (renderersRegistered) return;
        for (const tool of ["write", "insert", "delete"] as const) api.addToolRenderer(computerActionRenderer(tool, describe) as never);
        renderersRegistered = true;
      });
      // Write and Insert do not act on computer addresses; they refuse instead of treating the address as a file.
      for (const tool of ["write", "insert"] as const) api.tool(tool).addSemanticHandler({
        matches: input => owns(record(input).path),
        execute: (context, input) => refuseComputerEdit(resources, tool, context.sourceFor("path"), record(input)),
      });
      const deleteTool = api.tool("delete");
      deleteTool.addSemanticHandler({
        matches: input => owns(record(input).path),
        execute: (context, input) => deleteComputer(resources, context.sourceFor("path"), record(input), context.cwd, context.signal),
      });
    } })),
  ]);
  return { resources, guide, attached: () => accepted.size === 3,
    registerCapabilities(tools, namespace) {
      if (accepted.size === 0) return;
      registerIdeCapability(pi, {
        version: 1, id: "computer", tools,
        discovery: { name: "ide_capability_computer", namespace,
          description: "Use ide_capability_computer to load Computer Use resource syntax for Read, Search and Delete, including installed-app, existing-window and display discovery." },
        schemas: {
          ...(accepted.has("read") ? { read: { parameters: { path: computerPathHelp(IDE_TOOL_NAMES), views: computerViewsHelp(IDE_TOOL_NAMES) } } } : {}),
          ...(accepted.has("search") ? { search: { promptGuidelines: [COMPUTER_SEARCH_HELP] } } : {}),
          ...(accepted.has("text-editor") ? { delete: { parameters: { path: COMPUTER_DELETE_HELP } } } : {}),
        },
      });
    },
  };
}