Skip to content

Computer use for the Pi coding agent — Windows apps, browsers and desktops through native UI trees, screenshots and real input.

190 KB
Raw Blame
import { randomUUID, createHash } from "node:crypto";
import { nativeDisplay, selectedDisplaySource, type NativeDisplay } from "./selected-displays.ts";
import { collectSequence, cropSequenceFrame, frameSummary, runningSequenceMetadata, selectFrames, sequenceMetadata, SequenceStop, SequenceStore, validateSequence, type FrameSequence, type SequenceOptions, type FrameSelection, type SequenceFrame } from "./temporal-capture.ts";
import { changeLines, differs, frameChanges, framesBetween, sheetFrames, WATCH_LIMITS, WATCH_SHEET_FRAMES, writeWatchFiles, type WatchFiles, type WatchSpan } from "./watch.ts";
import type { PixelBox } from "./pixel-diff.ts";
import { rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { renderFrames, type FrameMark, type FrameRendering, type RenderedFrames } from "./contact-sheet.ts";
import type { GuideMark } from "./image-guides.ts";
import { createUiResource, projectUi, resolveUi, annotateUi, validateUiCapture, type UiResource, type UiRequest, type UiCaptureOptions } from "./ui-resources.ts";
import { isSemanticAction, semanticReceipt } from "./semantic-actions.ts";
import { isPointerMove, pointerReceipt } from "./pointer-actions.ts";
import { isCoordinateScroll, scrollReceipt } from "./scroll-actions.ts";
import { validWindowStatus, windowRestoreStatus } from "./window-restore.ts";
import { validWindowControlState, windowManagementStatus, type WindowManagementCommand } from "./window-management.ts";
import { nativeInput, requestedInputTime, validateInput, type InputCommand, type InputStep } from "./complete-input.ts";
import { notifyWindowObserver, notifyHighlight, shownTypedText, type WindowObserver, type HighlightResult, type HighlightMark } from "./native-visuals.ts";
import { screenRect, elementRegion, windowImageMapping, windowRegion, nativeClickPoint, type ScreenRect } from "./highlight-geometry.ts";
import { observationSource, sourceIdentity, type ObservationSource, type SourceIdentity, type NativePlatform } from "./observation-source.ts";
import { actionCapability, type ActionCapability } from "./action-capability.ts";
import { DEFAULT_MAX_LONG_EDGE, FULL_RESOLUTION, nativeImageCap, type CuaVisionSettings } from "./cua-image-settings.ts";
import { REMOTE_DESKTOP_DISPLAY, type DisplayMode } from "./display-mode.ts";
import { WEBVIEW2_DEBUGGING, type MachineHost, type ProcessHealth, type TextLine } from "./machine-host.ts";
import { setTimeout as delay } from "node:timers/promises";
import { cropDisplay, nativeRegion, validateDetail, type DetailContext, type DetailRequest } from "./native-detail.ts";
import type { StateDiff } from "./successor.ts";
import { combine, evaluateNative, validateWait, type ConditionResult, type WaitResult, type WaitSpec } from "./ui-conditions.ts";
import { computerSource } from "./computer-address.ts";
import { WebError, type WebAction, type WebBrowser, type WebLauncher, type WebLaunchOptions, type WebOpened, type WebPage, type WebSnapshotOptions, type WebTarget } from "./web-browser.ts";
import { ATTACH_APPROVAL_MS } from "./browser-command-line.ts";
import { derivedPng } from "./web-capture.ts";
import type { Derive } from "./web-geometry.ts";

/** Native Windows address. Models use session window references instead. */
export interface WindowTarget { pid: number; window_id: number }
/** Safe limits UI access to individually opened session windows; neither mode is an OS sandbox. */
export type ComputerMode = "safe" | "yolo";
/** Model-visible native text or screenshot. */
export type Content = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string };
/** Native MCP result; structured state is separate from images. */
export interface DriverResult {
  content: Content[];
  structuredContent?: Record<string, unknown>;
  isError?: boolean;
}
/** Owned backend with window-lifetime checks. Calls are serialized by ComputerSession. */
export interface Driver {
  /** Native execution platform. Omitted by older in-process fixtures, which model Windows. */
  platform?: NativePlatform;
  /** False where native lifetime binding is not implemented; never substitute reusable IDs. */
  windowLifetime?: boolean;
  /** Source domain override: a sandbox guest's windows are not host windows. */
  sourceDomain?: "windows-sandbox" | "windows-session" | "windows-remote";
  /** True only for a backend qualified for held paths and graceful in-flight release. */
  heldPaths?: boolean;
  /** True only for a qualified native runtime with bounded keyboard ownership. */
  heldKeys?: boolean;
  /** True only for a qualified Windows runtime with exact selected-display read contracts. */
  selectedDisplays?: boolean;
  /** True only for a qualified native runtime that admits both straight-drag endpoints against one capture. */
  captureBoundDrag?: boolean;
  /** True only for an exact qualified Windows runtime with UIA-only semantic dispatch. */
  semanticActions?: boolean;
  /** True only for the explicitly selected, byte-checked Windows native pointer runtime. */
  nativePointer?: boolean;
  /** True only for an explicitly selected, byte-checked Windows capture-bound wheel runtime. */
  coordinateScroll?: boolean;
  /** True only for a byte-qualified Windows state admission and no-input restore runtime. */
  windowRestore?: boolean;
  /** True only for an exact byte-qualified Windows state-bound management runtime. */
  windowManagement?: boolean;
  /** Fixed overview filter of the selected native runtime; never a tool-controlled resize switch. */
  overviewKernel?: CuaVisionSettings["kernel"];
  /** A display-only window read can return one rectangle of the window, encoding only that rectangle. */
  windowCrop?: boolean;
  /** Human-only native presentation, owned by this driver and never an input route. */
  visuals?: WindowObserver;
  call(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<DriverResult>;
  identity(target: WindowTarget, create: boolean, signal?: AbortSignal): Promise<string | undefined>;
  /**
   * A second driver process on the same desktop for captures that run while this one acts: a driver answers one call
   * at a time, so a capture sent during a long drag would wait for its end. Absent where only one connection exists.
   */
  companion?(): Driver | undefined;
  close(): Promise<void>;
}
/** One complete action or bounded known-target sequence, grounded in one window overview. */
export type Action = {
  window_ref: string;
  observation_id: string;
  delivery_mode?: "background" | "foreground";
  /** A click or scroll on an element token goes through UI Automation only; the driver refuses when no pattern carries it. */
  uia_only?: boolean;
} & (InputCommand | { action: "sequence"; sequence: InputStep[] });
/** Presentation uses only a category identifier, never action payload or execution-route names. */
function presentationAction(command: InputCommand | WebAction): string {
  if (command.action === "key" || command.action === "press") return "press_key";
  if (command.action === "set_value") return "fill";
  if (command.action === "select_all") return "hotkey";
  if (command.action === "hold") return command.actions?.length ? "input_timeline" : "hotkey";
  if (command.action === "timeline") return command.events?.every(event => event.type === "key_down" || event.type === "key_up") ? "hotkey" : "input_timeline";
  return command.action;
}
/** Display only bounded input metadata; printable keys can be text and are hidden unless the focus is known safe. */
function presentationDetail(command: InputCommand | WebAction, safeText = false): string | undefined {
  const chord = (keys: string[]) => {
    const last = keys.at(-1);
    if (!last || !/^(?:[a-z0-9\-=[\];',./\\`]|enter|return|tab|escape|esc|space|backspace|delete|insert|up|down|left|right|arrowup|arrowdown|arrowleft|arrowright|home|end|pageup|pagedown|f(?:[1-9]|1[0-9]|2[0-4])|ctrl|control|shift|alt|win|meta)$/i.test(last)) return undefined;
    if (keys.slice(0, -1).some(key => !/^(?:ctrl|control|shift|alt|win|meta)$/i.test(key))) return undefined;
    const modifiers = keys.slice(0, -1);
    // Ctrl+Alt can be AltGr, which enters printable text on many keyboard layouts.
    const nonTextModifier = modifiers.some(key => /^(?:ctrl|control|win|meta)$/i.test(key)) && !modifiers.some(key => /^alt$/i.test(key));
    return safeText || last.length > 1 || nonTextModifier ? keys.join("+") : undefined;
  };
  if (command.action === "press") return command.key ? shownTypedText(chord(command.key.split("+"))) : undefined;
  if (command.action === "key") return command.key ? shownTypedText(chord([...(command.modifiers ?? []), command.key])) : undefined;
  if (command.action === "hotkey" || command.action === "hold") {
    const keys = command.keys?.length ? chord(command.keys) : undefined;
    const actions = command.action === "hold" ? command.actions?.map(action => action.action).join(" → ") : undefined;
    return shownTypedText([keys, actions].filter(Boolean).join(" + "));
  }
  if (command.action === "select_all") return "Ctrl+A";
  if (command.action === "timeline" && command.events?.length) {
    const details = command.events.slice(0, 20).map(event => event.key ? (chord([event.key]) ? `${event.type.replaceAll("_", " ")} ${chord([event.key])}` : undefined) : event.type.replaceAll("_", " "));
    return shownTypedText(details.filter(Boolean).join(" → "));
  }
  if (command.action === "scroll" && "direction" in command) return command.direction ? shownTypedText(command.direction) : undefined;
  if (command.action === "click") {
    const count = "count" in command ? command.count : "click_count" in command ? command.click_count : undefined;
    return shownTypedText(`${command.button ?? "left"} click${count && count > 1 ? ` ×${count}` : ""}`);
  }
  if (command.action === "double_click" || command.action === "right_click") return command.action.replaceAll("_", " ");
  if (command.action === "drag") return shownTypedText(`${command.button ?? "left"} drag`);
  return undefined;
}
/** A stored Playwright snapshot of the active tab, with the tab list, any held dialog and events since the previous snapshot. */
export interface WebDom extends WebPage { truncated: boolean; matches?: number }
/** Delivery is not a task postcondition. Unknown input is never automatically replayed. */
export interface ComputerResult {
  content: Content[];
  details: {
    status: "listed" | "observed" | "delivered" | "refused" | "unknown" | "closed";
    /** The machine the request ran on, by its address name; set by the computer resources for every routed request. */
    machine?: string;
    /** The sandbox or isolated session on that machine the request ran in. */
    sandbox?: string;
    /** The sandbox an open_sandbox command started. */
    sandbox_ref?: string;
    mode: ComputerMode;
    target?: WindowTarget;
    window_ref?: string;
    /** The viewport a Read showed or an input acted in: rect in the window's full-resolution pixels, image the size of its image. */
    viewport?: { source: string; name: string; window_ref: string; rect: { x: number; y: number; width: number; height: number }; image: { width: number; height: number } };
    observation_id?: string;
    /** One-use window-state authority for native restore or management, never ordinary input. */
    window_state_id?: string;
    /** Management state also admits restore; restore-only state cannot manage windows. */
    window_state_kind?: "restore" | "management";
    opened?: boolean;
    state?: Record<string, unknown>;
    source?: ObservationSource;
    source_identity?: SourceIdentity;
    action_capability?: ActionCapability;
    agent_image?: Record<string, unknown>;
    highlight?: HighlightResult;
    ui?: UiResource;
    /** Fresh Playwright page structure for an owned browser window. */
    dom?: WebDom;
    /** Fresh window capture parts; omitted for non-overview results. */
    granularity?: "image" | "ui" | "full";
    captured?: { image: boolean; ui: boolean };
    /** A finished sequence, or one still collecting (`running`), such as a watch just started. */
    sequence?: ReturnType<typeof sequenceMetadata> | ReturnType<typeof runningSequenceMetadata>;
    selected_frames?: Omit<SequenceFrame, "image">[];
    /** A watch this result started or stopped: what it watches, whether it still runs, the actions it noted and, once stopped, its files. */
    watch?: { sequence_id: string; watching: string; running: boolean; marks: FrameMark[]; changed_frames?: number; files?: WatchFiles };
    /** An action's span in each running watch of its window, in the watch's milliseconds, with the frames captured during it. */
    watch_spans?: WatchSpan[];
    /** How a sheet or composite Read placed the selected frames in its one image. */
    rendered_frames?: Omit<RenderedFrames, "png">;
    /** Outcome of a bounded wait_for/verify; the conditions as requested, in order. */
    verification?: WaitResult & { spec: WaitSpec };
    /** The observation taken after an action and its compact difference from the action's input observation. */
    successor?: { observation_id?: string; diff?: StateDiff; reason?: string };
    /** A `changes` Read: this UI compared with the window's previous observation, or why there is no comparison. */
    changes?: { since?: string; diff?: StateDiff; reason?: string };
    /** The UI walk as a Playwright aria snapshot (YAML), for an aria Read. */
    aria?: string;
    /** Icons and text areas found by Cua perception, boxes in the observation's overview pixels. */
    visual_regions?: VisualRegion[];
    /** Text recognised by Windows OCR in this result's image, boxes in the observation's overview pixels. */
    text_lines?: TextLine[];
    /** The window's process: whether it responds, its memory and CPU time. */
    process?: ProcessHealth;
    /** Complete text of one native element from its observation. */
    element_text?: { hint: string; role: string | null; name: string | null; text: string; cut: boolean };
  };
  isError?: boolean;
}
/** One icon or text area detected in an observation's pixels; the box is in its overview pixels. */
export interface VisualRegion { kind: "icon" | "text"; x: number; y: number; width: number; height: number; text?: string; label?: string; confidence: number }

interface Observation { texts?: Map<string, string>; id: string; tokens: Set<string>; width?: number; height?: number; captureId?: string; bounds?: ScreenRect; frames: Map<string, ScreenRect>; ui?: UiResource; rawImage?: Content; source?: ObservationSource; settings: CuaVisionSettings; actionMark?: GuideMark; mapping?: DetailContext["windowMapping"];
  /** Kept after pixel-only input: coordinates stay valid until a freshness check finds a changed size or state; UI tokens and the native capture are spent. */
  pixelsOnly?: boolean }
interface WindowState { id: string; nativeId: string; kind: "restore" | "management" }
/** The unannotated pixels of a window's latest capture with pixels, kept as the baseline of its next pixel comparison. */
export interface WindowFrame { observation_id: string; png: Buffer; size: { maxLongEdge: number; scale: number } }
interface WindowEntry { ref: string; target: WindowTarget; identity: string; owned: boolean; pendingApproval?: boolean; attachingApproval?: boolean; observation?: Observation;
  visualBounds?: ScreenRect;
  /** Size of the window's latest image capture: points of typed input are in its pixels. Outlives the observation. */
  seen?: { width: number; height: number };
  windowState?: WindowState; dom?: { id: string; page: WebDom }; web?: WebBrowser; detail?: DetailContext; lastAction?: GuideMark; frame?: WindowFrame }
interface App { name: string; launch_path?: string; bundle_id?: string; kind?: string }
/** A watch of a window: its growing sequence, when its clock started, the actions noted in it and how to stop it. */
interface Watching { window_ref: string; watching: string; sequence?: FrameSequence; started: number; marks: FrameMark[]; stop: AbortController; finished: boolean; done: Promise<FrameSequence> }
/** A discovery query: `regex:<pattern>` is a case-insensitive regular expression, anything else a case-insensitive substring. */
export function textMatcher(query?: string): (text: string) => boolean {
  if (!query) return () => true;
  if (query.startsWith("regex:")) {
    let pattern: RegExp;
    try { pattern = new RegExp(query.slice(6), "i"); }
    catch (error) { throw new Error(`Invalid regular expression: ${error instanceof Error ? error.message : error}`); }
    return text => pattern.test(text);
  }
  const needle = query.toLowerCase();
  return text => text.toLowerCase().includes(needle);
}
interface DisplayEntry { ref: string; topology: string; native: NativeDisplay; detail?: DetailContext }
/** One input sequence: enough for a long drawing or form program in one call, still bounded so a runaway program stops. */
export const SEQUENCE_STEPS = 2000;
/** Requested waits, typing and drag durations of one sequence. */
export const SEQUENCE_REQUESTED_MS = 15 * 60_000;
/** Wall-clock budget of one sequence, including native call overhead. */
export const SEQUENCE_ELAPSED_MS = 30 * 60_000;
/** Aria snapshots above this many characters are cut. Read pages stored snapshots with offset/limit. */
const WEB_SNAPSHOT_LIMIT = 400_000;
/** Longest native element text retained for element:text reads; the walk shows 1024 characters per element. */
const ELEMENT_TEXT_LIMIT = 200_000;
const appName = (name: string) => name.trim().toLowerCase().replace(/^windows\s+/, "").replace(/\.exe$/, "");
/** File name of a Win32 launch path without directory and extension; packaged shell:appsFolder paths have none. */
const executableName = (path: string) => /^shell:/i.test(path) ? "" : appName(path.split(/[\\/]/).pop() ?? "");

/** Watch captures in flight at once. A third measured no faster: part of each driver call is serial. */
const WATCH_CONCURRENCY = 2;
/** A watch frame the driver cropped to its viewport: the image is the viewport, at most `cap` on its long edge. */
function nativeCropFrame(state: Record<string, unknown>, images: SequenceFrame["image"][], requested: PixelBox, cap: number, base: ObservationSource) {
  const applied = state.screenshot_crop as PixelBox | undefined;
  const width = state.screenshot_width, height = state.screenshot_height;
  const originalWidth = state.screenshot_original_width, originalHeight = state.screenshot_original_height;
  if (!images[0] || images[0].data.length > 16 * 1024 * 1024 || !applied || typeof width !== "number" || typeof height !== "number"
    || typeof originalWidth !== "number" || typeof originalHeight !== "number" || Math.max(width, height) > cap)
    throw new SequenceStop("capture_failed");
  const source: ObservationSource = { ...base, semantic_scope: "none", capabilities: { read: true, input: "none", detail: "unsupported" },
    image: { ...base.image, scope: "native_window_zoom", coordinate_space: "detail_image_pixels", dimensions: { width, height },
      original_dimensions: { width: originalWidth, height: originalHeight }, origin: { x: applied.x, y: applied.y, relative_to: "window" },
      crop: { requested, applied, coordinate_space: "window_physical_pixels" } } };
  return { image: images[0], source };
}

interface NamedWindow { pid: number; window_id: number; title: string }
interface CoveringWindow extends NamedWindow { relation: string }
/** The window a click result names as in front after the click closed its own window. */
function foregroundAfter(value: unknown): NamedWindow | undefined {
  const window = value as Partial<NamedWindow> | undefined;
  return window && Number.isSafeInteger(window.pid) && Number.isSafeInteger(window.window_id)
    ? { pid: window.pid!, window_id: window.window_id!, title: typeof window.title === "string" ? window.title : "" } : undefined;
}
/** The window a driver refusal names as covering the press point (code pointer_target_occluded). */
function coveringWindow(message: string): CoveringWindow | undefined {
  const match = /pointer_target_occluded: .*?window_id=(\d+) pid=(\d+) relation=(\w+) title=("(?:[^"\\]|\\.)*")/.exec(message);
  if (!match) return undefined;
  let title = "";
  try { title = String(JSON.parse(match[4])); } catch { /* A malformed title leaves the window untitled. */ }
  return { window_id: Number(match[1]), pid: Number(match[2]), relation: match[3], title };
}

/** Multi-window UI policy. Native work is queued; ownership and observations are not persisted across reloads. */
export class ComputerSession {
  private tail: Promise<unknown> = Promise.resolve();
  private readonly entries = new Map<string, WindowEntry>();

  private readonly sessionId = randomUUID();
  private displayOverview?: DetailContext;
  private readonly displayEntries = new Map<string, DisplayEntry>();
  /** Modes this session replaced, by display device; close puts them back. */
  private readonly replacedModes = new Map<string, DisplayMode>();
  private recording?: { timer: NodeJS.Timeout; started: number };
  private recordings = 0;
  private audioClips = 0;
  private readonly sequences = new SequenceStore();
  /** Watches by sequence id until stopWatch reports them; they capture through `watcher`, a second driver where one exists. */
  private readonly watches = new Map<string, Watching>();
  private watcher?: Driver;
  /** Temporary folders of stopped watches' files, removed when the session closes. */
  private readonly watchDirs = new Set<string>();
  private uiSerial = 0;
  private uiImages: Observation[] = [];
  /**
   * Playwright browsers opened by this session, keyed by `pid:window_id` of the window bound at launch.
   * Popup windows of the same process are ordinary native windows; their pages are reached as tabs of this browser.
   */
  private readonly browsers = new Map<string, WebBrowser>();
  private readonly unapprovedAgentBrowsers = new Set<WebBrowser>();
  private unapprovedAgentLaunches = 0;
  /** Native input already dispatched under the current policy, including long timelines on dialog windows. */
  private nativeInputsInFlight = 0;
  private unconfirmedUnsafeWork = false;
  /** Safe cannot be selected while a YOLO agent profile has an unauthenticated debugging port. */
  hasUnapprovedAgentBrowser(): boolean { return this.unconfirmedUnsafeWork || this.unapprovedAgentLaunches > 0 || this.nativeInputsInFlight > 0 || this.unapprovedAgentBrowsers.size > 0; }

  private get platform(): NativePlatform { return this.driver.platform ?? "windows"; }
  /** Label sources from a sandbox guest so they never pass for host windows or displays. */
  private domained<T extends { domain: SourceIdentity["domain"] }>(value: T): T {
    if (this.driver.sourceDomain) value.domain = this.driver.sourceDomain;
    return value;
  }

  private source(kind: ObservationSource["kind"], native: Record<string, unknown>, identity: ObservationSource["identity"], zoom = false): ObservationSource {
    return this.domained(observationSource(kind, native, identity, zoom, this.platform));
  }
  private geometry(state: Record<string, unknown>, display: boolean): string {
    if (display) return JSON.stringify([state.topology_id, state.display, state.screen_width, state.screen_height, state.scale_factor]);
    const bounds = state.window_bounds as Record<string, unknown> | undefined;
    if (!bounds || ![bounds.x, bounds.y, bounds.width, bounds.height].every(v => typeof v === "number" && Number.isFinite(v)) ||
      (bounds.width as number) <= 0 || (bounds.height as number) <= 0) throw new Error("Window geometry is missing.");
    return JSON.stringify([bounds.x, bounds.y, bounds.width, bounds.height]);
  }

  private overview(state: Record<string, unknown>, id: string, display: boolean, images: Content[]): DetailContext | undefined {
    const width = state.screenshot_width as number; const height = state.screenshot_height as number;
    const originalWidth = (state.screenshot_original_width ?? (display ? state.screen_width : width)) as number;
    const originalHeight = (state.screenshot_original_height ?? (display ? state.screen_height : height)) as number;
    if (![width, height, originalWidth, originalHeight].every(v => Number.isSafeInteger(v) && v > 0)) return;
    const image = images.length === 1 ? images[0] : undefined;
    if (image?.type !== "image" || image.mimeType !== "image/png") return;
    const bytes = Buffer.from(image.data, "base64");
    if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ||
      bytes.readUInt32BE(16) !== width || bytes.readUInt32BE(20) !== height) return;
    try { return { id, width, height, originalWidth, originalHeight, geometry: this.geometry(state, display), ...(!display ? { windowMapping: windowImageMapping(state) } : {}) }; }
    catch { return undefined; }
  }

  constructor(private readonly driver: Driver, private readonly getMode: () => ComputerMode = () => "yolo",
    private readonly settings: () => CuaVisionSettings = () => ({ kernel: "lanczos3", maxLongEdge: DEFAULT_MAX_LONG_EDGE, scale: 1, version: 0, route: "cua_native" }),
    private readonly observer: WindowObserver | undefined = driver.visuals,
    private readonly launchWeb?: WebLauncher,
    /** Display modes, the owned file folder, process health and OCR on this desktop's machine; absent in a sandbox guest. */
    readonly host?: MachineHost) {}

  private overviewSettings(): CuaVisionSettings {
    const settings = this.settings();
    return { ...settings, kernel: this.driver.overviewKernel ?? settings.kernel };
  }
  private queue<T = ComputerResult>(work: () => Promise<T>): Promise<T> {
    const result = this.tail.then(work, work);
    this.tail = result.catch(() => undefined);
    return result;
  }

  private result(status: ComputerResult["details"]["status"], message: string,
    state?: Record<string, unknown>, extra: Partial<ComputerResult["details"]> = {}, images: Content[] = []): ComputerResult {
    const details = { ...extra, status, mode: this.getMode(), ...(state ? { state } : {}) };
    // Codemode gets the full structured resource. Plain tool text is a bounded, valid JSON projection.
    const visibleUi = extra.ui && { ...extra.ui, elements: extra.ui.elements.slice(0, 30).map(e => ({ ...e,
      name: e.name?.slice(0, 128) ?? null, value: e.value?.slice(0, 128) ?? null,
      text_truncated: e.text_truncated || (e.name?.length ?? 0) > 128 || (e.value?.length ?? 0) > 128 })) };
    const projection = visibleUi && { ui_elements: visibleUi.elements.length, full_ui_elements: extra.ui!.elements.length,
      note: "Plain text shows at most 30 elements with 128-character names/values. Use structured ui or local search/scope for the rest." };
    // The aria snapshot follows the JSON as plain YAML instead of an escaped string.
    const display = extra.dom ? { ...details, dom: { ...extra.dom, snapshot: undefined } } : extra.sequence ? { ...details, sequence: { ...extra.sequence, source: extra.sequence.frames[0]?.source,
      frames: extra.sequence.frames.map(frameSummary) }, text_projection: { note: "Frame summaries only; full per-frame source metadata is in the structured sequence resource." } }
      : extra.selected_frames ? { ...details, source: extra.selected_frames[0]?.source, selected_frames: extra.selected_frames.map(frameSummary),
        text_projection: { note: "Frame summaries only; full per-frame source metadata accompanies each structured frame." } }
      : visibleUi ? { ...details, ui: visibleUi, text_projection: projection,
      state: state && Object.fromEntries(Object.entries(state).filter(([key]) => !["elements", "tree_markdown", "_note"].includes(key))),
      agent_image: extra.agent_image && { ...extra.agent_image, annotation: extra.agent_image.annotation && {
        ...(extra.agent_image.annotation as Record<string, unknown>), labels: undefined } } } : details;
    const pretty = JSON.stringify(display, null, 2);
    const json = pretty.length <= 24_000 ? pretty : JSON.stringify(display);
    const visible = json.length <= 24_000 ? json : extra.sequence || extra.selected_frames ? JSON.stringify({ ...details,
      ...(extra.sequence ? { sequence: { ...extra.sequence, source: extra.sequence.frames[0]?.source, frames: extra.sequence.frames.slice(0, 5).map(frameSummary) } } : {}),
      ...(extra.selected_frames ? { selected_frames: extra.selected_frames.slice(0, 5).map(frameSummary) } : {}),
      text_projection: { note: "Plain text shows five frame summaries. Full metadata is available in the structured result.", metadata_omitted: true } })
      : visibleUi ? JSON.stringify({ status, mode: details.mode,
      window_ref: extra.window_ref, observation_id: extra.observation_id,
      agent_image: extra.agent_image && { image_id: extra.agent_image.image_id, output: extra.agent_image.output },
      ui: { ...visibleUi, elements: visibleUi.elements.slice(0, 5) },
      text_projection: { ...projection, ui_elements: Math.min(5, visibleUi.elements.length), metadata_omitted: true } })
      : json.slice(0, 24_000) + "\n[Text truncated; narrow the observation.]";
    const mark = extra.highlight;
    const where = mark?.region ? `; screen region ${mark.region.x},${mark.region.y} ${mark.region.width}x${mark.region.height}`
      : mark?.point ? `; screen point ${mark.point.x},${mark.point.y}` : "";
    const summary = mark ? `\nHuman mark: ${mark.kind ?? "target"} ${mark.status}${where}${mark.reason ? ` (${mark.reason})` : ""}.` : "";
    return { content: [{ type: "text", text: `${message}${summary}\n${visible}${extra.dom ? `\n${extra.dom.snapshot}` : ""}` }, ...images], details,
      ...(["refused", "unknown"].includes(status) ? { isError: true } : {}) };
  }

  private allowed(entry: WindowEntry): boolean {
    return this.getMode() === "yolo" || entry.owned;
  }

  private inputAllowed(entry: WindowEntry): boolean {
    return this.allowed(entry) && !(entry.pendingApproval && this.getMode() === "safe");
  }
  private denied(ref: string): ComputerResult {
    return this.result("refused", "Safe mode allows only windows opened by this session. Only the user can change mode with /computer-mode.",
      undefined, { window_ref: ref });
  }

  private addressed(entry: WindowEntry, nativeDesktop?: unknown): Partial<ComputerResult["details"]> {
    return { target: entry.target, window_ref: entry.ref,
      source_identity: this.domained(sourceIdentity({ session_id: this.sessionId, window_ref: entry.ref, ...entry.target }, null, nativeDesktop, this.platform)) };
  }

  private async failed(error: unknown, dispatched: boolean): Promise<ComputerResult> {
    this.entries.clear(); this.displayEntries.clear(); this.displayOverview = undefined; this.uiImages = []; this.sequences.clear();
    await this.driver.close().catch(() => undefined);
    const reason = error instanceof Error ? error.message : String(error);
    return this.result(dispatched ? "unknown" : "refused", dispatched
      ? `Outcome unknown: the application may have changed. Discover and observe before deciding; do not replay blindly. ${reason}`
      : `Computer use failed before input dispatch. Discover again. ${reason}`);
  }

  private async state(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<Record<string, unknown>> {
    const native = await this.driver.call(name, args, signal);
    if (native.isError) throw new Error(JSON.stringify(native.structuredContent ?? native.content));
    if (native.structuredContent) return native.structuredContent;
    for (const block of native.content) {
      if (block.type !== "text") continue;
      try {
        const parsed = JSON.parse(block.text);
        if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
      } catch { /* Not every native diagnostic is JSON. */ }
    }
    throw new Error(`Driver returned no structured ${name} result.`);
  }

  private async register(target: WindowTarget, signal?: AbortSignal): Promise<WindowEntry | undefined> {
    if (this.driver.windowLifetime === false) return undefined;
    if (!Number.isSafeInteger(target.pid) || target.pid <= 0 || !Number.isSafeInteger(target.window_id) || target.window_id <= 0)
      return undefined;
    const identity = await this.driver.identity(target, true, signal);
    if (!identity) return undefined;
    const visualBounds = screenRect((target as WindowTarget & { bounds?: unknown }).bounds);
    for (const entry of this.entries.values()) {
      if (entry.target.pid === target.pid && entry.target.window_id === target.window_id && entry.identity === identity) {
        if (visualBounds) entry.visualBounds = visualBounds;
        return entry;
      }
    }
    const entry: WindowEntry = { ref: "w-" + randomUUID(), target: { pid: target.pid, window_id: target.window_id }, identity, owned: false, visualBounds,
      web: this.browsers.get(`${target.pid}:${target.window_id}`) };
    this.entries.set(entry.ref, entry);
    return entry;
  }

  /** Source-associated cue only: no UI strings, guessed coordinates, capture or input. */
  private activity(entry: WindowEntry, mark: HighlightMark, signal?: AbortSignal, image?: Content,
    outcome?: "delivered" | "refused" | "unknown" | "verified", action?: string, typedText?: string): Promise<HighlightResult | undefined> {
    if (!this.allowed(entry) || signal?.aborted) return Promise.resolve(undefined);
    return notifyWindowObserver(this.observer, entry.target, entry.identity, entry.visualBounds ?? entry.observation?.bounds, signal, mark, image, action, outcome, typedText);
  }
  private async current(ref: string, signal?: AbortSignal): Promise<WindowEntry | undefined> {
    const entry = this.entries.get(ref);
    if (!entry || !this.allowed(entry)) return undefined;
    if (await this.driver.identity(entry.target, false, signal) === entry.identity) return this.allowed(entry) ? entry : undefined;
    this.entries.delete(ref);
    return undefined;
  }

  private async list(signal?: AbortSignal): Promise<Record<string, unknown>[]> {
    const state = await this.state("list_windows", {}, signal);
    if (!Array.isArray(state.windows)) throw new Error("Driver returned no window list.");
    return state.windows as Record<string, unknown>[];
  }

  /** Discover permitted window metadata. Safe never adopts outside-session windows or grants their same-PID siblings. */
  windows(signal?: AbortSignal, query?: string): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before discovery.");
      try {
        const matches = textMatcher(query);
        const windows = [];
        const owned = [...this.entries.values()].filter(entry => entry.owned);
        const nativeWindows = this.getMode() === "safe" && !owned.length ? [] : await this.list(signal);
        for (const native of nativeWindows) {
          // An anchored regex names the title or the app alone, so each is tried as well as both together.
          if (![native.title, native.app_name, `${native.app_name} ${native.title}`].some(text => matches(String(text ?? "")))) continue;
          const known = owned.find(entry => entry.target.pid === native.pid && entry.target.window_id === native.window_id);
          const entry = this.getMode() === "safe"
            ? known && await this.current(known.ref, signal)
            : await this.register(native as unknown as WindowTarget, signal);
          if (this.getMode() === "safe" && (!entry || !entry.owned)) continue;
          windows.push({ ...native, ...(entry ? { ...this.addressed(entry, native.virtual_desktop), owned: entry.owned } : { unavailable: this.driver.windowLifetime === false ? `window_lifetime_unavailable: ${this.platform}. Window control is deferred; no reusable-ID substitute.` : "Cannot establish window lifetime identity.",
            source_identity: this.domained(sourceIdentity({ session_id: this.sessionId, pid: native.pid as number, window_id: native.window_id as number }, null, undefined, this.platform)) }) });
        }
        return this.result("listed", "Permitted window discovery. Safe lists session-opened windows only. Use window_ref only where native lifetime is supported; Linux/macOS currently expose metadata without control. Native desktop membership is sampled and incomplete; discovery does not activate or switch desktops.",
          { windows: this.getMode() === "safe" ? windows.filter(window => "owned" in window && window.owned) : windows });
      } catch (error) { return this.failed(error, false); }
    });
  }

  /** Discover installed app identifiers. Opening accepts an exact name or the returned app_id. */
  apps(signal?: AbortSignal, query?: string): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before app discovery.");
      try {
        const state = await this.state("list_apps", {}, signal);
        if (!Array.isArray(state.apps)) throw new Error("Driver returned no app list.");
        const matches = textMatcher(query);
        const apps = (state.apps as App[]).filter(app => (app.launch_path || app.bundle_id) && matches(app.name))
          .map(app => ({ ...app, app_id: app.launch_path ?? app.bundle_id }));
        return this.result("listed", "Installed native apps. Open by exact name or app_id when window lifetime is supported.", { platform: this.platform, apps });
      } catch (error) { return this.failed(error, false); }
    });
  }

  /**
   * Launch one visible browser with a disposable or persistent agent profile and bind its native window.
   * Safe agent profiles start normally and attach after approval. Closing the session closes the owned browser.
   * attachBrowser also covers browsers that already run.
   */
  openBrowser(options: WebLaunchOptions = {}, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted || !this.launchWeb || this.driver.windowLifetime === false)
        return this.result("refused", "Opening a browser needs Windows window lifetime and the Playwright browser backend; nothing was launched.");
      const approval = options.profile === "agent" && this.getMode() === "safe";
      if (approval && options.browser === "firefox") return this.result("refused", "Safe agent profiles require a Chromium browser with approval-based remote debugging.");
      const unapproved = options.profile === "agent" && !approval && options.browser !== "firefox";
      if (unapproved) this.unapprovedAgentLaunches++;
      let launched: Awaited<ReturnType<WebLauncher>> | undefined;
      try {
        launched = await this.launchWeb(approval ? { ...options, approval: true } : options, signal);
        if (unapproved) this.unapprovedAgentBrowsers.add(launched.browser);
        if (unapproved && this.getMode() !== "yolo") throw new Error("Mode changed while the unapproved browser was starting.");
        const { pid } = launched.opened;
        // Chromium creates its native window asynchronously after Playwright sees the first page.
        let windows: Record<string, unknown>[] = [];
        for (let attempt = 0; attempt < 20 && windows.length !== 1; attempt++) {
          if (attempt) await delay(250, undefined, { signal });
          // A prompt can appear before the browser frame, even as the only window of this process.
          const frames = ((await this.state("list_windows", { pid }, signal)).windows as Record<string, unknown>[] ?? [])
            .filter(window => window.pid === pid && typeof window.title === "string" &&
              (/ - (?:Microsoft\u200b? Edge|Google Chrome|Mozilla Firefox)$/i.test(window.title) ||
                options.browser === "firefox" && / — (?:Mozilla Firefox|Nightly)$/i.test(window.title)));
          const titled = launched.opened.title && frames.filter(window => (window.title as string).startsWith(launched!.opened.title));
          windows = titled && titled.length === 1 ? titled : frames;
        }
        if (windows.length !== 1) throw Error(`Expected one identifiable browser frame for process ${pid}, found ${windows.length}.`);
        const target = windows[0] as unknown as WindowTarget;
        this.browsers.set(`${target.pid}:${target.window_id}`, launched.browser);
        const entry = await this.register(target, signal);
        if (!entry) throw Error("The browser window lifetime could not be established.");
        if (unapproved && this.getMode() !== "yolo") throw new Error("Mode changed while the unapproved browser was starting.");
        entry.owned = true;
        if (approval) { entry.pendingApproval = true; entry.web = undefined; }
        return this.result("observed", `Opened a new visible browser with ${options.profile ? "the agent's persistent profile (logins made in it stay for later sessions)" : "a disposable profile"}. ${approval ? "Sign in if needed, enable remote debugging in the browser, then attach this window and approve the connection." : options.profile && options.browser !== "firefox" ? "The local debugging port lets other local processes control this browser and access its sessions." : ""} Read its window with dom for page structure, or with image/ui/full for native pixels and Windows UI.`,
          { operation: "open_browser", ...launched.opened }, { ...this.addressed(entry), opened: true });
      } catch (error) {
        const reason = error instanceof Error ? error.message : String(error);
        if (!launched) {
          const unknown = error instanceof WebError && error.transport;
          if (unknown && unapproved) this.unconfirmedUnsafeWork = true;
          return this.result(unknown ? "unknown" : "refused", `Browser launch failed: ${reason}`, { operation: "open_browser" });
        }
        for (const [key, browser] of this.browsers) if (browser === launched.browser) this.browsers.delete(key);
        const cleanup = await launched.browser.close().then(() => "confirmed", () => "unconfirmed");
        if (cleanup === "confirmed") this.unapprovedAgentBrowsers.delete(launched.browser);
        return this.result(cleanup === "confirmed" ? "refused" : "unknown", `The browser started but its window could not be bound (cleanup ${cleanup}). ${reason}`,
          { operation: "open_browser", cleanup });
      } finally { if (unapproved) this.unapprovedAgentLaunches--; }
    });
  }
  /**
   * Attach Playwright to the running Chromium-based browser that shows this window (YOLO only): Chrome and Edge, or a
   * browser embedded in another app (Electron, CEF, WebView2, VS Code's integrated browser). The wait for approval
   * runs outside the session queue, so another call can approve the extension's page with ordinary input meanwhile.
   * Closing the session detaches; the browser and its tabs stay open.
   */
  async attachBrowser(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
    const prepared = await this.queue<ComputerResult | { pid: number; title: string; owned?: WebBrowser }>(async () => {
      if (signal?.aborted || !this.launchWeb || this.driver.windowLifetime === false)
        return this.result("refused", "Attaching needs Windows window lifetime and the Playwright browser backend; nothing was attached.");
      const entry = await this.current(ref, signal);
      if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
      if (entry.web) return this.result("refused", "Playwright already works in this browser; read the window with dom.", undefined, this.addressed(entry));
      if (entry.owned && entry.pendingApproval) {
        if (entry.attachingApproval) return this.result("refused", "An approval connection is already pending.");
        const owned = this.browsers.get(`${entry.target.pid}:${entry.target.window_id}`);
        if (!owned) return this.result("refused", "The owned browser is no longer available.");
        entry.attachingApproval = true;
        return { pid: entry.target.pid, title: "", owned };
      }
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode attaches only to its own agent profile after approval.");
      const windows = (await this.state("list_windows", { pid: entry.target.pid }, signal)).windows as Record<string, unknown>[] ?? [];
      const title = windows.find(window => window.window_id === entry.target.window_id)?.title;
      return { pid: entry.target.pid, title: typeof title === "string" ? title : "" };
    });
    if ("content" in prepared) return prepared;
    let launched: Awaited<ReturnType<WebLauncher>>;
    try {
      if (prepared.owned) {
        const opened = await prepared.owned.request<WebOpened>("approve", {}, signal, ATTACH_APPROVAL_MS + 30_000);
        launched = { browser: prepared.owned, opened };
      } else launched = await this.launchWeb!({ attach: { pid: prepared.pid, title: prepared.title } }, signal);
    }
    catch (error) {
      const unknown = error instanceof WebError && (error.transport || error.timeout);
      let cleanup: string | undefined;
      if (prepared.owned) await this.queue(async () => {
        const entry = this.entries.get(ref);
        if (entry) entry.attachingApproval = false;
        if (unknown) {
          if (entry) this.browsers.delete(`${entry.target.pid}:${entry.target.window_id}`);
          cleanup = await prepared.owned!.close().then(() => "confirmed", () => "unconfirmed");
          this.entries.delete(ref);
        }
      });
      return this.result(unknown ? "unknown" : "refused", "Attaching failed: " + (error instanceof Error ? error.message : String(error))
        + (cleanup ? ` Owned browser cleanup ${cleanup}.` : ""), { operation: "attach_browser", ...(cleanup ? { cleanup } : {}) });
    }
    return this.queue(async () => {
      const entry = this.entries.get(ref);
      if (!entry || entry.web || (!prepared.owned && this.getMode() !== "yolo")) {
        const cleanup = await launched.browser.close().then(() => "confirmed", () => "unconfirmed");
        return this.result(cleanup === "confirmed" ? "refused" : "unknown", `The window, attachment or access mode changed while attaching; the connection was released (cleanup ${cleanup}).`, { operation: "attach_browser" });
      }
      this.browsers.set(`${entry.target.pid}:${entry.target.window_id}`, launched.browser);
      entry.web = launched.browser;
      entry.pendingApproval = false;
      entry.attachingApproval = false;
      const via = { cdp: "its debugging endpoint", toggle: "the remote debugging turned on in the browser", extension: "the Playwright Extension", vscode: "VS Code's browser bridge" }[launched.opened.via ?? "extension"];
      return this.result(launched.opened.navigation_error ? "unknown" : "delivered", `Attached Playwright to this browser through ${via}. ${launched.opened.navigation_error ? `The requested navigation outcome is unknown: ${launched.opened.navigation_error}. Do not replay it without checking the page. ` : ""}Read the window with dom for its active tab; every tab is listed there and switch_tab changes the active one. ${prepared.owned ? "Closing the computer session closes this owned browser." : "Closing the computer session detaches without closing the browser."}`,
        { operation: "attach_browser", ...launched.opened }, this.addressed(entry));
    });
  }
  /**
   * Open a new window of an installed app, or in YOLO an executable by its Windows path with command-line
   * arguments (for example an app with a script to run). The result binds exactly one genuinely new window.
   */
  open(app: string | { path: string; args: string[]; debugging?: boolean }, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before app opening.");
      if (this.driver.windowLifetime === false) return this.result("refused",
        `window_lifetime_unavailable: ${this.platform}. Window opening/control is deferred; no application was launched.`);
      if (typeof app !== "string" && this.getMode() !== "yolo")
        return this.result("refused", "Starting an executable by path with arguments runs arbitrary programs; Safe mode allows only installed apps by name. Only the user can select YOLO with /computer-mode.");
      let dispatched = false;
      try {
        if (typeof app !== "string") {
          const name = app.path.split("\\").at(-1)!.replace(/\.exe$/i, "");
          if (app.debugging) {
            // WebView2 reads its Chromium switches from the environment, which CUA's launch cannot set.
            if (!this.host) return this.result("refused", "Starting with WebView2 debugging needs this machine's Windows host; nothing was launched.");
            const host = this.host;
            return await this.launchNew({ name }, async () => ({ pid: await host.startProcess(app.path, app.args, { WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS: WEBVIEW2_DEBUGGING }) }), signal, () => dispatched = true);
          }
          return await this.launchNew({ name }, () => this.state("launch_app", { path: app.path, additional_arguments: app.args }, signal), signal, () => dispatched = true);
        }
        const state = await this.state("list_apps", {}, signal);
        if (!Array.isArray(state.apps)) throw new Error("Driver returned no app list.");
        const apps = (state.apps as App[]).filter(item => item.launch_path);
        const exact = apps.filter(item => appName(item.name) === appName(app) || item.launch_path === app || item.bundle_id === app);
        // An executable name such as "notepad" is used only when no installed name or app_id matches.
        const matches = exact.length ? exact : apps.filter(item => executableName(item.launch_path!) === appName(app));
        const paths = [...new Map(matches.map(item => [item.launch_path, item])).values()];
        if (paths.length !== 1) {
          if (paths.length) return this.result("refused", `Ambiguous app name: ${paths.slice(0, 5).map(item => item.name).join(", ")}. Open one by its exact app_id.`);
          const wanted = appName(app);
          const close = [...new Set(apps.map(item => item.name).filter(name => { const have = appName(name); return have.length > 2 && (wanted.includes(have) || have.includes(wanted)); }))];
          return this.result("refused", `Installed app not found.${close.length ? ` Did you mean: ${close.slice(0, 5).join(", ")}?` : ""} Search the machine's apps (computer:machine/<name>/apps) for exact names.`);
        }
        const selected = paths[0];
        const aumid = selected.launch_path!.match(/^shell:appsFolder\\(.+!.+)$/i)?.[1];
        return await this.launchNew(selected, () => this.state("launch_app", aumid ? { aumid } : { launch_path: selected.launch_path }, signal), signal, () => dispatched = true);
      } catch (error) { return this.failed(error, dispatched); }
    });
  }

  /** Launch with native arguments and bind the one new window of that process or app; called inside the queue. */
  private async launchNew(selected: { name: string }, start: () => Promise<Record<string, unknown>>, signal: AbortSignal | undefined, dispatch: () => void): Promise<ComputerResult> {
    const before = new Set<string>();
    for (const native of await this.list(signal)) before.add(`${native.pid}:${native.window_id}`);
    if (signal?.aborted) return this.result("refused", "Cancelled before launch dispatch.");
    dispatch();
    const launched = await start();
    const pid = launched.pid;
    if (!Number.isSafeInteger(pid) || (pid as number) <= 0) throw new Error("Launch returned no usable process ID; do not relaunch blindly.");
    let candidates: WindowEntry[] = [];
    for (let attempt = 0; attempt < 20; attempt++) {
      const nativeWindows = await this.list(signal);
      candidates = [];
      for (const native of nativeWindows) {
        // Packaged activation may return a broker PID while the new window belongs to an existing app process.
        const reported = Array.isArray(launched.windows) && launched.windows.some(window => {
          const item = window as WindowTarget;
          return item.pid === native.pid && item.window_id === native.window_id;
        });
        if (native.pid !== pid && appName(String(native.app_name ?? "")) !== appName(selected.name) && !reported) continue;
        // Treat baseline addresses as existing even if a handle is recycled during launch: fail closed.
        if (before.has(`${native.pid}:${native.window_id}`)) continue;
        const entry = await this.register(native as unknown as WindowTarget, signal);
        if (entry) candidates.push(entry);
      }
      if (candidates.length) break;
      if (attempt < 19) await delay(250, undefined, { signal });
    }
    if (candidates.length !== 1) return this.result("unknown", candidates.length
      ? "Launch produced multiple possible new windows. Discover them before choosing; ownership is not established."
      : "Launch did not establish a genuinely new window. It may have reused an existing window or opened a tab. Discover before deciding; do not relaunch blindly.");
    const entry = candidates[0];
    entry.owned = true;
    const observed = await this.capture(entry, signal);
    return this.result(observed.details.status, observed.details.status === "observed"
      ? "New application window opened. First observation is ready for one action."
      : "A new window appeared but its first observation failed. Discover before deciding.",
      observed.details.state, { ...observed.details, opened: true }, observed.content.filter(block => block.type === "image"));
  }

  /** Reproject proven window-local native geometry onto fresh CUA frames, never DPI guesses. */
  private actionMark(mark: GuideMark | undefined, mapping: DetailContext["windowMapping"]): GuideMark | undefined {
    if (!mark?.region) return mark;
    if (!mapping) return { reason: "Current native-to-image mapping is unavailable.", basis: mark.basis };
    const r = mark.region;
    return { basis: mark.basis, region: { x: (r.x - mapping.offsetX) / mapping.scale - 4,
      y: (r.y - mapping.offsetY) / mapping.scale - 4, width: r.width / mapping.scale + 8, height: r.height / mapping.scale + 8 } };
  }
  private imageMetadata(images: Content[], state: Record<string, unknown>, settings: CuaVisionSettings): Record<string, unknown> {
    const image = images.find(block => block.type === "image");
    if (!image || image.type !== "image") throw new Error("No native grounding image.");
    const bytes = Buffer.from(image.data, "base64");
    return { route: state.display === "primary" ? "cua_display" : "cua_window", configuration: settings,
      output: { width: state.screenshot_width, height: state.screenshot_height, bytes: bytes.length,
        mimeType: image.mimeType, hash: createHash("sha256").update(bytes).digest("hex") },
      coordinate_space: state.display === "primary" ? "display_image_pixels" : "window_image_pixels",
      native: { capture_id: state.capture_id, agent_overlay_capture: state.agent_overlay_capture } };
  }

  /** A temporal cue represents one returned operation; background capture samples stay silent. */
  private async frameActivity(source: ObservationSource | undefined, image?: Content, signal?: AbortSignal, action = "frames"): Promise<void> {
    if (!source || signal?.aborted || !this.observer) return;
    const bounds = screenRect(source.image.native_bounds);
    if (source.kind === "window") {
      const entry = this.entries.get(String(source.identity.window_ref));
      if (entry && this.allowed(entry)) await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,
        { kind: "inspect" }, image, action);
    } else if (this.getMode() === "yolo") {
      const display = String(source.identity.display_id ?? "primary");
      const original = source.image.original_dimensions;
      const physical = bounds ?? (display === "primary" && original ? { x: 0, y: 0, ...original } : undefined);
      await notifyHighlight(this.observer, physical ? { display, bounds: physical, mark: { kind: "inspect" },
        ...(image?.type === "image" ? { image } : {}), action } : undefined, signal);
    }
  }
  /** Capture one authorized source into a private bounded resource; no frame grants input or detail authority. */
  captureSequence(target: { window_ref?: string; display?: string }, options: SequenceOptions, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      try {
        validateSequence(options);
        if ((target.window_ref !== undefined) === (target.display !== undefined)) return this.result("refused", "Choose exactly one window or display.");
        const entry = target.window_ref ? this.entries.get(target.window_ref) : undefined;
        // primary is the machine's main display; any other display is one its displays listing issued.
        const display = target.display !== undefined && target.display !== "primary" ? this.displayEntries.get(target.display) : undefined;
        if (target.display !== undefined && target.display !== "primary" && (!this.driver.selectedDisplays || !display))
          return this.result("refused", "Unknown or unavailable display reference. Read the machine's displays again; no primary fallback.");
        const permitted = () => entry ? this.allowed(entry) : target.display !== undefined && this.getMode() === "yolo";
        const denied = () => { this.sequences.clear(); return this.result("refused", "Sequence source access denied. Safe permits session-owned windows only; retained pixels withheld."); };
        if (!permitted()) return denied();
        const settings = { ...this.overviewSettings(), ...(options.maxLongEdge !== undefined ? { maxLongEdge: options.maxLongEdge } : {}),
          ...(options.scale !== undefined ? { scale: options.scale } : {}) };
        const cap = nativeImageCap(settings);
        if (entry) { entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined; }
        else if (display) display.detail = undefined;
        else this.displayOverview = undefined;
        let first = true;
        const sequence = await collectSequence(options, async active => {
          if (!permitted()) throw new SequenceStop("access_revoked");
          if (entry && !await this.current(entry.ref, active)) throw new SequenceStop("target_lost");
          if (!permitted()) throw new SequenceStop("access_revoked");
          const nativeStarted = performance.now();
          const native = entry ? await this.driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap }, active)
            : await this.readDisplay(display, cap, active);
          const native_completed_at = performance.now();
          if (!permitted()) throw new SequenceStop("access_revoked");
          active.throwIfAborted();
          const state = native.structuredContent;
          if (native.isError || !state || state.status === "refused" || state.effect === "refused") throw new SequenceStop("capture_failed");
          if (!entry && !this.displayStateMatches(state, display)) throw new SequenceStop("capture_failed");
          if (entry && ((state.pid !== undefined && state.pid !== entry.target.pid) ||
            (state.window_id !== undefined && state.window_id !== entry.target.window_id))) throw new SequenceStop("target_lost");
          const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");
          const context = this.overview(state, "sequence-frame", !entry, images);
          if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024)
            throw new SequenceStop("capture_failed");
          let source = display ? selectedDisplaySource(state, this.sessionId, display.ref) : this.source(entry ? "window" : "display", state, entry
            ? { session_id: this.sessionId, window_ref: entry.ref, ...entry.target }
            : { session_id: this.sessionId, display_id: "primary" });
          source.semantic_scope = "none"; source.capabilities = { read: true, input: "none", detail: "unsupported" };
          let image = images[0];
          if (options.region) ({ image, source } = await cropSequenceFrame(image, source, options.region, first));
          first = false;
          if (entry && !await this.current(entry.ref, active)) throw new SequenceStop("target_lost");
          if (!permitted()) throw new SequenceStop("access_revoked");
          return { image, source, native_completed_at, native_latency_ms: native_completed_at - nativeStarted };
        }, signal);
        if (!permitted() || sequence.stop_reason === "access_revoked") return denied();
        sequence.configuration = settings;
        this.sequences.put(sequence);
        const metadata = sequenceMetadata(sequence);
        const complete = ["completed", "frame_limit"].includes(sequence.stop_reason);
        if (complete) await this.frameActivity(sequence.frames.at(-1)?.source, undefined, signal, "sequence");
        if (!permitted()) return denied();
        return this.result(complete ? "observed" : "refused", `Sequence ${sequence.stop_reason}: ${sequence.frames.length} frames, ${sequence.skipped_slots.length} skipped slots. No images forwarded; select frames explicitly. Read only; observe again before input.`,
          undefined, { sequence: metadata });
      } catch (error) {
        return this.result("refused", `Sequence did not start. ${error instanceof Error ? error.message : "Invalid capture request."}`);
      }
    });
  }

  /** Select retained pixels only after rechecking source identity and current access; no native recapture. */
  sequenceFrames(id: string, selection: FrameSelection, signal?: AbortSignal, rendering?: FrameRendering): Promise<ComputerResult> {
    return this.queue(async () => {
      const sequence = this.sequences.get(id);
      if (!sequence) return this.result("refused", "Sequence is unavailable or evicted. Capture again; resources do not survive reload/close.");
      const source = sequence.frames[0]?.source;
      const entry = source?.kind === "window" ? this.entries.get(String(source.identity.window_ref)) : undefined;
      const permitted = () => source?.kind === "window" ? !!entry && this.allowed(entry) : source?.kind === "display" && this.getMode() === "yolo";
      const denied = () => { this.sequences.clear(); return this.result("refused", "Retained sequence source is no longer accessible; pixels and metadata withheld."); };
      if (!permitted()) return denied();
      try {
        if (signal?.aborted) return this.result("refused", "Cancelled before frame selection.");
        if (entry && !await this.current(entry.ref, signal)) return denied();
        if (!permitted()) return denied();
        const frames = selectFrames(sequence, selection);
        const metadata = frames.map(({ image: _image, ...frame }) => frame);
        if (signal?.aborted) return this.result("refused", "Cancelled during frame selection; pixels withheld.");
        if (rendering) {
          let rendered;
          try { rendered = await renderFrames(frames, rendering); }
          catch (error) { return this.result("refused", `Frames not rendered: ${error instanceof Error ? error.message : error}`); }
          if (signal?.aborted) return this.result("refused", "Cancelled during frame rendering; pixels withheld.");
          if (!permitted()) return denied();
          const { png, ...layout } = rendered;
          const crop = layout.region ? ` of region ${layout.region.x},${layout.region.y} ${layout.region.width}x${layout.region.height}` : "";
          const summary = rendering.as === "sheet"
            ? `Contact sheet of ${frames.length} frames${crop}: ${layout.columns}x${layout.rows} tiles of ${layout.tiles[0].width}x${layout.tiles[0].height} in time order, left to right, each under its "#index · time" label; image ${layout.width}x${layout.height}.\n` +
              layout.tiles.map((tile, i) => `Tile ${i + 1} (row ${Math.floor(i / layout.columns!) + 1}, column ${i % layout.columns! + 1}): frame ${tile.index} at ${tile.timestamp_ms.toFixed(1)} ms`).join("\n")
            : `Per-pixel ${rendering.as === "max" ? "maximum" : "minimum"} of ${frames.length} frames${crop} (frames ${frames.map(frame => frame.index).join(", ")}); image ${layout.width}x${layout.height}. ` +
              `${rendering.as === "max" ? "Bright" : "Dark"} detail from any frame shows at once, for example text revealed by a moving light.`;
          const image: Content = { type: "image", mimeType: "image/png", data: png.toString("base64") };
          await this.frameActivity(frames.at(-1)?.source, image, signal);
          if (!permitted() || signal?.aborted) return denied();
          return this.result("observed", `${summary}\nRead only; observe again before input.`, undefined, { selected_frames: metadata, rendered_frames: layout }, [image]);
        }
        // Interleave frame labels with their images so both direct calls and Codemode preserve correspondence.
        const result = this.result("observed", `Selected ${frames.length} read-only sequence frames. Their indexes and measured timestamps label the image order.`, undefined,
          { selected_frames: metadata });
        for (const frame of frames) result.content.push({ type: "text", text: `Sequence ${id} · frame ${frame.index} · ${frame.timestamp_ms.toFixed(1)} ms` }, frame.image);
        await this.frameActivity(frames.at(-1)?.source, frames.at(-1)?.image, signal);
        if (!permitted() || signal?.aborted) return denied();
        return result;
      } catch {
        if (!permitted()) return denied();
        return this.result("refused", "Frame selection failed. Check captured-frame indexes and the source's current lifetime.");
      }
    });
  }

  /**
   * Start watching a window, or with `crop` (a viewport's rectangle in full-resolution window pixels) a part of it.
   * Frames come from a second driver process in the background, outside the session's queue, so the agent acts
   * meanwhile and a long drag is seen while it runs. Returns once the first frame is in; the sequence then grows until
   * stopWatch, its time limit, its frame or byte limit, or the session closing.
   */
  async startWatch(window_ref: string, watching: string, options: { fps: number; timeout_ms: number; maxLongEdge: number; crop?: PixelBox }, signal?: AbortSignal): Promise<ComputerResult> {
    const entry = this.entries.get(window_ref);
    if (!entry || !this.allowed(entry)) return this.result("refused", "Unknown or unavailable window reference. Search its machine's windows again.");
    const permitted = () => this.entries.get(window_ref) === entry && this.allowed(entry);
    const driver = this.watcher ??= this.driver.companion?.() ?? this.driver;
    // Watch frames come from the window's own pixels at a size that gives the crop maxLongEdge pixels on its long edge.
    let cap = Math.min(FULL_RESOLUTION, options.maxLongEdge);
    // With a native crop the driver copies and encodes only the viewport, in the window's physical pixels.
    let nativeCrop: PixelBox | undefined;
    const fullSize = (state: Record<string, unknown>) => {
      const width = state.screenshot_original_width, height = state.screenshot_original_height;
      if (typeof width !== "number" || typeof height !== "number" || width <= 0 || height <= 0) return undefined;
      const fit = Math.min(1, FULL_RESOLUTION / Math.max(width, height));
      return { width: Math.round(width * fit), height: Math.round(height * fit) };
    };
    if (options.crop) {
      let probe: DriverResult;
      try { probe = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: 64 }, signal); }
      catch (error) { return this.result("refused", `The watch did not start: ${error instanceof Error ? error.message : error}`); }
      const full = probe.isError || !probe.structuredContent ? undefined : fullSize(probe.structuredContent);
      if (!full) return this.result("refused", "The watch did not start: the window could not be captured.");
      const long = Math.max(options.crop.width, options.crop.height);
      cap = Math.max(64, Math.min(FULL_RESOLUTION, Math.ceil(Math.max(full.width, full.height) * Math.min(options.maxLongEdge, long) / long)));
      if (driver.windowCrop) {
        const kx = (probe.structuredContent!.screenshot_original_width as number) / full.width;
        const ky = (probe.structuredContent!.screenshot_original_height as number) / full.height, crop = options.crop;
        nativeCrop = { x: Math.floor(crop.x * kx), y: Math.floor(crop.y * ky), width: Math.max(1, Math.ceil(crop.width * kx)), height: Math.max(1, Math.ceil(crop.height * ky)) };
        cap = Math.max(64, Math.min(FULL_RESOLUTION, options.maxLongEdge, long));
      }
    }
    const capture = async (active: AbortSignal) => {
      if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
      const nativeStarted = performance.now();
      const native = await driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: cap,
        ...(nativeCrop ? { display_only: true, crop: nativeCrop } : {}) }, active);
      const native_completed_at = performance.now();
      active.throwIfAborted();
      if (!permitted()) throw new SequenceStop(this.entries.has(window_ref) ? "access_revoked" : "target_lost");
      const state = native.structuredContent;
      if (native.isError || !state || state.status === "refused" || state.effect === "refused") throw new SequenceStop("capture_failed");
      // Identity is not asked here: its peer takes one request at a time and the session's own calls use it.
      if ((state.pid !== undefined && state.pid !== entry.target.pid) || (state.window_id !== undefined && state.window_id !== entry.target.window_id))
        throw new SequenceStop("target_lost");
      const images = native.content.filter((block): block is SequenceFrame["image"] => block.type === "image");
      if (nativeCrop) return { ...nativeCropFrame(state, images, nativeCrop, cap, this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target })),
        native_completed_at, native_latency_ms: native_completed_at - nativeStarted };
      const context = this.overview(state, "watch-frame", false, images);
      if (!context || Math.max(context.width, context.height) > cap || images[0].data.length > 16 * 1024 * 1024) throw new SequenceStop("capture_failed");
      let source = this.source("window", state, { session_id: this.sessionId, window_ref, ...entry.target });
      source.semantic_scope = "none"; source.capabilities = { read: true, input: "none", detail: "unsupported" };
      let image = images[0];
      const full = options.crop && fullSize(state);
      if (options.crop && full) {
        // The viewport follows the window: its rectangle is relative to the window's top-left corner.
        const kx = context.width / full.width, ky = context.height / full.height, crop = options.crop;
        const region = { x: Math.round(crop.x * kx), y: Math.round(crop.y * ky), width: Math.max(1, Math.round(crop.width * kx)), height: Math.max(1, Math.round(crop.height * ky)) };
        if (region.x >= context.width || region.y >= context.height) throw new SequenceStop("capture_failed");
        ({ image, source } = await cropSequenceFrame(image, source, region, false));
      }
      return { image, source, native_completed_at, native_latency_ms: native_completed_at - nativeStarted };
    };
    const stop = new AbortController();
    const watch: Watching = { window_ref, watching, started: 0, marks: [], stop, finished: false, done: Promise.resolve(undefined as unknown as FrameSequence) };
    // A frame of a large window takes longer than a frame interval; a second capture in flight keeps the rate up.
    watch.done = collectSequence({ duration_ms: options.timeout_ms, fps: options.fps, max_frames: WATCH_LIMITS.max_frames, timeout_ms: options.timeout_ms + 20_000 }, capture,
      undefined, undefined, { limits: WATCH_LIMITS, stop: stop.signal, concurrency: WATCH_CONCURRENCY, started: (sequence, at) => {
        watch.sequence = sequence; watch.started = at;
        sequence.configuration = { ...this.overviewSettings(), maxLongEdge: cap };
        this.sequences.put(sequence, true);
        this.watches.set(sequence.sequence_id, watch);
      } }).then(sequence => {
      watch.finished = true;
      if (this.watches.get(sequence.sequence_id) === watch) this.sequences.put(sequence);
      return sequence;
    });
    // Return with the first frame, so a watch that cannot capture says so now; starting the second driver takes a few seconds.
    const deadline = performance.now() + 30_000;
    while (!watch.finished && !watch.sequence?.frames.length && performance.now() < deadline && !signal?.aborted) await delay(20);
    const sequence = watch.sequence!;
    if (!sequence?.frames.length) {
      stop.abort();
      const ended = await watch.done;
      this.watches.delete(ended.sequence_id);
      return this.result("refused", `The watch did not start: ${signal?.aborted ? "cancelled" : ended.stop_reason === "stopped" ? "no frame came within 30 s" : ended.stop_reason}.`);
    }
    const size = sequence.frames[0].source.image.dimensions;
    await this.frameActivity(sequence.frames[0].source, undefined, signal, "watch");
    if (!this.allowed(entry) || signal?.aborted) { watch.stop.abort(); return this.result("refused", "Watch access revoked or cancelled."); }
    return this.result("observed", `Watching ${watching} at up to ${options.fps} FPS in the background, ${size.width}x${size.height} frames. ` +
      `Act on the window as usual: each action's receipt names its span and frames in this watch. Read the sequence with frames:last for the newest frame; computer_record watch_stop at the sequence returns the contact sheet, change statistics and files. ` +
      `It stops by itself after ${Math.round(options.timeout_ms / 1000)} s or ${WATCH_LIMITS.max_frames} frames, and when the session closes.`,
      undefined, { ...this.addressed(entry), sequence: runningSequenceMetadata(sequence, performance.now() - watch.started), watch: { sequence_id: sequence.sequence_id, watching, running: true, marks: [] } });
  }

  /** Note an action on a window in each running watch of it: the action's span in the watch's time and the frames captured during it so far. */
  markWatches(window_ref: string, label: string, start: number, end: number): WatchSpan[] {
    const spans: WatchSpan[] = [];
    for (const watch of this.watches.values()) {
      if (watch.window_ref !== window_ref || watch.finished || !watch.sequence) continue;
      const mark = { label, start_ms: start - watch.started, end_ms: end - watch.started };
      watch.marks.push(mark);
      spans.push({ sequence_id: watch.sequence.sequence_id, label, from_ms: mark.start_ms, to_ms: mark.end_ms, frames: framesBetween(watch.sequence.frames, mark.start_ms, mark.end_ms) });
    }
    return spans;
  }
  /** Whether this sequence is a watch that has not been stopped with stopWatch yet. */
  isWatch(id: string): boolean { return this.watches.has(id); }

  /**
   * Stop a watch: a contact sheet of the frames that changed with the actions marked, change statistics of every
   * frame, and the frames as files in `dir` (a temporary folder removed when the session closes, unless given), with an
   * animated WebP when asked. The sequence keeps its frames for frames: Reads.
   */
  async stopWatch(id: string, files: { dir?: string; animation?: boolean } = {}): Promise<ComputerResult> {
    const watch = this.watches.get(id);
    if (!watch) return this.result("refused", this.sequences.get(id) ? "This sequence is no running watch; Read it with a frames: view." : "No watch of this session has this address. Watches do not survive reload or close.");
    watch.stop.abort();
    const sequence = await watch.done;
    this.watches.delete(id);
    const entry = this.entries.get(watch.window_ref);
    if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") {
      this.sequences.clear();
      return this.result("refused", "The watched window is no longer accessible; its frames are withheld.");
    }
    const reasons: Record<FrameSequence["stop_reason"], string> = { stopped: "stopped", completed: "stopped by itself at its time limit", frame_limit: "stopped by itself at its frame limit",
      byte_limit: "stopped by itself at its memory limit", timeout: "stopped by itself at its time limit", cancelled: "cancelled", target_lost: "stopped: the window closed",
      access_revoked: "stopped: access to the window ended", capture_failed: "stopped: a capture failed" };
    const frames = sequence.frames;
    const changes = await frameChanges(frames);
    const distinct = frames.filter((_, i) => differs(frames, i)).length;
    const lines = [`Watch of ${watch.watching} ${reasons[sequence.stop_reason]}: ${frames.length} frames over ${(sequence.elapsed_ms / 1000).toFixed(1)} s` +
      `${sequence.actual_fps ? ` at ${sequence.actual_fps.toFixed(1)} FPS` : ""}; ${distinct === 1 ? "1 differs" : `${distinct} differ`} from the frame before.`];
    if (watch.marks.length) lines.push("Actions (ms of the watch):", ...watch.marks.map(mark => {
      const span = framesBetween(frames, mark.start_ms, mark.end_ms);
      return `- ${mark.label} ${Math.round(mark.start_ms)}–${Math.round(mark.end_ms)} ms, ${span ? `frames ${span.from}–${span.to}` : "no frame during it"}`;
    }));
    const moving = changeLines(changes, watch.marks);
    lines.push(moving.length ? "Biggest changes against the frame before (share of the frame's pixels, box in its pixels):" : "No frame changed against the one before.", ...moving);
    const content: Content[] = [];
    let rendered: Omit<RenderedFrames, "png"> | undefined;
    if (frames.length) {
      const { png, ...layout } = await renderFrames(sheetFrames(frames, watch.marks), { as: "sheet", marks: watch.marks });
      rendered = layout;
      lines.push(`Contact sheet: the frames that changed (at most ${WATCH_SHEET_FRAMES}, kept around each action and spread over the rest) and the last one, ${layout.columns}x${layout.rows} tiles in time order, each under "#index · time"; an orange label means an action ran then. Frames: ${layout.tiles.map(tile => `#${tile.index}`).join(", ")}.`);
      content.push({ type: "image", mimeType: "image/png", data: png.toString("base64") });
    }
    const dir = files.dir ?? join(tmpdir(), "pi-computer-use-watch", id);
    let written: WatchFiles | undefined;
    try {
      written = await writeWatchFiles(dir, sequence, watch.marks, changes, { watching: watch.watching }, files.animation);
      if (!files.dir) this.watchDirs.add(dir);
      lines.push(`Files in ${dir}: ${written.frames} PNG ${written.frames === 1 ? "frame" : "frames"} (each frame that differs from the one before), watch.json with every frame's time, file, change and actions${written.animation ? ", and watch.webp, an animation in real time" : ""}.` +
        (files.dir ? "" : " The folder is removed when the computer session closes."));
    } catch (error) { lines.push(`The frames were not written to ${dir}: ${error instanceof Error ? error.message : error}`); }
    lines.push("Read the sequence with a frames: view for other frames.");
    const result = this.result(frames.length ? "observed" : "refused", lines.join("\n"), undefined,
      { sequence: sequenceMetadata(sequence), ...(rendered ? { rendered_frames: rendered } : {}),
        watch: { sequence_id: id, watching: watch.watching, running: false, marks: watch.marks, changed_frames: distinct, ...(written ? { files: written } : {}) } });
    result.content.push(...content);
    await this.frameActivity(frames.at(-1)?.source, content.find(block => block.type === "image"), undefined, "watch");
    if (entry ? !this.allowed(entry) : this.getMode() !== "yolo") return this.result("refused", "Watch access revoked; pixels withheld.");
    return result;
  }
  /** Enumerate native Windows monitors in YOLO. References grant read access, never input. */
  displays(signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const denied = () => { this.displayEntries.clear(); return this.result("refused", "Safe mode forbids display discovery; native metadata withheld."); };
      if (this.getMode() !== "yolo") return denied();
      if (!this.driver.selectedDisplays) return this.result("refused", "Selected displays require an explicitly qualified Windows runtime.");
      if (signal?.aborted) return this.result("refused", "Cancelled before display discovery.");
      try {
        // Remote Desktop is asked alongside the listing; when it cannot be told, the listing says nothing about it.
        const remote = this.host?.displayModes.remoteSession(signal).catch(() => false) ?? Promise.resolve(false);
        const state = await this.state("list_displays", {}, signal);
        const remoteSession = await remote;
        if (this.getMode() !== "yolo") return denied();
        if (signal?.aborted) return this.result("refused", "Cancelled during display discovery; metadata withheld.");
        if (state.platform !== "windows" || typeof state.topology_id !== "string" || !state.topology_id || !Array.isArray(state.displays) || !state.displays.length)
          throw new Error("CUA returned no native Windows display inventory.");
        const topology = state.topology_id;
        const displays = state.displays.map(nativeDisplay);
        if (new Set(displays.map(d => d.display_id)).size !== displays.length) throw new Error("CUA returned duplicate native display identities.");
        const current = new Set<string>();
        const addressed = displays.map(d => {
          let entry = [...this.displayEntries.values()].find(e => e.topology === topology && e.native.display_id === d.display_id);
          if (!entry) { entry = { ref: "d-" + randomUUID(), topology, native: d }; this.displayEntries.set(entry.ref, entry); }
          current.add(entry.ref);
          return { ...d, display_ref: entry.ref, source_identity: this.domained(sourceIdentity({ session_id: this.sessionId, display_ref: entry.ref, display_id: d.display_id, topology_id: topology }, d.display_id)) };
        });
        for (const ref of this.displayEntries.keys()) if (!current.has(ref)) this.displayEntries.delete(ref);
        return this.result("listed", "Native connected Windows displays. Read a returned display address for its pixels. Image pixels start at zero; bounds are host-global. No input authority." +
          (remoteSession ? ` These displays belong to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""),
          { platform: "windows", topology_id: topology, ...(remoteSession ? { remote_session: true } : {}), displays: addressed });
      } catch (error) {
        this.displayEntries.clear();
        if (this.getMode() !== "yolo") return denied();
        if (signal?.aborted) return this.result("refused", "Cancelled during display discovery; metadata withheld.");
        return this.result("refused", `Native display discovery failed. ${error instanceof Error ? error.message : error}`);
      }
    });
  }

  /**
   * Read the controlled machine's clipboard in YOLO: its formats, text and, when asked, its image as PNG.
   * The clipboard is shared with the user, so it is read only on this explicit call.
   */
  clipboardRead(image: boolean, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids clipboard access: it is shared with the user's own apps. Only the user can select YOLO with /computer-mode.");
      if (signal?.aborted) return this.result("refused", "Cancelled before reading the clipboard.");
      try {
        if (image && !this.host) return this.result("refused", "Clipboard images need the machine host, which this desktop does not have.");
        const state = await this.state("clipboard_read", { include_text: true }, signal);
        const png = image ? await this.host!.clipboardImage() : undefined;
        const text = typeof state.text === "string" ? state.text : undefined;
        return this.result("observed", `Clipboard formats: ${JSON.stringify(state.types ?? state.available_types ?? [])}.${text !== undefined ? `\nText (${text.length} characters):\n${text}` : " No text."}${image ? png ? " Image attached." : " No image." : ""}`,
          { types: state.types ?? state.available_types, ...(text !== undefined ? { text } : {}) }, {}, png ? [{ type: "image", mimeType: "image/png", data: png.toString("base64") }] : []);
      } catch (error) { return this.result("refused", `Clipboard read failed: ${error instanceof Error ? error.message : error}`); }
    });
  }

  /**
   * Record the main display to video in YOLO, into this session's folder, for at most `seconds`.
   * The driver captures with ffmpeg; recording stops on record_stop, at the limit or when the session closes.
   */
  recordStart(seconds: number, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids display recording. Only the user can select YOLO with /computer-mode.");
      if (!this.host) return this.result("refused", "Recording needs the session folder, which this desktop does not have.");
      if (this.recording) return this.result("refused", "A recording is already running; stop it first.");
      const folder = await this.host.files();
      const native = await this.driver.call("start_recording", { output_dir: folder.windows, record_video: true, include_accessibility_tree: false }, signal);
      const state = native.structuredContent ?? {};
      if (native.isError || state.video_active !== true) {
        await this.driver.call("stop_recording", {}).catch(() => undefined);
        return this.result("refused", `Recording did not start${state.last_error ? `: ${state.last_error}` : ""}. Video needs ffmpeg on that machine (for example winget install Gyan.FFmpeg).`, state);
      }
      const timer = setTimeout(() => { void this.recordStop(); }, seconds * 1000);
      this.recording = { timer, started: Date.now() };
      return this.result("delivered", `Recording the main display for up to ${seconds} s. Stop it earlier with computer_record video_stop to fetch the video.`, state);
    });
  }

  /** Record the machine's playback audio (YOLO) into the session folder as audio-<n>.wav. */
  recordAudio(seconds: number): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids audio recording: it can capture the user's calls and media. Only the user can select YOLO with /computer-mode.");
      if (!this.host) return this.result("refused", "Audio recording needs the session folder, which this desktop does not have.");
      try {
        const name = await this.host.recordAudio(seconds, `audio-${++this.audioClips}.wav`);
        return this.result("delivered", `Recorded ${seconds} s of the playback audio as ${name} in the session folder.`, { name });
      } catch (error) { return this.result("refused", `Audio not recorded: ${error instanceof Error ? error.message : error}`); }
    });
  }

  /** Stop the running recording; the video stays in the session folder as recording-<n>.mp4. */
  recordStop(): Promise<ComputerResult> {
    return this.queue(async () => {
      const current = this.recording;
      if (!current || !this.host) return this.result("refused", "No recording is running, or it already stopped at its limit; list the session folder with computer_files to see saved videos.");
      clearTimeout(current.timer);
      this.recording = undefined;
      const native = await this.driver.call("stop_recording", {});
      const name = `recording-${++this.recordings}.mp4`;
      try { await this.host.rename("recording.mp4", name); }
      catch (error) { return this.result("unknown", `Recording stopped, but its video was not found: ${error instanceof Error ? error.message : error}`, native.structuredContent); }
      return this.result("delivered", `Recorded ${Math.round((Date.now() - current.started) / 1000)} s of the main display as ${name} in the session folder.`, { ...native.structuredContent, name });
    });
  }

  /** Turn WebView2 remote debugging on or off for one app's next start (YOLO only); see MachineHost.webview2Debugging. */
  webview2Debugging(app: string, enabled: boolean): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "WebView2 debugging opens an app's pages to automation, so Safe mode refuses it. Only the user can select YOLO with /computer-mode.");
      if (!this.host) return this.result("refused", "This desktop has no Windows host for WebView2 settings.");
      try {
        const value = await this.host.webview2Debugging(app, enabled);
        return this.result("delivered", enabled
          ? `WebView2 debugging is on for ${app} (${value}). It applies when the app next starts, whoever starts it, and stays until turned off; restart the app, then attach its window with computer_open.`
          : `WebView2 debugging is off for ${app}. It applies when the app next starts.`, { operation: "webview2_debugging", app, enabled, value: value ?? null });
      } catch (error) { return this.result("refused", `Changing the WebView2 policy failed: ${error instanceof Error ? error.message : error}`); }
    });
  }
  /** Replace the controlled machine's clipboard in YOLO with text or an image file already on that machine. */
  clipboardWrite(value: { text: string } | { imagePath: string }, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids clipboard access: it is shared with the user's own apps. Only the user can select YOLO with /computer-mode.");
      if (signal?.aborted) return this.result("refused", "Cancelled before writing the clipboard.");
      try {
        const state = await this.state("clipboard_write", "text" in value ? { text: value.text } : { image_path: value.imagePath }, signal);
        return this.result("delivered", `Clipboard replaced with ${"text" in value ? `text (${value.text.length} characters)` : "an image"}. Paste with ctrl+v in the target window.`, state);
      } catch (error) { return this.result("refused", `Clipboard write failed: ${error instanceof Error ? error.message : error}`); }
    });
  }

  /** Change a discovered display's resolution in YOLO. Closing the session restores the mode it replaced. */
  resizeDisplay(display: string, mode: DisplayMode, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids display changes. Only the user can select YOLO with /computer-mode.");
      if (!this.host) return this.result("refused", "Display resizing is not available for this desktop.");
      const entry = this.displayEntries.get(display);
      if (!entry) return this.result("refused", "Unknown or outdated display reference. Read the displays again.");
      if (signal?.aborted) return this.result("refused", "Cancelled before the display change.");
      let change;
      try { change = await this.host.displayModes.set(entry.native.monitor_handle, mode, signal); }
      catch (error) {
        // A cancelled or timed-out change may still have reached Windows.
        if (signal?.aborted) return this.result("unknown", "Cancelled during the display change; the resolution may have changed. Read the displays again.");
        const remote = await this.host.displayModes.remoteSession(signal).catch(() => false);
        return this.result("refused", `Display not changed. ${error instanceof Error ? error.message : error}` +
          (remote ? ` This display belongs to a Remote Desktop session: ${REMOTE_DESKTOP_DISPLAY}` : ""));
      }
      if (!this.replacedModes.has(change.device)) this.replacedModes.set(change.device, change.previous);
      const original = this.replacedModes.get(change.device)!;
      if (original.width === change.current.width && original.height === change.current.height) this.replacedModes.delete(change.device);
      // New geometry: display references and window coordinates from before no longer hold.
      this.displayEntries.clear(); this.displayOverview = undefined;
      return this.result("delivered", `Display ${change.device} is now ${change.current.width}x${change.current.height} (was ${change.previous.width}x${change.previous.height}). ` +
        `Read the displays and windows again before acting; old display references are gone. Closing the computer session restores ${original.width}x${original.height}.`);
    });
  }

  private readDisplay(entry: DisplayEntry | undefined, cap: number, signal?: AbortSignal): Promise<DriverResult> {
    return entry ? this.driver.call("get_display_state", { display_id: entry.native.display_id, topology_id: entry.topology, max_image_dimension: cap }, signal)
      : this.driver.call("get_desktop_state", { max_image_dimension: cap }, signal);
  }

  private displayStateMatches(state: Record<string, unknown> | undefined, entry?: DisplayEntry): state is Record<string, unknown> {
    if (!state || state.platform !== "windows") return false;
    if (!entry) return state.display === "primary";
    const d = nativeDisplay(state.display);
    return state.topology_id === entry.topology && d.display_id === entry.native.display_id &&
      state.screenshot_original_width === d.bounds.width && state.screenshot_original_height === d.bounds.height;
  }
  /** Read primary or a discovered native display in YOLO. Display pixels never ground input. */
  observeDisplay(display: string, signal?: AbortSignal, zoom?: DetailRequest, size?: { maxLongEdge?: number; scale?: number }): Promise<ComputerResult> {
    return this.queue(async () => {
      const denied = () => {
        this.displayOverview = undefined; this.displayEntries.clear();
        return this.result("refused", "Safe mode forbids full-display reading. Only the user can select YOLO with /computer-mode.");
      };
      if (this.getMode() !== "yolo") return denied();
      if (signal?.aborted) return this.result("refused", "Cancelled before display capture.");
      const entry = display === "primary" ? undefined : this.displayEntries.get(display);
      if (display !== "primary" && (!this.driver.selectedDisplays || !entry)) return this.result("refused", "Unknown or unavailable display reference. Discover native displays again; no primary fallback.");
      if (zoom) return this.displayDetail(zoom, signal, entry);
      const retire = () => { if (entry) entry.detail = undefined; else this.displayOverview = undefined; };
      retire();
      try {
        const settings = { ...this.overviewSettings(), ...size }; const cap = nativeImageCap(settings);
        const native = await this.readDisplay(entry, cap, signal);
        if (this.getMode() !== "yolo") return denied();
        if (signal?.aborted) return this.result("refused", "Cancelled during display capture; pixels withheld.");
        const state = native.structuredContent;
        if (native.isError || state?.status === "refused" || state?.effect === "refused") {
          if (entry) this.displayEntries.delete(entry.ref);
          // The native reason, such as a locked PC, tells the agent whether discovering again can help.
          const reason = native.content.filter(c => c.type === "text").map(c => c.text).join(" ").slice(0, 600);
          return this.result("refused", `CUA refused exact-display capture${reason ? `: ${reason}` : "."} No fallback.`);
        }
        const images = native.content.filter(block => block.type === "image" && block.mimeType === "image/png");
        if (!this.displayStateMatches(state, entry) || images.length !== 1 ||
          !Number.isSafeInteger(state!.screenshot_width) || !Number.isSafeInteger(state!.screenshot_height) ||
          (state!.screenshot_width as number) <= 0 || (state!.screenshot_height as number) <= 0 ||
          Math.max(state!.screenshot_width as number, state!.screenshot_height as number) > cap)
          throw new Error("CUA returned no valid scoped display image.");
        const source = entry ? selectedDisplaySource(state!, this.sessionId, entry.ref) : this.source("display", state!, { session_id: this.sessionId, display_id: "primary" });
        const id = randomUUID(); const context = this.overview(state!, id, true, images);
        if (entry) { if (!context) throw new Error("CUA returned invalid selected-display raster geometry."); entry.detail = context; }
        else this.displayOverview = context;
        const highlight = await notifyHighlight(this.observer, context ? { display: entry?.native.display_id ?? "primary",
          bounds: entry?.native.bounds ?? { x: 0, y: 0, width: context.originalWidth, height: context.originalHeight },
          ...(images[0]?.type === "image" ? { image: images[0] } : {}) } : undefined, signal);
        if (this.getMode() !== "yolo") return denied();
        if (signal?.aborted) { retire(); return this.result("refused", "Cancelled during display frame; pixels withheld."); }
        return this.result("observed", `Native display ${entry?.native.display_id ?? "primary"} overview: ${state!.screenshot_width}x${state!.screenshot_height} local pixels. Read only; zoom uses this image_id. Paint freshness is unverified.`,
          state, { source, ...(highlight ? { highlight } : {}), agent_image: { ...this.imageMetadata(images, state!, settings), image_id: id } }, images);
      } catch (error) {
        if (this.getMode() !== "yolo") return denied();
        if (signal?.aborted) { retire(); return this.result("refused", "Cancelled during display capture; diagnostics withheld."); }
        return this.failed(error, false);
      }
    });
  }

  private async displayDetail(request: DetailRequest, signal?: AbortSignal, entry?: DisplayEntry): Promise<ComputerResult> {
    const overview = entry ? entry.detail : this.displayOverview;
    let cap: number;
    try { cap = validateDetail(request, overview, false); }
    catch (error) { return this.result("refused", (error as Error).message); }
    if (entry) entry.detail = undefined; else this.displayOverview = undefined;
    try {
      if (this.getMode() !== "yolo" || signal?.aborted) return this.result("refused", "Display detail permission or request cancelled.");
      const native = await this.readDisplay(entry, 0, signal);
      if (this.getMode() !== "yolo" || signal?.aborted) return this.result("refused", "Display detail revoked or cancelled; pixels withheld.");
      const state = native.structuredContent;
      const images = native.content.filter((block): block is Extract<Content, { type: "image" }> => block.type === "image");
      if (entry && (native.isError || state?.status === "refused" || state?.effect === "refused")) this.displayEntries.delete(entry.ref);
      if (native.isError || state?.status === "refused" || state?.effect === "refused" || !this.displayStateMatches(state, entry) ||
        images.length !== 1 || state.screenshot_width !== overview!.originalWidth || state.screenshot_height !== overview!.originalHeight ||
        this.geometry(state, true) !== overview!.geometry)
        return this.result("refused", "Fresh display geometry or capture is incompatible. Take a new overview.");
      const capturedAt = new Date().toISOString();
      const region = nativeRegion(request, overview!);
      const detail = await cropDisplay(images[0], overview!, region, cap);
      if (this.getMode() !== "yolo" || signal?.aborted) return this.result("refused", "Display detail revoked or cancelled during processing; pixels withheld.");
      const output = { screenshot_width: detail.width, screenshot_height: detail.height, display: entry ? state!.display : "primary", topology_id: entry?.topology };
      const source = entry ? selectedDisplaySource({ ...state, ...output }, this.sessionId, entry.ref) : this.source("display", output, { session_id: this.sessionId, display_id: "primary" });
      source.captured_at = capturedAt;
      source.image.scope = entry ? "selected_display_region" : "primary_display_region"; source.image.coordinate_space = "detail_image_pixels"; source.capabilities.detail = "unsupported";
      source.image.original_dimensions = { width: overview!.originalWidth, height: overview!.originalHeight };
      source.image.origin = { x: region.left, y: region.top, relative_to: entry ? "display" : "primary_display" };
      const configuration = { maxLongEdge: request.maxLongEdge ?? 1200, scale: request.scale ?? 1, kernel: "lanczos3" as const, version: 0, route: "cua_native" as const };
      const agent_image = { ...this.imageMetadata([detail.image], output, configuration), route: "cua_fresh_display_crop",
        coordinate_space: "detail_image_pixels", configuration: { ...configuration, route: "adapter_crop_resize" },
        region: { overview: { image_id: overview!.id, dimensions: { width: overview!.width, height: overview!.height }, x: request.x, y: request.y, width: request.width, height: request.height }, native: region },
        native: { resolution: "full", intermediate_pixels: "withheld" }, input: "none" };
      const bounds = entry?.native.bounds ?? { x: 0, y: 0, width: overview!.originalWidth, height: overview!.originalHeight };
      const highlight = await notifyHighlight(this.observer, { display: entry?.native.display_id ?? "primary", bounds,
        mark: { kind: "detail", region: { x: bounds.x + region.left, y: bounds.y + region.top, width: region.width, height: region.height } }, image: detail.image }, signal);
      if (this.getMode() !== "yolo" || signal?.aborted) return this.result("refused", "Display detail revoked or cancelled during frame; pixels withheld.");
      return this.result("observed", `Fresh display ${entry?.native.display_id ?? "primary"} crop: native-local ${region.left},${region.top} ${region.width}x${region.height}, output ${detail.width}x${detail.height}. Read only; no nested crop or input. Paint freshness is unverified.`,
        undefined, { source, agent_image, ...(highlight ? { highlight } : {}) }, [detail.image]);
    } catch { return this.result("refused", "Display detail failed; private intermediate pixels and diagnostics withheld. Take a new overview."); }
  }

  private async windowDetail(entry: WindowEntry, request: DetailRequest, signal?: AbortSignal): Promise<ComputerResult> {
    const overview = entry.detail;
    try { validateDetail(request, overview, true); }
    catch (error) { return this.result("refused", (error as Error).message); }
    entry.detail = undefined; entry.observation = undefined;
    try {
      if (!this.allowed(entry) || signal?.aborted || !await this.current(entry.ref, signal) || !this.allowed(entry) || signal?.aborted)
        return this.result("refused", "Window closed, detail revoked or cancelled. Observe again.");
      const native = await this.driver.call("zoom", { ...entry.target, x1: request.x, y1: request.y, x2: request.x + request.width, y2: request.y + request.height }, signal);
      if (!this.allowed(entry) || signal?.aborted || !await this.current(entry.ref, signal) || !this.allowed(entry) || signal?.aborted)
        return this.result("refused", "Window closed, detail revoked or cancelled; pixels withheld.");
      const state = native.structuredContent;
      const images = native.content.filter(block => block.type === "image" && block.mimeType === "image/jpeg");
      if (native.isError || state?.status === "refused" || state?.effect === "refused" || !state || images.length !== 1 ||
        !Number.isSafeInteger(state.width) || !Number.isSafeInteger(state.height) || (state.width as number) <= 0 || (state.width as number) > 500 || (state.height as number) <= 0)
        return this.result("refused", "CUA refused or returned invalid window zoom.");
      const source = this.source("window", state, { session_id: this.sessionId, window_ref: entry.ref, ...entry.target }, true);
      source.capabilities.input = "none"; source.semantic_scope = "none";
      const image = images[0] as Extract<Content, { type: "image" }>; const bytes = Buffer.from(image.data, "base64");
      const windows = await this.list(signal);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Window detail revoked or cancelled; pixels withheld.");
      const window = windows.find(w => w.pid === entry.target.pid && w.window_id === entry.target.window_id);
      if (!window) return this.result("refused", "Window closed during detail; pixels withheld.");
      const bounds = screenRect(window.bounds);
      Object.assign(source, this.domained(sourceIdentity(source.identity, null, window.virtual_desktop, this.platform)));
      const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,
        { kind: "detail", region: bounds ? windowRegion(request, overview!.windowMapping, bounds) : undefined }, image);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Window detail revoked or cancelled during frame; pixels withheld.");
      return this.result("observed", "Fresh native window zoom. CUA owns JPEG padding and geometry. Read only; take a new overview before input or another detail.",
        undefined, { ...this.addressed(entry, window.virtual_desktop), ...(highlight ? { highlight } : {}), source, agent_image: { route: "cua_fresh_window_zoom", coordinate_space: "zoom_image_pixels",
          configuration: { padding: "20% each side", maxWidth: 500, route: "cua_native" }, native: state,
          region: { overview: { image_id: overview!.id, dimensions: { width: overview!.width, height: overview!.height }, x: request.x, y: request.y, width: request.width, height: request.height } },
          output: { width: state.width, height: state.height, bytes: bytes.length, mimeType: image.mimeType, hash: createHash("sha256").update(bytes).digest("hex") }, input: "none" } }, images);
    } catch { return this.result("refused", "Window detail failed; pixels and diagnostics withheld. Observe again."); }
  }
  private async capture(entry: WindowEntry, signal?: AbortSignal, query?: string, options: UiCaptureOptions = {}, quiet = false, deferPresentation = false): Promise<ComputerResult> {
    validateUiCapture(options);
    // A native capture keeps the page's DOM snapshot: reading changes nothing, so both stay current until input.
    entry.windowState = undefined;
    const granularity = options.granularity ?? "full";
    const hasUi = granularity !== "image"; const hasImage = granularity !== "ui";
    if (!hasUi && query !== undefined) return this.result("refused", "Image-only observation cannot query UI labels.");
    if (!this.allowed(entry)) return this.denied(entry.ref);
    entry.observation = undefined; entry.detail = undefined;
    const current = await this.current(entry.ref, signal);
    if (!this.allowed(entry)) return this.denied(entry.ref);
    if (!current) return this.result("refused", "Window closed or its native handle was reused. Discover again.");
    const settings = structuredClone({ ...this.overviewSettings(), ...(options.maxLongEdge !== undefined ? { maxLongEdge: options.maxLongEdge } : {}),
      ...(options.scale !== undefined ? { scale: options.scale } : {}), ...(options.hints !== undefined ? { hints: options.hints } : {}), ...(options.guides ? { guides: options.guides } : {}) });
    const native = await this.driver.call("get_window_state", {
      ...entry.target,
      ...(hasUi ? { max_elements: options.max_elements ?? 200, max_depth: options.max_depth ?? 15, timeout_ms: 5000, ...(query ? { query } : {}) }
        : { include_accessibility_tree: false }),
      ...(hasImage ? { max_image_dimension: nativeImageCap(settings) } : { include_screenshot: false }),
    }, signal);
    if (!this.allowed(entry)) return this.denied(entry.ref);
    if (signal?.aborted) return this.result("refused", "Cancelled during observation; captured data withheld.");
    if (native.isError) return this.result("refused", native.structuredContent?.code === "human_stop"
      ? native.content.filter(c => c.type === "text").map(c => c.text).join("\n") : "Driver refused observation.", native.structuredContent, this.addressed(entry, native.structuredContent?.virtual_desktop));
    if (!native.structuredContent) throw new Error("Driver returned no structured window state.");
    const state = { ...native.structuredContent };
    if (hasImage && typeof state.screenshot_error === "string" && state.screenshot_error.trim()) {
      if (!await this.current(entry.ref, signal)) return this.result("refused", "Window changed during observation. Discover again.");
      if (!this.allowed(entry)) return this.denied(entry.ref);
      if (signal?.aborted) return this.result("refused", "Cancelled during observation; diagnostics withheld.");
      const screenshot_error = state.screenshot_error.trim().slice(0, 2000);
      return this.result("refused", `Window screenshot unavailable. Native reason: ${screenshot_error}`,
        { screenshot_error }, this.addressed(entry, state.virtual_desktop));
    }
    // Native screenshot-only reads may preserve their old AX snapshot. Never expose it as current.
    if (!hasUi) for (const key of ["elements", "tree_markdown", "snapshot_id", "invalidated_snapshot_ids", "elements_complete", "truncated", "truncation_reason", "total_element_count", "element_count", "focused_element"]) delete state[key];
    // Tree-only native receipts can describe an older pixel frame. It gives this observation no image authority.
    if (!hasImage) for (const key of Object.keys(state)) if (key.startsWith("screenshot_") || ["capture_id", "agent_overlay_capture", "capture_backend"].includes(key)) delete state[key];
    if (hasUi && !Array.isArray(state.elements)) throw new Error("Driver returned no structured UI elements.");
    if (hasUi && !hasImage) state.elements = (state.elements as Record<string, unknown>[]).map(element => {
      const { screenshot_frame: _oldPixels, ...current } = element; return current;
    });
    const images = hasImage ? native.content.filter(block => block.type === "image") : [];
    if (hasImage && !images.length) throw new Error("Driver returned no grounding screenshot.");
    if (!this.allowed(entry)) return this.denied(entry.ref);
    if (!await this.current(entry.ref, signal)) return this.result("refused", "Window changed during observation. Discover again.");
    if (!this.allowed(entry)) return this.denied(entry.ref);
    const id = randomUUID();
    entry.detail = hasImage ? this.overview(state, id, false, images) : undefined;
    if (granularity === "image" && !entry.detail) throw Error("Driver returned no proven image-only pixel geometry.");
    const ui = hasUi ? createUiResource(state, entry.ref, id, ++this.uiSerial) : undefined;
    if (ui && !hasImage) for (const element of ui.elements) {
      delete element.bounds.window_image; element.hint_display = "off";
    }
    const elements = hasUi ? state.elements as { element_token?: unknown; frame?: unknown; label?: unknown; value?: unknown }[] : [];
    // The UI resource cuts each text at 1024 characters; keep longer texts privately for element:text reads.
    const texts = new Map<string, string>();
    if (ui) elements.filter(element => typeof element.element_token === "string").forEach((element, index) => {
      const full = [element.label, element.value].filter((item): item is string => typeof item === "string" && item.length > 1024);
      if (full.length && ui.elements[index]) texts.set(ui.elements[index].hint, full.join("\n").slice(0, ELEMENT_TEXT_LIMIT));
    });
    entry.visualBounds = screenRect(state.window_bounds);
    // Only an image the agent is shown sets the space of its points; successors and other quiet captures keep it.
    if (hasImage && !quiet && typeof state.screenshot_width === "number" && typeof state.screenshot_height === "number") entry.seen = { width: state.screenshot_width, height: state.screenshot_height };
    entry.observation = { id, ui, settings, ...(texts.size ? { texts } : {}),
      width: hasImage && typeof state.screenshot_width === "number" ? state.screenshot_width : undefined,
      height: hasImage && typeof state.screenshot_height === "number" ? state.screenshot_height : undefined,
      tokens: new Set(elements.map(element => element.element_token).filter((token): token is string => typeof token === "string")),
      captureId: hasImage && typeof state.capture_id === "string" ? state.capture_id : undefined,
      bounds: screenRect(state.window_bounds),
      frames: new Map(elements.flatMap(element => {
        const bounds = screenRect(state.window_bounds); const region = bounds && elementRegion(element.frame, bounds);
        return typeof element.element_token === "string" && region ? [[element.element_token, region] as [string, ScreenRect]] : [];
      })) };
    if (signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Cancelled during observation; captured data withheld."); }
    const source = this.source("window", state, { session_id: this.sessionId, window_ref: entry.ref, ...entry.target });
    if (!hasUi) source.semantic_scope = "none";
    if (!hasImage) source.capabilities.detail = "unsupported";
    // Publish only the final operation image, after annotation. Quiet verification samples stay silent.
    let highlight: HighlightResult | undefined;
    if (!this.allowed(entry)) { entry.observation = undefined; entry.detail = undefined; return this.denied(entry.ref); }
    if (signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Cancelled during window frame; captured data withheld."); }
    const observed = entry.observation!;
    observed.source = source;
    const extra = { ...this.addressed(entry, state.virtual_desktop), observation_id: id,
      source, granularity, captured: { image: hasImage, ui: hasUi } };
    if (!hasImage) {
      highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, { kind: "inspect" });
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; data withheld.");
      return this.result("observed", `Fresh UI-only window observation: ${ui!.elements.length} UI elements. No pixels or image detail; current hints/tokens only.`, state, { ...extra, ...(highlight ? { highlight } : {}), ui: structuredClone(ui!) });
    }
    const raw = images[0];
    if (raw?.type === "image" && Buffer.byteLength(raw.data, "base64") <= 12 * 1024 * 1024) {
      observed.rawImage = raw;
      entry.frame = { observation_id: id, png: Buffer.from(raw.data, "base64"), size: { maxLongEdge: settings.maxLongEdge, scale: settings.scale } };
      const active = new Set([...this.entries.values()].map(e => e.observation));
      this.uiImages = [...this.uiImages.filter(old => active.has(old)), observed];
      for (const old of this.uiImages.splice(0, Math.max(0, this.uiImages.length - 3))) old.rawImage = undefined;
    }
    observed.mapping = entry.detail?.windowMapping;
    observed.actionMark = this.actionMark(entry.lastAction, observed.mapping);
    const annotationUi = ui ?? createUiResource({ elements: [] }, entry.ref, id, 0);
    const rendered = await annotateUi(raw, annotationUi, observed.width, observed.height, hasUi && (options.hints ?? settings.hints ?? true), undefined, annotationUi.elements,
      { options: settings.guides, action: observed.actionMark });
    if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled during annotation; pixels withheld."); }
    highlight = quiet || deferPresentation ? undefined : await notifyWindowObserver(this.observer, entry.target, entry.identity, state.window_bounds, signal, undefined, rendered.image);
    if (!this.allowed(entry) || signal?.aborted) { entry.observation = undefined; entry.detail = undefined; return this.result("refused", "Observation revoked or cancelled; pixels withheld."); }
    const metadata = this.imageMetadata([rendered.image], state, settings);
    const message = hasUi
      ? `Fresh full window observation: ${ui!.elements.length} UI elements; ${rendered.annotation.labels.length} image hints. Structure ${ui!.complete === true ? "complete" : ui!.complete === false ? "partial" : "completeness unknown"}; paint freshness is unverified. Hints belong to this observation only.`
      : "Fresh image-only window observation. Accessibility was not collected; no current UI hints/tokens. Pixel input uses this window image; paint freshness is unverified.";
    return this.result("observed", message, state, { ...extra, ...(highlight ? { highlight } : {}), ...(ui ? { ui: structuredClone(ui) } : {}),
      agent_image: { ...metadata, ...(rendered.image !== raw ? { route: "cua_native_ui_hints" } : {}), annotation: rendered.annotation, image_id: id } }, [rendered.image]);
  }

  /** Resolve a permitted window bound to a browser this session opened through Playwright. */
  private async webEntry(ref: string, signal?: AbortSignal): Promise<WindowEntry | ComputerResult> {
    let entry: WindowEntry | undefined;
    try { entry = await this.current(ref, signal); }
    catch (error) { return this.result("refused", `Window lifetime check failed: ${error instanceof Error ? error.message : String(error)}`); }
    if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window. Discover again.");
    if (!entry.web) return this.result("refused", "DOM is available in browsers opened or attached with computer_open. A Safe agent profile needs an approved connection first. Use image/ui/full Reads for other windows.",
      undefined, this.addressed(entry));
    return entry;
  }

  private async webFailure(entry: WindowEntry, error: unknown, acting = false, signal?: AbortSignal, action?: string, typedText?: string): Promise<ComputerResult> {
    const reason = error instanceof Error ? error.message : String(error);
    const uncertain = acting && error instanceof WebError && (error.timeout || error.transport);
    if (acting) await this.activity(entry, { kind: "action" }, signal, undefined, uncertain ? "unknown" : "refused", action, typedText);
    return this.result(uncertain ? "unknown" : "refused", uncertain
      ? `The action may have reached the page. ${reason} Read the window with dom before deciding; never replay it blindly.`
      : `Playwright refused: ${reason}`, undefined, this.addressed(entry));
  }

  /** Store a page snapshot as the window's only current DOM observation. */
  private webObservation(entry: WindowEntry, page: WebPage): { id: string; page: WebDom } {
    const truncated = page.snapshot.length > WEB_SNAPSHOT_LIMIT;
    entry.dom = { id: randomUUID(), page: { ...page, truncated,
      snapshot: truncated ? page.snapshot.slice(0, WEB_SNAPSHOT_LIMIT) + "\n# Snapshot truncated; narrow it with dom:{\"target\":...,\"depth\":...}." : page.snapshot } };
    return entry.dom;
  }

  /** Fresh Playwright aria snapshot ("ai" mode, iframes included) of the page shown in an owned browser window. */
  observeDom(ref: string, options: WebSnapshotOptions = {}, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      // A DOM snapshot keeps the window's native capture: reading changes nothing, so both stay current until input.
      entry.dom = undefined; entry.windowState = undefined;
      try {
        const observed = this.webObservation(entry, await entry.web!.page("snapshot", options, signal));
        await this.activity(entry, { kind: "inspect" }, signal);
        if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; DOM withheld.");
        return this.result("observed", "Fresh Playwright aria snapshot. Use [ref=...] values as target.ref while this stays the latest snapshot of the window.",
          undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
      } catch (error) { return this.webFailure(entry, error); }
    });
  }

  /** Return the window's current DOM observation without querying the page. */
  checkDom(ref: string, id: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      if (entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));
      return this.result("observed", "Cached Playwright aria snapshot; the page may have changed since.", undefined,
        { ...this.addressed(entry), observation_id: id, dom: entry.dom.page });
    });
  }

  /** Filter the current snapshot's lines without querying the page. */
  searchDom(ref: string, id: string, query: string, limit = 50, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      if (entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));
      const needle = query.toLowerCase();
      const lines = entry.dom.page.snapshot.split("\n").filter(line => line.toLowerCase().includes(needle));
      return this.result("observed", `${lines.length} snapshot lines match. Search covers only this snapshot${entry.dom.page.truncated ? ", which was truncated" : ""}.`, undefined,
        { ...this.addressed(entry), observation_id: id, dom: { ...entry.dom.page, snapshot: lines.slice(0, limit).join("\n"), matches: lines.length } });
    });
  }

  /** Playwright screenshot of the page viewport, one element, or the viewport with that element highlighted. */
  webImage(ref: string, id: string | undefined, kind: "page" | "crop" | "highlight", target?: WebTarget, signal?: AbortSignal, css = false): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      // Without an observation it shoots the live page: a locator, or a ref of the page's latest snapshot.
      if (id !== undefined && entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again.", undefined, this.addressed(entry));
      try {
        const shot = await entry.web!.request<{ url: string; title: string; png: string; box?: unknown; derive?: Derive }>("screenshot",
          { ...(target ? { target } : {}), highlight: kind === "highlight", ...(css ? { css } : {}) }, signal);
        const plain = Buffer.from(shot.png, "base64");
        const bytes = shot.derive ? await derivedPng(plain, shot.derive) : plain;
        if (bytes.length < 24 || !bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) throw Error("Playwright did not return a PNG.");
        const image: Content = { type: "image", data: bytes.toString("base64"), mimeType: "image/png" };
        // CSS boxes do not prove a Windows screen transform. Associate the preview with its window only.
        await this.activity(entry, { kind: "inspect" }, signal, image);
        if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation revoked or cancelled; pixels withheld.");
        return this.result("observed", `Playwright ${kind === "page" ? "page viewport" : kind === "crop" ? "element" : "highlighted viewport"} screenshot. These are page pixels, not native window coordinates; they grant no input.`,
          { url: shot.url, title: shot.title, ...(shot.box ? { box_css: shot.box } : {}) },
          { ...this.addressed(entry), ...(id !== undefined ? { observation_id: id } : {}), agent_image: { route: "playwright_screenshot", kind, coordinate_space: "page_image_pixels", input_authority: false,
            output: { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20), bytes: bytes.length, mimeType: "image/png",
              hash: createHash("sha256").update(bytes).digest("hex") } } },
          [image]);
      } catch (error) { return this.webFailure(entry, error); }
    });
  }

  /** Run one Playwright action against the current DOM observation and return the next snapshot. */
  actDom(ref: string, id: string, action: WebAction, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      if (entry.dom?.id !== id) return this.result("refused", "This DOM observation is no longer current. Read the window with dom again; nothing was sent.", undefined, this.addressed(entry));
      entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
      try {
        const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
        await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));
        if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
        return this.result("delivered", `Playwright performed ${action.action}${action.target && action.action !== "evaluate" ? " after its actionability checks" : ""}. Check the fresh snapshot below for the application's response.`,
          undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
      } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }
    });
  }

  /**
   * Perform one page action in the active tab of a browser window without a DOM observation: navigation, tabs,
   * dialogs, files and page code. Refs in a target resolve against the page's latest snapshot. The result carries
   * the page's next snapshot as a new observation.
   */
  actPage(ref: string, action: WebAction, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.webEntry(ref, signal);
      if (!("ref" in entry)) return entry;
      if (!this.allowed(entry)) return this.denied(ref);
      entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
      try {
        const observed = this.webObservation(entry, await entry.web!.page("act", action, signal));
        await this.activity(entry, { kind: "action" }, signal, undefined, "delivered", presentationAction(action), presentationDetail(action));
        if (!this.allowed(entry) || signal?.aborted) return this.result("unknown", "Action completed, but access ended or cancellation arrived; DOM withheld. Do not replay blindly.");
        return this.result("delivered", `Playwright performed ${action.action}. Check the fresh snapshot below for the page's response.`,
          undefined, { ...this.addressed(entry), observation_id: observed.id, dom: observed.page });
      } catch (error) { return this.webFailure(entry, error, true, signal, presentationAction(action), presentationDetail(action)); }
    });
  }

  /** Whether this window is a browser this session opened or attached, whose pages Playwright drives. */
  isBrowserWindow(ref: string): boolean {
    return this.entries.get(ref)?.web !== undefined;
  }

  /** Validate a wait_for/verify command for this window: owned browsers take Playwright targets and page checks. */
  waitSpec(ref: string, command: Record<string, unknown>): WaitSpec {
    return validateWait(command, this.entries.get(ref)?.web !== undefined);
  }

  /**
   * Sample conditions until all hold for `stable` consecutive samples, or until `timeout_ms` passes.
   * Native windows take a fresh bounded UI walk per sample; owned browsers ask Playwright. Nothing is
   * dispatched. The last sample becomes the window's current observation with fresh input authority.
   */
  waitFor(ref: string, spec: WaitSpec, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const started = performance.now();
      let entry: WindowEntry | undefined;
      try { entry = await this.current(ref, signal); }
      catch (error) { return this.result("refused", `Window lifetime check failed: ${error instanceof Error ? error.message : String(error)}`); }
      if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window. Discover again.");
      if (!this.allowed(entry)) return this.denied(entry.ref);
      let samples = 0, stable = 0, results: ConditionResult[] = [], last: ComputerResult | undefined;
      entry.dom = undefined; entry.windowState = undefined;
      const finish = async (verdict: WaitResult["verdict"], timedOut: boolean, final: ComputerResult | undefined, why?: string): Promise<ComputerResult> => {
        if (!this.allowed(entry!)) return this.denied(entry!.ref);
        await this.activity(entry!, { kind: "verify" }, signal, undefined, verdict === "satisfied" ? "verified" : "unknown");
        if (!this.allowed(entry!)) return this.denied(entry!.ref);
        if (signal?.aborted) {
          verdict = "unknown";
          why = "Cancelled; the last sample does not decide the conditions.";
        }
        const verification = { verdict, conditions: results, samples, elapsed_ms: Math.round(performance.now() - started), timed_out: timedOut,
          ...(why ? { reason: why } : {}), spec };
        const message = `Verification ${verdict} after ${samples} sample(s) in ${verification.elapsed_ms} ms${timedOut ? " (timed out)" : ""}.${why ? ` ${why}` : ""}`;
        const d = final?.details;
        return this.result("observed", message, undefined, { ...this.addressed(entry!), verification,
          ...(d?.observation_id ? { observation_id: d.observation_id } : {}), ...(d?.ui ? { ui: d.ui } : {}), ...(d?.dom ? { dom: d.dom } : {}) });
      };
      while (true) {
        if (signal?.aborted) return finish("unknown", false, last, "Cancelled; the last sample does not decide the conditions.");
        if (!this.allowed(entry)) return this.denied(entry.ref);
        if (entry.web) {
          try { results = await entry.web.request<ConditionResult[]>("check", { conditions: spec.conditions }, signal); }
          catch (error) { results = []; return finish("unknown", false, undefined, `Playwright could not sample the page: ${error instanceof Error ? error.message : String(error)}`); }
        } else {
          const sample = await this.capture(entry, signal, undefined, { granularity: "ui" }, true);
          if (sample.details.status !== "observed" || !sample.details.ui) {
            results = spec.conditions.map((_, index) => ({ index, verdict: "unknown" as const, reason: "No current UI sample." }));
            const why = sample.content.filter(block => block.type === "text").map(block => block.text).join("\n").split("\n{", 1)[0];
            return finish("unknown", false, undefined, why);
          }
          last = sample;
          results = evaluateNative(sample.details.ui, spec.conditions, entry.observation?.texts);
        }
        samples++;
        const verdict = combine(results);
        stable = verdict === "satisfied" ? stable + 1 : 0;
        if (stable >= spec.stable) break;
        if (performance.now() - started + spec.interval_ms > spec.timeout_ms) {
          if (entry.web) last = await this.webFinal(entry, signal);
          return finish(verdict, spec.timeout_ms > 0, last);
        }
        try { await delay(spec.interval_ms, undefined, { signal }); } catch { /* checked at the top of the loop */ }
      }
      if (entry.web) last = await this.webFinal(entry, signal);
      return finish("satisfied", false, last);
    });
  }

  /**
   * The complete text of one element in the current native observation, without another walk. Texts longer
   * than the walk's 1024-character cut come from the same capture, up to 200,000 characters.
   */
  elementText(ref: string, id: string, hint: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Element text read cancelled.");
      const entry = await this.current(ref, signal);
      if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
      if (!this.allowed(entry)) return this.denied(entry.ref);
      const observed = entry.observation;
      if (!observed || observed.id !== id || !observed.ui) return this.result("refused", "This observation is no longer current or has no UI. Read the window again.", undefined, this.addressed(entry));
      let element;
      try { element = resolveUi(observed.ui, hint); }
      catch (error) { return this.result("refused", (error as Error).message, undefined, this.addressed(entry)); }
      const text = observed.texts?.get(hint) ?? [element.name, element.value].filter(item => item).join("\n");
      const cut = text.length >= ELEMENT_TEXT_LIMIT;
      const region = observed.frames.get(element.element_token);
      await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Element text access revoked or cancelled.");
      return this.result("observed", `Element ${hint} text from this observation: ${text.length} characters${cut ? ", cut at the 200,000-character limit" : ""}. Page it with offset/limit.`,
        undefined, { ...this.addressed(entry), observation_id: id, element_text: { hint, role: element.role, name: element.name, text, cut } });
    });
  }

  /** The window's latest unannotated capture with pixels, the baseline for comparing its pixels; undefined when none is kept. */
  windowFrame(ref: string): WindowFrame | undefined { return this.entries.get(ref)?.frame; }

  /**
   * A full-resolution capture of a window for input in one of its viewports: it becomes the window's current
   * observation. Quiet, like a successor, so points of the window's latest image keep their meaning.
   */
  observeFull(ref: string, signal?: AbortSignal, granularity: "image" | "full" = "full"): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
      try {
        const entry = await this.current(ref, signal);
        if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
        if (!this.allowed(entry)) return this.denied(entry.ref);
        return await this.capture(entry, signal, undefined, { granularity, maxLongEdge: FULL_RESOLUTION, scale: 1, hints: false }, true);
      } catch (error) { return this.failed(error, false); }
    });
  }

  /**
   * A window's pixels at full resolution for a viewport, without UI. Only looks: the window's observations, its pixel
   * baseline and the size of the image the agent last saw stay as they are.
   */
  windowPixels(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before capture.");
      try {
        const entry = await this.current(ref, signal);
        if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
        if (!this.allowed(entry)) return this.denied(entry.ref);
        const native = await this.driver.call("get_window_state", { ...entry.target, include_accessibility_tree: false, max_image_dimension: FULL_RESOLUTION }, signal);
        if (!this.allowed(entry)) return this.denied(entry.ref);
        if (signal?.aborted) return this.result("refused", "Cancelled during capture; pixels withheld.");
        const state = native.structuredContent ?? {};
        if (native.isError || typeof state.screenshot_error === "string" && state.screenshot_error.trim())
          return this.result("refused", `Window screenshot unavailable${typeof state.screenshot_error === "string" ? `. Native reason: ${state.screenshot_error.trim().slice(0, 2000)}` : "."}`, undefined, this.addressed(entry, state.virtual_desktop));
        const images = native.content.filter(block => block.type === "image");
        const width = state.screenshot_width, height = state.screenshot_height;
        if (images.length !== 1 || !Number.isSafeInteger(width) || !Number.isSafeInteger(height) || (width as number) <= 0 || (height as number) <= 0)
          throw new Error("Driver returned no window image.");
        if (!await this.current(entry.ref, signal) || !this.allowed(entry)) return this.result("refused", "Window changed during capture. Discover again.");
        entry.visualBounds = screenRect(state.window_bounds);
        return this.result("observed", `Full-resolution window pixels: ${width}x${height}.`, { window_bounds: entry.visualBounds, screenshot_width: width, screenshot_height: height },
          this.addressed(entry, state.virtual_desktop), images);
      } catch (error) { return this.failed(error, false); }
    });
  }

  /** Present a post-processed observation without recapturing or changing its input authority. */
  presentObservation(ref: string, result: ComputerResult, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.current(ref, signal);
      if (!entry || !this.allowed(entry) || signal?.aborted || entry.observation?.id !== result.details.observation_id)
        return this.result("refused", "Observation source revoked or cancelled; data withheld.");
      const image = result.content.filter(block => block.type === "image").at(-1);
      const highlight = await this.activity(entry, { kind: "inspect" }, signal, image);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Observation source revoked or cancelled; data withheld.");
      return { ...result, details: { ...result.details, ...(highlight ? { highlight } : {}) } };
    });
  }
  /** Present a final viewport image without changing observation/input authority. */
  presentViewport(ref: string, image: Content, captured: ComputerResult, rect: PixelBox, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = await this.current(ref, signal);
      if (!entry || !this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");
      const bounds = screenRect(captured.details.state?.window_bounds);
      const observed = entry.observation;
      const mapping = observed?.mapping;
      const sameBounds = bounds && observed?.bounds && (["x", "y", "width", "height"] as const).every(key => bounds[key] === observed.bounds![key]);
      // The earlier AX witnesses must prove the same full-resolution width and native source.
      const region = sameBounds && mapping && observed?.width && observed.height && mapping.scale * observed.width === captured.details.state?.screenshot_width && mapping.scale * observed.height === captured.details.state?.screenshot_height
        ? windowRegion({ image_id: "viewport", ...rect }, { ...mapping, scale: 1 }, bounds) : undefined;
      const highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, bounds, signal,
        region ? { kind: "detail", region } : { kind: "inspect" }, image);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Viewport source revoked or cancelled; pixels withheld.");
      return this.result("observed", "", undefined, highlight ? { highlight } : {});
    });
  }
  /**
   * A fresh observation without the human frame, for successors: UI-only by default, or with
   * pixels at a given overview size so they compare with an earlier capture.
   */
  observeQuiet(ref: string, signal?: AbortSignal, options: UiCaptureOptions = { granularity: "ui" }): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
      try {
        const entry = await this.current(ref, signal);
        if (!entry) return this.entries.has(ref) ? this.denied(ref) : this.result("refused", "Unknown or closed window_ref. Discover again.");
        if (!this.allowed(entry)) return this.denied(entry.ref);
        return await this.capture(entry, signal, undefined, options, true);
      } catch (error) { return this.failed(error, false); }
    });
  }

  /** A fresh DOM observation after browser verification, so later actions use current refs. */
  private async webFinal(entry: WindowEntry, signal?: AbortSignal): Promise<ComputerResult | undefined> {
    try {
      const observed = this.webObservation(entry, await entry.web!.page("snapshot", {}, signal));
      return this.result("observed", "", undefined, { observation_id: observed.id, dom: observed.page });
    } catch { return undefined; }
  }

  /** Acquire fresh one-use native state without pixels, a UI walk or ordinary-input authority. */
  observeWindowState(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      try {
        const known = this.entries.get(ref);
        if (known && !this.allowed(known)) return this.denied(ref);
        // Sampling state only looks: the window's observations stay current; restore and management retire them.
        if (known) known.windowState = undefined;
        if (signal?.aborted) return this.result("refused", "Cancelled before window state acquisition.");
        const entry = await this.current(ref, signal);
        if (!entry) return this.result("refused", "Unknown, closed or reused window. Discover again.");
        if (!this.driver.windowRestore && !this.driver.windowManagement) return this.result("refused", "Window state needs an explicitly selected qualified native runtime. Ask the user to manage the window manually; no keys or clicks as a workaround.", undefined, this.addressed(entry));
        const kind = this.driver.windowManagement ? "management" : "restore";
        const native = await this.driver.call(kind === "management" ? "get_window_control_state" : "get_window_status", { ...entry.target }, signal);
        if (!this.allowed(entry)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Cancelled during window state acquisition; no authority issued.");
        if (!await this.current(ref, signal)) return this.result("refused", "Window closed or was reused during state acquisition.");
        if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "Window state revoked or cancelled; no authority issued.");
        const state = native.structuredContent;
        if (native.isError || !state || !(kind === "management" ? validWindowControlState(state, entry.target) : validWindowStatus(state, entry.target)) ||
          native.content.some(block => block.type === "image"))
          return this.result("refused", "Native window state refused or its exact-source receipt is invalid. No input fallback.", state, this.addressed(entry));
        const id = randomUUID();
        entry.windowState = { id, nativeId: state.state_id as string, kind };
        return this.result("observed", kind === "management"
          ? "Fresh native window state. computer_window changes it; desktop-frame units are not pointer coordinates."
          : "Fresh native window state. computer_window restore restores and activates this window.", state,
          { ...this.addressed(entry, kind === "management" ? (state.state as Record<string, unknown>).virtual_desktop : undefined), window_state_id: id, window_state_kind: kind });
      } catch (error) { return this.failed(error, false); }
    });
  }

  /**
   * Ask a window to close, like its close button: a browser this session opened closes with its disposable profile,
   * an attached one is detached first, other windows get WM_CLOSE. The app may keep the window open, for example to
   * ask about unsaved work; the result says whether it closed.
   */
  closeWindow(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      try {
        const known = this.entries.get(ref);
        if (known && !this.allowed(known)) return this.denied(ref);
        const entry = await this.current(ref, signal);
        if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
        if (!this.allowed(entry)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Cancelled before closing the window.");
        const key = `${entry.target.pid}:${entry.target.window_id}`, browser = this.browsers.get(key);
        if (browser) {
          this.browsers.delete(key);
          entry.web = undefined; entry.dom = undefined;
          await browser.close();
          this.unapprovedAgentBrowsers.delete(browser);
        }
        if (!this.host) {
          if (browser) { this.entries.delete(ref); return this.result("delivered", "The browser this session opened was closed.", undefined, this.addressed(entry)); }
          return this.result("refused", "Closing windows needs the machine's own Windows host, which this environment lacks. Click the window's close button, or Delete it with ?force=true when this session opened it.", undefined, this.addressed(entry));
        }
        const outcome = await this.host.closeWindow(entry.target.pid, entry.target.window_id, 5000);
        if (outcome === "refused") return this.result("refused", "Windows refused the close request; the window may belong to an elevated app. Click its close button instead.", undefined, this.addressed(entry));
        if (outcome === "open") return this.result("delivered", "Close was requested, but the window is still open after 5 s; it may be asking about unsaved work. Read it to see what it shows.", { closed: false }, this.addressed(entry));
        this.entries.delete(ref);
        return this.result("closed", "The window closed; its address expired.", { closed: true }, this.addressed(entry));
      } catch (error) { return this.failed(error, true); }
    });
  }
  /**
   * Force-close the process of a window this session opened, when every window of that process is session-owned.
   * Unsaved work in it is lost; a process that also shows other windows is never killed.
   */
  killOwned(ref: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = this.entries.get(ref);
      if (!entry) return this.result("refused", "Unknown or closed window. Discover again.");
      if (!entry.owned) return this.result("refused", "Only windows this session opened can be force-closed; close other windows through their own UI.");
      if (signal?.aborted) return this.result("refused", "Cancelled before closing the process.");
      try {
        const windows = (await this.state("list_windows", { pid: entry.target.pid }, signal)).windows as WindowTarget[] ?? [];
        const owned = new Set([...this.entries.values()].filter(e => e.owned).map(e => `${e.target.pid}:${e.target.window_id}`));
        const foreign = windows.filter(w => w.pid === entry.target.pid && !owned.has(`${w.pid}:${w.window_id}`));
        if (foreign.length) return this.result("refused", `Process ${entry.target.pid} also shows ${foreign.length} window(s) this session did not open; it was not killed. Close this window through its own UI.`);
        const native = await this.driver.call("kill_app", { pid: entry.target.pid }, signal);
        if (native.isError) return this.result("refused", `The process was not killed: ${native.content.filter(c => c.type === "text").map(c => c.text).join(" ")}`, native.structuredContent);
        for (const [key, other] of this.entries) if (other.target.pid === entry.target.pid) this.entries.delete(key);
        return this.result("delivered", `Process ${entry.target.pid} was force-closed; unsaved work in it is lost and its window addresses expired.`, native.structuredContent);
      } catch (error) { return this.failed(error, true); }
    });
  }

  /** The native target of a known window, read before work that may forget the window. */
  windowTarget(ref: string): WindowTarget | undefined { return this.entries.get(ref)?.target; }

  /** Whether a window still responds, with its process's memory and CPU; undefined when unknown or not measurable here. */
  processHealth(target: WindowTarget | undefined): Promise<ProcessHealth | undefined> {
    return target === undefined || !this.host ? Promise.resolve(undefined) : this.host.processHealth(target.pid, target.window_id).catch(() => undefined);
  }

  /** The unannotated overview pixels of a window observation, for text recognition in its own coordinates. */
  observationImage(ref: string, id: string): Content | undefined {
    const observed = this.entries.get(ref)?.observation;
    return observed?.id === id && !observed.pixelsOnly ? observed.rawImage : undefined;
  }

  /**
   * Icons and text areas that Cua perception (OmniParser icons, PP-OCR text) finds in a current window observation's
   * capture, with boxes in its overview pixels. Needs the perception extension installed on that machine.
   */
  visualRegions(ref: string, id: string, signal?: AbortSignal): Promise<VisualRegion[]> {
    return this.queue(async () => {
      const observed = this.entries.get(ref)?.observation;
      if (observed?.id !== id || !observed.captureId) throw new Error("This observation's native capture is no longer current. Read the window with regions again.");
      const native = await this.driver.call("parse_visual_regions", { capture_id: observed.captureId, options: { max_regions: 300 } }, signal);
      if (native.isError) throw new Error((native.content.filter(c => c.type === "text").map(c => c.text).join(" ") || "parse_visual_regions failed")
        + " The regions view needs the Cua perception extension on this machine; if it is missing, ask the user before installing it "
        + "(about 420 MB) and ask them to run /computer-setup perception.");
      const parsed = native.structuredContent as { capture: { screenshot: { width: number } }; regions: {
        kind: "icon" | "text"; bounds: { x: number; y: number; width: number; height: number }; text?: string | null; label?: string | null; confidence: number }[] };
      const scale = (observed.width ?? parsed.capture.screenshot.width) / parsed.capture.screenshot.width;
      const at = (value: number) => Math.round(value * scale);
      return parsed.regions.map(region => ({ kind: region.kind, x: at(region.bounds.x), y: at(region.bounds.y), width: at(region.bounds.width), height: at(region.bounds.height),
        // OmniParser's detector has one class; its "icon-class-0" label names nothing.
        ...(region.text ? { text: region.text } : {}), ...(region.label && !/^icon-class-\d+$/.test(region.label) ? { label: region.label } : {}), confidence: Math.round(region.confidence * 100) / 100 }));
    });
  }

  /** Recognise text with Windows OCR on this desktop's machine; boxes are in the given image's pixels, a region only reads that part. */
  async recognizeText(image: Content, region?: ScreenRect, signal?: AbortSignal): Promise<TextLine[]> {
    if (!this.host) throw new Error("Text recognition is not available for this desktop.");
    if (image.type !== "image") throw new Error("Text recognition needs an image.");
    if (signal?.aborted) throw new Error("Cancelled before text recognition.");
    return this.host.ocr(Buffer.from(image.data, "base64"), region);
  }

  /** Check cached state authority without acquiring metadata or renewing its one-use native lease. */
  checkWindowState(ref: string, id: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      try {
        const known = this.entries.get(ref);
        if (known && !this.allowed(known)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Cancelled before window state reference check.");
        const entry = await this.current(ref, signal);
        if (!entry || entry.windowState?.id !== id) return this.result("refused", "Retired or closed window state. Read a fresh state.");
        if (!this.allowed(entry)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Window state reference check cancelled.");
        return this.result("observed", "Existing window state; no native acquisition or renewal of one-use authority.", undefined,
          { ...this.addressed(entry), window_state_id: id, window_state_kind: entry.windowState.kind });
      } catch (error) { return this.failed(error, false); }
    });
  }
  /** Consume one admitted window state for explicit restore and activation; never capture, inject input or replay. */
  restoreWindow(ref: string, id: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = this.entries.get(ref), observed = entry?.windowState;
      if (!entry || !observed || observed.id !== id) return this.result("refused", "Retired or missing window state. Read a new state before restore.");
      entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
      if (!this.allowed(entry)) return this.denied(ref);
      if (signal?.aborted) return this.result("refused", "Cancelled before native restore dispatch.");
      let dispatched = false;
      try {
        if (!await this.current(ref, signal)) return this.result("refused", "Window closed or was reused before restore.");
        if (!this.allowed(entry)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Cancelled before native restore dispatch.");
        if (!this.driver.windowRestore) return this.result("refused", "Native restore is unavailable. Ask the user to restore manually; no input fallback.");
        dispatched = true;
        const native = await this.driver.call("restore_window", { ...entry.target, state_id: observed.nativeId }, signal);
        const status = windowRestoreStatus(native.structuredContent, entry.target, observed.nativeId, native.isError === true);
        // A transport cancellation, lost identity or permission change cannot retract a submitted native transition.
        // Keep the native receipt even when this final identity check fails or throws.
        let current = false;
        try { current = !!await this.current(ref, signal); } catch { /* The delivered prefix remains available below. */ }
        const outcome = signal?.aborted || !this.allowed(entry) || !current ? "unknown" : status;
        const message = outcome === "delivered" ? "Native restoration and target foreground confirmed. Keyboard focus is measured separately; Read fresh pixels to inspect paint."
          : outcome === "refused" ? "Native restore refused before mutation. Ask the user to restore manually; no keys or clicks were used."
          : "Restore outcome incomplete or unknown. Retained native prefix follows; do not replay. Read fresh window state/pixels before deciding.";
        return this.result(outcome, message, native.structuredContent, this.addressed(entry));
      } catch (error) { return this.failed(error, dispatched); }
    });
  }
  /** Consume admitted control state once; retain unknown native prefixes without replay or input fallback. */
  manageWindow(ref: string, id: string, command: WindowManagementCommand, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      const entry = this.entries.get(ref), observed = entry?.windowState;
      if (!entry || !observed || observed.id !== id) return this.result("refused", "Retired or missing window state. Read a fresh state.");
      entry.dom = undefined; entry.windowState = undefined; entry.observation = undefined; entry.detail = undefined;
      if (!this.allowed(entry)) return this.denied(ref);
      if (signal?.aborted) return this.result("refused", "Cancelled before native management dispatch.");
      let dispatched = false;
      try {
        if (!await this.current(ref, signal)) return this.result("refused", "Window closed or was reused before management.");
        if (!this.allowed(entry)) return this.denied(ref);
        if (signal?.aborted) return this.result("refused", "Cancelled before native management dispatch.");
        if (observed.kind !== "management" || !this.driver.windowManagement)
          return this.result("refused", "This state/runtime does not admit management. Read qualified control state; no input fallback.");
        const { action, ...geometry } = command;
        dispatched = true;
        const native = await this.driver.call("manage_window", { ...entry.target, state_id: observed.nativeId, operation: action, ...geometry }, signal);
        const status = native.content.some(block => block.type === "image") ? "unknown"
          : windowManagementStatus(native.structuredContent, entry.target, observed.nativeId, command, native.isError === true);
        let current = false;
        try { current = !!await this.current(ref, signal); } catch { /* Preserve the submitted prefix. */ }
        const outcome = signal?.aborted || !this.allowed(entry) || !current ? "unknown" : status;
        return this.result(outcome, outcome === "delivered"
          ? "Native management postcondition confirmed. Foreground/focus are separate measurements; Read fresh pixels to inspect paint."
          : outcome === "refused" ? "Native management refused before mutation. No input fallback."
          : "Management outcome incomplete or unknown. Retained native prefix follows; do not replay. Read fresh state/pixels.",
          native.structuredContent, this.addressed(entry));
      } catch (error) { return this.failed(error, dispatched); }
    });
  }
  /** Read fresh window parts for an existing reference. Each mode retires this window's previous observation. */
  observe(ref: string, signal?: AbortSignal, query?: string, zoom?: DetailRequest, ui?: UiRequest, options?: UiCaptureOptions, deferPresentation = false): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before observation.");
      try {
        const known = this.entries.get(ref);
        if (known && !this.allowed(known)) return this.denied(ref);
        const entry = await this.current(ref, signal);
        if (known && !this.allowed(known)) return this.denied(ref);
        if ((ui && (zoom || query !== undefined || options)) || (zoom && (query !== undefined || options))) return this.result("refused", "Choose native overview, zoom, or a local UI operation; do not combine them.");
        if (options) {
          try { validateUiCapture(options); }
          catch (error) { return this.result("refused", (error as Error).message); }
        }
        if (zoom && !known?.detail) return this.result("refused", "Detail needs a current overview. Observe again.");
        return entry ? ui ? await this.refineUi(entry, ui, signal) : zoom ? await this.windowDetail(entry, zoom, signal) : await this.capture(entry, signal, query, options, false, deferPresentation) : this.result("refused", "Unknown or closed window_ref. Discover again.");
      } catch (error) { return this.failed(error, false); }
    });
  }

  private async refineUi(entry: WindowEntry, request: UiRequest, signal?: AbortSignal): Promise<ComputerResult> {
    const observed = entry.observation;
    if (!observed || observed.id !== request.observation_id) return this.result("refused", "Retired UI observation. Observe again.");
    if (!observed.ui) return this.result("refused", "Image-only observation has no UI resource. Request a UI or full observation.");
    if (!observed.width && ["highlight", "scope", "crop"].includes(request.operation))
      return this.result("refused", "This local image operation needs a full observation; UI-only captured no pixels.");
    let ui: UiResource;
    try { ui = projectUi(observed.ui, request); }
    catch (error) { return this.result("refused", (error as Error).message); }
    if (request.operation === "crop") {
      const selected = ui.elements[0];
      const region = selected?.bounds.window_image;
      if (!region || !observed.width || !observed.height) return this.result("refused", "Element has no proven screenshot-local bounds.");
      const clipped = { x: Math.max(0, region.x), y: Math.max(0, region.y),
        width: Math.min(observed.width, region.x + region.width) - Math.max(0, region.x),
        height: Math.min(observed.height, region.y + region.height) - Math.max(0, region.y) };
      if (clipped.width <= 0 || clipped.height <= 0) return this.result("refused", "Element is outside the observed image.");
      return this.windowDetail(entry, { image_id: observed.id, ...clipped }, signal);
    }
    if (["highlight", "scope"].includes(request.operation)) {
      if (!observed.rawImage) return this.result("refused", "Snapshot image was evicted. Read a new overview; its hints will change.");
      const rendered = await annotateUi(observed.rawImage, ui, observed.width, observed.height,
        observed.settings.hints ?? true, request.operation === "highlight" ? request.hint : undefined, observed.ui.elements,
        { options: observed.settings.guides, action: observed.actionMark });
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; pixels withheld.");
      const selected = request.hint && observed.ui.elements.find(element => element.hint === request.hint);
      const region = selected && observed.frames.get(selected.element_token);
      await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal, rendered.image);
      if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; pixels withheld.");
      const metadata = this.imageMetadata([rendered.image], { screenshot_width: observed.width, screenshot_height: observed.height }, observed.settings);
      return this.result("observed", `Read-only ${request.operation} of the existing UI snapshot: ${ui.elements.length}/${ui.matches} elements. No native reread or input; pixels are from the original observation.`,
        undefined, { ...this.addressed(entry), observation_id: observed.id, ui, source: observed.source,
          agent_image: { ...metadata, route: "cua_snapshot_ui_hints", annotation: rendered.annotation, image_id: observed.id } }, [rendered.image]);
    }
    const selected = request.hint && ui.elements.find(element => element.hint === request.hint);
    const region = selected && observed.frames.get(selected.element_token);
    await this.activity(entry, { kind: "inspect", ...(region ? { region } : {}) }, signal);
    if (!this.allowed(entry) || signal?.aborted) return this.result("refused", "UI inspection revoked or cancelled; data withheld.");
    return this.result("observed", `Read-only UI ${request.operation}: ${ui.elements.length}/${ui.matches} matches in the collected snapshot. ${ui.complete === true ? "Native collection reported complete." : "Collection may be partial; absence does not prove nonexistence."}`,
      undefined, { ...this.addressed(entry), observation_id: observed.id, ui });
  }
  /** Check an issued observation for cached resource reading without capturing or renewing input authority. */
  checkObservation(target: { window_ref?: string; display?: string }, id: string, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before observation reference check.");
      if ((target.window_ref !== undefined) === (target.display !== undefined))
        return this.result("refused", "Choose one observed window or display reference.");
      if (target.display !== undefined) {
        if (this.getMode() !== "yolo") return this.result("refused", "Safe mode forbids display observation reading.");
        const detail = target.display === "primary" ? this.displayOverview : this.displayEntries.get(target.display)?.detail;
        return detail?.id === id ? this.result("observed", "Existing read-only display observation; no fresh pixels or input authority.")
          : this.result("refused", "Retired display observation. Read a new overview.");
      }
      try {
        const known = this.entries.get(target.window_ref!);
        if (known && !this.allowed(known)) return this.denied(known.ref);
        const entry = await this.current(target.window_ref!, signal);
        if (!entry || entry.observation?.id !== id) return this.result("refused", "Retired or closed window observation. Read a new overview.");
        if (!this.allowed(entry)) return this.denied(entry.ref);
        if (signal?.aborted) return this.result("refused", "Observation reference check cancelled.");
        return this.result("observed", "Existing window observation; no native capture or renewal of input authority.", undefined,
          { ...this.addressed(entry), observation_id: id });
      } catch (error) { return this.failed(error, false); }
    });
  }
  /** Consume one overview for a complete action or bounded sequence. Never replay a completed prefix. */
  act(action: Action, signal?: AbortSignal): Promise<ComputerResult> {
    return this.queue(async () => {
      if (signal?.aborted) return this.result("refused", "Cancelled before action dispatch.");
      const entry = this.entries.get(action.window_ref);
      if (entry && !this.allowed(entry)) return this.denied(entry.ref);
      if (entry?.pendingApproval && this.getMode() === "safe") return this.result("refused", "The user must enable and approve this browser connection; Safe mode does not send input to it before approval.");
      const observed = entry?.observation;
      if (!entry || !observed || action.observation_id !== observed.id)
        return this.result("refused", "Stale or missing observation_id/window_ref. Observe again.");
      if (observed.pixelsOnly) {
        const change = await this.observationChange(entry, observed, signal).catch((error: unknown) => `the window state could not be read (${error instanceof Error ? error.message : error})`);
        if (change) { entry.observation = undefined; return this.result("refused", `Observation expired: ${change}. Read the window again; no input was dispatched.`); }
      }
      const { window_ref: _ref, observation_id: _id, delivery_mode, uia_only, ...command } = action;
      const batch = command.action === "sequence";
      let commands: InputStep[];
      try {
        if (delivery_mode !== undefined && !["background", "foreground"].includes(delivery_mode)) throw new Error("Unsupported delivery mode.");
        if (uia_only !== undefined && (uia_only !== true || delivery_mode === "foreground" || batch || !["click", "scroll"].includes(String(command.action))))
          throw new Error("uia_only takes one background click or scroll on an element.");
        if (batch) {
          if (Object.keys(command).some(key => !["action", "sequence"].includes(key)) || !Array.isArray(command.sequence) || command.sequence.length < 1 || command.sequence.length > SEQUENCE_STEPS)
            throw new Error(`A sequence needs 1..${SEQUENCE_STEPS} steps and no conflicting top-level action fields.`);
          commands = command.sequence;
        } else commands = [command];
        commands = commands.map(step => {
          const { hint, ...nativeStep } = step;
          if (hint !== undefined) {
            if (typeof hint !== "string" || step.element_token !== undefined || step.x !== undefined || step.y !== undefined)
              throw Error("Use exactly one current hint or native token/pixel target.");
            if (!observed.ui) throw Error("Image-only observation has no current hints.");
            if (observed.pixelsOnly) throw Error("UI hints of this observation expired with the previous input. Use pixel coordinates, or Read the window again.");
            nativeStep.element_token = resolveUi(observed.ui, hint).element_token;
          }
          const element = observed.ui?.elements.find(e => e.element_token === nativeStep.element_token);
          if (element?.enabled === false) throw Error("The observed target is disabled. No input was dispatched.");
          if (!isSemanticAction(nativeStep.action) && element?.bounds.window_image && observed.width && observed.height) {
            const r = element.bounds.window_image;
            if (r.x >= observed.width || r.y >= observed.height || r.x + r.width <= 0 || r.y + r.height <= 0)
              throw Error("The observed target is outside the image. No input was dispatched.");
          }
          validateInput(nativeStep, observed, batch);
          if (isCoordinateScroll(nativeStep)) {
            if (!this.driver.coordinateScroll) throw Error("Coordinate scroll requires the explicitly selected native scroll runtime. No input was dispatched.");
            if (!observed.captureId) throw Error("Coordinate scroll needs this overview's native capture. Observe again.");
          }
          if (isPointerMove(nativeStep.action)) {
            if (!this.driver.nativePointer) throw Error("Pointer movement requires the explicitly selected native pointer runtime. No input was dispatched.");
            if (nativeStep.element_token ? !observed.frames.has(nativeStep.element_token) : !observed.captureId)
              throw Error("Pointer movement needs a current token with proven geometry or this overview's native capture. Observe again.");
          }
          if (isSemanticAction(nativeStep.action)) {
            if (!this.driver.semanticActions) throw Error("Semantic actions require the explicitly selected qualified Windows semantic runtime. No input was dispatched.");
            if (delivery_mode === "foreground") throw Error("Semantic actions use accessibility only: no foreground delivery.");
          }
          return nativeStep;
        });
        if (observed.pixelsOnly && commands.some(step => step.element_token !== undefined || isSemanticAction(step.action)))
          throw new Error("UI tokens of this observation expired with the previous input. Use pixel coordinates, or Read the window again.");
        if (this.driver.captureBoundDrag && !observed.captureId && !observed.pixelsOnly && commands.some(step => step.action === "drag" && step.via === undefined))
          throw new Error("Straight drag requires this overview's native capture. Observe again; no batch input was dispatched.");
        if (commands.some(step => step.action === "hold") && !this.driver.heldKeys)
          throw new Error("Key holds require the explicitly selected qualified Windows key-hold runtime. No input was dispatched.");
        if (commands.some(step => step.via !== undefined) && !this.driver.heldPaths)
          throw new Error("Continuous paths require the explicitly selected qualified Windows fork. No batch input was dispatched.");
        if (batch && commands.reduce((ms, step) => ms + requestedInputTime(step), 0) > SEQUENCE_REQUESTED_MS)
          throw new Error(`A sequence exceeds ${SEQUENCE_REQUESTED_MS / 60000} minutes of requested waits/typing/drags.`);
        commands = commands.map(step => ({ ...step, ...(step.keys ? { keys: [...step.keys] } : {}), ...(step.modifiers ? { modifiers: [...step.modifiers] } : {}), ...(step.via ? { via: step.via.map(point => ({ ...point })) } : {}), ...(step.actions ? { actions: step.actions.map(pointer => ({ ...pointer, ...(pointer.via ? { via: pointer.via.map(point => ({ ...point })) } : {}) })) } : {}) }));
      } catch (error) { return this.result("refused", (error as Error).message); }

      const started = performance.now();
      const trace = { total: commands.length, attempted: 0, completed: 0, elapsed_ms: 0,
        steps: [] as Record<string, unknown>[], stop_reason: "" };
      let inFlight = false;
      const finish = (result: ComputerResult, reason = "") => {
        if (!batch) return result;
        trace.elapsed_ms = performance.now() - started; trace.stop_reason = reason;
        const status = result.details.status;
        return this.result(status, `Sequence ${status}: ${trace.completed}/${trace.total} steps delivered; ${trace.attempted} attempted in ${Math.round(trace.elapsed_ms)}ms. ${reason || "Observe again to verify application state."} Never replay the completed prefix.`,
          { ...(result.details.state ?? {}), sequence: trace }, { ...this.addressed(entry), ...(result.details.highlight ? { highlight: result.details.highlight } : {}) });
      };
      try {
        let capture = observed.captureId;
        for (const [index, step] of commands.entries()) {
          if (signal?.aborted) return finish(this.result("refused", "Cancelled."), "Cancelled before the next dispatch.");
          if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before the next dispatch.");
          if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached; no more steps dispatched.`);
          if (step.wait_ms) await delay(step.wait_ms, undefined, { signal });
          const current = await this.current(entry.ref, signal);
          if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked before dispatch.");
          if (!current) return finish(this.result("refused", "Window closed or its native handle was reused. Discover again."), "Window lifetime changed.");
          if (signal?.aborted) return finish(this.result("refused", "Cancelled."), "Cancelled before dispatch.");
          // The native capture is single-use: the first step that binds to it spends it, later pixel steps use the window's current mapping.
          const { name, args } = nativeInput(step, entry.target, delivery_mode ?? "background", capture, this.driver.captureBoundDrag, uia_only);
          if (args.capture_id !== undefined) capture = undefined;
          // AX bounds are native. Only native click receipts establish an exact pixel action mark.
          const mark = { kind: "action" as const, region: step.element_token ? observed.frames.get(step.element_token) : undefined };
          const activityAction = presentationAction(step);
          // A password field's text is never shown.
          const typedElement = observed.ui?.elements.find(element => step.element_token ? element.element_token === step.element_token : element.focused);
          const focusedElement = observed.ui?.elements.find(element => element.focused);
          const typedText = ["type_text", "set_value"].includes(step.action)
            ? !typedElement?.password ? shownTypedText(step.text) : undefined
            : presentationDetail(step, !batch && step.action !== "timeline" && !!focusedElement && !focusedElement.password);
          let highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal, mark, undefined, activityAction, undefined, typedText);
          if (signal?.aborted) return finish(this.result("refused", "Cancelled during action mark; input was not dispatched."), "Cancelled during action mark.");
          if (!this.inputAllowed(entry)) return finish(this.denied(entry.ref), "Permission revoked during action mark.");
          if (batch && performance.now() - started >= SEQUENCE_ELAPSED_MS) return finish(this.result("refused", "Sequence time budget reached."), `${SEQUENCE_ELAPSED_MS / 60000} min elapsed budget reached during admission; no more steps dispatched.`);
          // Input may change the page, so the window's DOM snapshot retires with its native capture.
          entry.observation = undefined; entry.detail = undefined; entry.dom = undefined;
          const pointerAction = isPointerMove(step.action) || isCoordinateScroll(step) || ["click", "right_click", "double_click", "drag"].includes(step.action) || (step.action === "hold" && !!step.actions?.length);
          if (pointerAction) entry.lastAction = { reason: "Pointer outcome is not yet confirmed." };
          const requested = { ...step, ...(step.text !== undefined ? { text_length: step.text.length } : {}),
            ...(step.via ? { via_points: step.via.length } : {}), ...(step.actions ? { actions: step.actions.map(pointer => { const { via, ...compact } = pointer; return { ...compact, ...(via ? { via_points: via.length } : {}) }; }) } : {}) };
          delete requested.text; delete requested.element_token;
          delete requested.via;
          const row: Record<string, unknown> = { index, action: step.action, native_tool: name, requested,
            requested_wait_ms: step.wait_ms ?? 0, dispatch_ms: performance.now() - started, status: "unknown" };
          trace.steps.push(row); trace.attempted++; inFlight = true;
          // A sequence stops between steps: its current step finishes and releases its own keys and buttons,
          // and the driver connection survives the cancellation. A single action stays cancellable in flight.
          const nativeSignal = batch ? undefined : signal;
          const yoloDispatch = this.getMode() === "yolo";
          this.nativeInputsInFlight++;
          let native: DriverResult;
          try {
            native = await this.driver.call(name, args, nativeSignal);
            const effect = String(native.structuredContent?.effect), status = String(native.structuredContent?.status);
            if (yoloDispatch && ((native.isError && effect !== "refused" && status !== "refused") || ["unknown", "partial"].includes(status) || ["unknown", "partial"].includes(effect)))
              this.unconfirmedUnsafeWork = true;
          } catch (error) {
            // Cancellation or transport loss may leave a scheduled native timeline running after this promise rejects.
            if (yoloDispatch) this.unconfirmedUnsafeWork = true;
            throw error;
          } finally { this.nativeInputsInFlight--; }
          row.completion_ms = performance.now() - started;
          if (!this.inputAllowed(entry) || (!batch && signal?.aborted)) return finish(this.result("unknown",
            "Input was dispatched before permission was revoked or cancellation. Outcome withheld; native input may still have completed. Do not replay blindly."),
            "In-flight outcome withheld after revocation/cancellation.");
          const state = native.structuredContent ?? {};
          const nativeMessage = native.content.filter(c => c.type === "text").map(c => c.text).join("\n");
          // Three physical Escape presses stop input: nothing more is sent, and a running string or timeline stops part way.
          const stopped = /human_stop:/.test(nativeMessage) || state.code === "human_stop";
          const stopRefused = stopped && (state.status === "refused" || state.effect === "refused");
          // A real press at a point another window covers is refused by the driver before anything is pressed there.
          const covering = coveringWindow(nativeMessage);
          const refused = stopRefused || state.status === "refused" || state.effect === "refused" || ["invalid_action_target", "invalid_arguments", "stale_element_token",
            "conflicting_element_target", "background_unavailable", "background_occluded", "background_uipi_blocked", "key_hold_unavailable", "held_path_unavailable", "input_busy"].includes(String(state.code));
          const uncertain = native.isError || ["unknown", "partial"].includes(String(state.status)) || ["unknown", "partial"].includes(String(state.effect));
          const inconsistent = !refused && !uncertain && (isSemanticAction(step.action) ? !semanticReceipt(step.action, state)
            : isPointerMove(step.action) ? delivery_mode !== "foreground" || !pointerReceipt(step, state)
            : isCoordinateScroll(step) && (delivery_mode !== "foreground" || !scrollReceipt(step, state)));
          const status = refused ? "refused" : uncertain || inconsistent ? "unknown" : "delivered";
          row.status = status; inFlight = false;
          const capability = actionCapability(step.action, name, delivery_mode ?? "background", status, state);
          row.action_capability = capability;
          if (status === "delivered") trace.completed++;
          const resolvedPoint = status === "delivered" && step.x !== undefined && step.action === "click" && observed.bounds
            ? nativeClickPoint(native.content, entry.target.pid, observed.bounds) : undefined;
          if (status === "delivered" && pointerAction) {
            const target = step.element_token ? observed.frames.get(step.element_token) : undefined;
            const mapping = observed.mapping;
            const sameMapping = resolvedPoint && observed.bounds && mapping && step.x !== undefined && step.y !== undefined &&
              Math.abs(resolvedPoint.x - (observed.bounds.x + mapping.offsetX + step.x * mapping.scale)) <= 1 &&
              Math.abs(resolvedPoint.y - (observed.bounds.y + mapping.offsetY + step.y * mapping.scale)) <= 1;
            const r = target ?? (sameMapping ? { ...resolvedPoint, width: 1, height: 1 } : undefined);
            entry.lastAction = r && observed.bounds
              ? { region: { ...r, x: r.x - observed.bounds.x, y: r.y - observed.bounds.y },
                basis: target ? "completed_native_token_target" : "completed_native_click_receipt" }
              : { reason: "Native action did not report usable target geometry." };
          }
          highlight = await notifyWindowObserver(this.observer, entry.target, entry.identity, observed.bounds, signal,
            { kind: "action", ...(resolvedPoint ? { point: resolvedPoint } : step.element_token && mark.region ? { region: mark.region } : {}) }, undefined, activityAction, status, typedText);
          if (!this.inputAllowed(entry) || (!batch && signal?.aborted)) {
            entry.lastAction = { reason: "In-flight pointer outcome was withheld after revocation or cancellation." };
            return finish(this.result("unknown", "Input was dispatched before revocation or cancellation. Do not replay blindly."), "In-flight outcome withheld after input.");
          }
          const cover = covering && await this.coverText(covering, signal);
          // A click that closed its window, such as OK in a dialog, names the window of the app the driver found in front.
          const after = status === "delivered" ? foregroundAfter(state.foreground_after) : undefined;
          const front = after && `The click closed this window; ${await this.windowName(after, signal)} of the same app is now in front.\n`;
          const result = this.result(status, stopped ? `The human stopped computer use with Escape. ${status === "refused" ? "Nothing was sent." : "Input stopped part way; observe before anything else once computer use is on again."} Do not retry or work around it; tell the human and wait until they turn it on again with /computer-mode.`
            : cover && status === "refused" ? `Nothing was pressed: ${cover}` : `${cover ? `A press stopped: ${cover}\n` : ""}${front ?? ""}Driver returned ${status} for ${name} (requested: ${delivery_mode ?? "background"}). ${isSemanticAction(step.action) ? "A UIA receipt is not app or focus verification. " : ""}Observe again to verify application state.${step.action === "hold" && typeof state.summary === "string" ? `\n${state.summary}` : ""}`,
            { ...state, ...(step.element_token || pointerAction ? { coordinate_grounding: { input_space: step.element_token ? "native_element_token" : "window_image_pixels", transform: "cua_native_once",
              overview: { width: observed.width, height: observed.height }, ...(step.x !== undefined ? { requested: { x: step.x, y: step.y } } : {}),
              resolved_native_screen: resolvedPoint ?? null, native_target_mark: isSemanticAction(step.action) ? null : entry.lastAction ?? null } } : {}),
            ...(native.isError || !native.structuredContent ? {
              native_message: nativeMessage,
            } : {}) }, { ...this.addressed(entry), action_capability: capability, ...(highlight ? { highlight } : {}) });
          if (!batch || status !== "delivered" || index === commands.length - 1) {
            // Pixel-only input keeps the overview usable: the next call checks the window size instead of capturing again.
            if (status === "delivered" && this.driver.windowManagement && observed.bounds && commands.every(item => item.element_token === undefined && !isSemanticAction(item.action)))
              entry.observation = { ...observed, captureId: undefined, pixelsOnly: true };
            return finish(result, status === "delivered" ? "" : `Stopped after native ${status}; later steps were not dispatched.`);
          }
        }
        throw new Error("No complete input step.");
      } catch (error) {
        if (inFlight && trace.steps.length) trace.steps.at(-1)!.completion_ms = performance.now() - started;
        return finish(await this.failed(error, inFlight), inFlight ? "Transport/dispatch failed; current step is uncertain." : "Stopped before next dispatch.");
      }
    });
  }

  /**
   * After input whose window could not be read again: when the window is gone from the window list (closed or hidden),
   * say so and name the app's frontmost remaining window. The list is in the desktop's z-order, front first. It takes
   * the native target, since a closed window's reference is already dropped.
   */
  closedWindowNote(target: WindowTarget, signal?: AbortSignal): Promise<string | undefined> {
    return this.queue(async () => {
      if (this.driver.windowLifetime === false) return undefined;
      const windows = await this.list(signal).catch(() => undefined);
      if (!windows || windows.some(window => window.pid === target.pid && window.window_id === target.window_id)) return undefined;
      const front = windows.find(window => window.pid === target.pid && Number.isSafeInteger(window.window_id));
      return `This window closed or is hidden; ${front ? `the app's frontmost window is ${await this.windowName({ pid: target.pid,
        window_id: front.window_id as number, title: typeof front.title === "string" ? front.title : "" }, signal)}` : "the app has no other visible window"}.`;
    });
  }

  /** A window the driver named by its handle: its title, and its address where this session may act on it. YOLO adopts it; Safe never does. */
  private async windowName(window: NamedWindow, signal?: AbortSignal): Promise<string> {
    const entry = this.getMode() === "yolo" ? await this.register(window, signal).catch(() => undefined) : undefined;
    return `${window.title ? `"${window.title}"` : "an untitled window"}${entry ? ` (${computerSource("window", entry.ref)})` : ""}`;
  }

  /** Names the window that covers a press point, with its address where this session may act on it. */
  private async coverText(covering: CoveringWindow, signal?: AbortSignal): Promise<string> {
    const named = await this.windowName(covering, signal);
    const then = "Close or move it, or act on it instead.";
    if (covering.relation === "owned_window")
      return `${named}, a window this window owns, covers the point. It stays in front of this window even when this window is active. ${then}`;
    return `${covering.relation === "same_process" ? "another window of this app" : "a window of another app"}, ${named}, covers the point. ${then}`;
  }

  /** Why a kept pixel observation no longer matches its window, or undefined while it still does. Movement keeps window-local points valid. */
  private async observationChange(entry: WindowEntry, observed: Observation, signal?: AbortSignal): Promise<string | undefined> {
    const native = await this.driver.call("get_window_control_state", { pid: entry.target.pid, window_id: entry.target.window_id }, signal);
    const state = native.structuredContent?.state as { bounds?: ScreenRect; minimized?: boolean; visible?: boolean } | undefined;
    if (native.isError || !state?.bounds) return "the window state could not be read";
    if (state.minimized) return "the window was minimized";
    if (state.visible === false) return "the window is no longer visible";
    const before = observed.bounds!, now = state.bounds;
    if (now.width !== before.width || now.height !== before.height) return `the window was resized from ${before.width}x${before.height} to ${now.width}x${now.height}`;
    return undefined;
  }
  /** Whether a window still holds this observation for input. */
  holdsObservation(windowRef: string, id: string): boolean { return this.entries.get(windowRef)?.observation?.id === id; }
  /**
   * What typed input can use in a window now: its current observation (a DOM snapshot of a browser page, or a native
   * capture with its size and, until the first input spends them, its UI hints) and the size of its latest image,
   * which is the space of the agent's points.
   */
  inputView(ref: string): { dom?: string; native?: { id: string; width?: number; height?: number; ui?: UiResource; full: boolean }; seen?: { width: number; height: number } } {
    const entry = this.entries.get(ref), observed = entry?.observation;
    // A full-resolution observation's pixels are the window's own: viewport rectangles apply to them as they are.
    const full = !!observed && observed.width !== undefined && nativeImageCap(observed.settings) >= FULL_RESOLUTION;
    return { ...(entry?.dom ? { dom: entry.dom.id } : {}),
      ...(observed ? { native: { id: observed.id, width: observed.width, height: observed.height, full, ...(observed.ui && !observed.pixelsOnly ? { ui: observed.ui } : {}) } } : {}),
      ...(entry?.seen ? { seen: { ...entry.seen } } : {}) };
  }
  /** Whether this session issued a still-registered window reference. */
  hasWindow(ref: string): boolean { return this.entries.has(ref); }
  /** Whether this session issued a still-registered display reference. */
  hasDisplay(ref: string): boolean { return this.displayEntries.has(ref); }
  /** Whether this session captured the sequence and still retains it. */
  hasSequence(id: string): boolean { return this.sequences.get(id) !== undefined; }

  /** Close session-opened browsers (deleting their profiles), detach from attached ones and release the native runtime. Other windows/files are not closed or saved. */
  close(): Promise<ComputerResult> {
    return this.queue(async () => {
      const watches = [...this.watches.values()];
      this.watches.clear();
      for (const watch of watches) watch.stop.abort();
      await Promise.allSettled(watches.map(watch => watch.done));
      if (this.watcher && this.watcher !== this.driver) await this.watcher.close().catch(() => undefined);
      this.watcher = undefined;
      await Promise.allSettled([...this.watchDirs].map(dir => rm(dir, { recursive: true, force: true })));
      this.watchDirs.clear();
      this.entries.clear(); this.displayEntries.clear(); this.displayOverview = undefined; this.uiImages = []; this.sequences.clear();
      const browsers = [...this.browsers.values()];
      this.browsers.clear();
      const cleanup = await Promise.allSettled(browsers.map(browser => browser.close()));
      browsers.forEach((browser, i) => { if (cleanup[i].status === "fulfilled") this.unapprovedAgentBrowsers.delete(browser); });
      const modes = [...this.replacedModes];
      this.replacedModes.clear();
      const restored = await Promise.allSettled(modes.map(([device, mode]) => this.host!.displayModes.restore(device, mode)));
      if (this.recording) { clearTimeout(this.recording.timer); this.recording = undefined; }
      const files = await this.host?.removeFiles().then(() => true, () => false) ?? true;
      await this.driver.close();
      const failed = cleanup.filter(result => result.status === "rejected").length;
      const unrestored = modes.filter((_, i) => restored[i].status === "rejected").map(([device, mode]) => `${device} (${mode.width}x${mode.height})`);
      if (unrestored.length) return this.result("unknown", `Native runtime released, but restoring the display mode of ${unrestored.join(", ")} failed. Signing out of Windows also restores it.`);
      if (!files) return this.result("unknown", "Native runtime released, but removing the session's file folder failed.");
      if (failed || this.hasUnapprovedAgentBrowser()) return this.result("unknown", `Native runtime released, but prior browser or input cleanup is unconfirmed (${failed} of ${browsers.length} managed browsers failed). A browser, local debugging port or scheduled input may still be active; verify before restarting this session.`);
      return this.result("closed", "Owned computer-use runtime and session-opened browsers released; attached browsers were detached and stay open. Other application documents were not closed or saved.");
    });
  }
}