Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
43.3 KB
Raw Blame
// Computer use, portable host (Windows + Linux X11): the `computer-use` MCP
// server of contract v2, run in the app's own process (the desktop's
// Electron main, the CLI's node). Native work happens in native-worker.mjs.
//
// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui
// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.
//
// No questions (owner decision 2026-10-21): a session on the person's screen
// opens only when the person turned the real screen on (a setting, off by
// default; policy.mjs sessionRules) and then acts without asking. What stays:
// the deny list, hit-testing of every point, the person-input pause, the
// stuck-input release, Stop and the server switch. `virtual` (virtual-host.mjs)
// serves app_id "virtual:<name>", the agent's own display, on the same
// connection.

import { spawn } from "node:child_process";
import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
import { mkdir, rename, unlink, writeFile } from "node:fs/promises";
import { createServer } from "node:net";
import path from "node:path";
import { fileURLToPath } from "node:url";

import {
  actResult,
  CONTRACT_VERSION,
  failure,
  INSTRUCTIONS,
  mapPoint,
  mcpResult,
  normalizeLimits,
  observeResult,
  osName,
  SERVER_NAME,
  toolDefinitions,
  TOOLS,
  validateAction,
  validateCall,
} from "./contract.mjs";
import { compareGrids, encodeJpeg, encodePng, fitLongSide, luminanceGrid, paintBoxes } from "./image.mjs";
import { sha256 } from "./media.mjs";
import { deniedApp, REAL_SCREEN_NOT_OPTED_IN_MESSAGE, serverRealScreen, sessionRules } from "./policy.mjs";

const RPC = new Set(["initialize", "ping", "tools/list", "tools/call", "notifications/initialized", "notifications/cancelled"]);
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
// For timeouts raced against an answer: never keeps a short-lived process alive.
const timeout = (ms) => new Promise((resolve) => { const timer = setTimeout(resolve, ms); timer.unref?.(); });
const now = () => new Date().toISOString();
const record = (value) => typeof value === "object" && value !== null && !Array.isArray(value);

/**
 * Starts the native worker. `nodePath` runs it (the app's own binary with
 * `nodeEnv`, e.g. ELECTRON_RUN_AS_NODE=1); `koffiPath` is koffi's entry.
 * Held keys/buttons are released by a one-shot `--release` worker if it exits.
 */
export function createNativeWorker({ workerPath = fileURLToPath(new URL("./native-worker.mjs", import.meta.url)), nodePath = process.execPath, nodeEnv = {}, koffiPath, timeoutMs = 10_000 } = {}) {
  const env = { ...process.env, ...nodeEnv, ...(koffiPath ? { OMNIRUSH_KOFFI: koffiPath } : {}) };
  const worker = spawn(nodePath, [workerPath], { env, stdio: ["ignore", "ignore", "ignore", "ipc"], windowsHide: true, serialization: "advanced" });
  const pending = new Map();
  const listeners = new Set();
  let nextId = 0, closed = false, held = { keys: [], buttons: [] };
  let readyResolve;
  const ready = new Promise((resolve) => { readyResolve = resolve; });
  const readyTimer = setTimeout(() => readyResolve({ supported: false, code: "native_unavailable", error: "Desktop access did not start. Restart the app." }), timeoutMs);
  worker.on("message", (value) => {
    if (!record(value)) return;
    if (value.event === "ready") { clearTimeout(readyTimer); readyResolve(value); return; }
    if (value.event === "held") { held = { keys: value.keys ?? [], buttons: value.buttons ?? [] }; return; }
    if (value.event) { for (const listener of listeners) listener(value); return; }
    const request = pending.get(value.id);
    if (!request) return;
    pending.delete(value.id); clearTimeout(request.timer);
    if (record(value.error)) request.reject(Object.assign(new Error(String(value.error.message)), { code: value.error.code }));
    else request.resolve(value.result);
  });
  const fail = (error) => {
    if (closed) return;
    clearTimeout(readyTimer); readyResolve({ supported: false, code: "native_unavailable", error: error.message });
    closed = true;
    for (const request of pending.values()) { clearTimeout(request.timer); request.reject(error); }
    pending.clear();
    if (held.keys.length || held.buttons.length) {
      // The worker died with input held: release it from a fresh one.
      const releaser = spawn(nodePath, [workerPath, "--release", JSON.stringify(held)], { env, stdio: "ignore", windowsHide: true });
      releaser.on("error", () => {});
      held = { keys: [], buttons: [] };
    }
    for (const listener of listeners) listener({ event: "unavailable" });
  };
  worker.on("error", fail);
  worker.on("exit", () => fail(new Error("Desktop access stopped. Restart the app.")));
  return {
    ready,
    pid: worker.pid,
    get held() { return held; },
    onEvent(listener) { listeners.add(listener); return () => listeners.delete(listener); },
    call(method, params = {}, ms = timeoutMs) {
      if (closed) return Promise.reject(Object.assign(new Error("Desktop access has stopped."), { code: "native_unavailable" }));
      const id = ++nextId;
      return new Promise((resolve, reject) => {
        const timer = setTimeout(() => { pending.delete(id); reject(Object.assign(new Error("Desktop operation timed out."), { code: "timeout" })); }, ms);
        pending.set(id, { resolve, reject, timer });
        worker.send({ id, method, params });
      });
    },
    kill() { worker.kill(); },
    async close() { if (closed) return; await this.call("shutdown").catch(() => {}); closed = true; if (worker.exitCode === null) worker.kill(); },
  };
}

function windowRef(w) {
  return { id: String(w.id), app_id: w.appId, app_name: w.appName, bundle_or_exe: w.executable, pid: w.pid, title: w.title, origin: { x: w.bounds.x, y: w.bounds.y }, bounds: { ...w.bounds } };
}

function appIdentity(w) {
  return { pid: w.pid, name: w.appName, exe: w.executable, app_id: w.appId, title: w.title };
}

function describeAction(a) {
  if (!a) return "";
  if (a.type === "type") return `type ${JSON.stringify(a.text.length > 40 ? `${a.text.slice(0, 40)}…` : a.text)}`;
  if (a.type === "key") return `key ${[...(a.modifiers ?? []), a.key].join("+")}`;
  if (a.type === "drag") return `drag ${a.path.length} points`;
  if (a.type === "scroll") return `scroll ${a.delta} at ${a.x},${a.y}`;
  if (a.type === "wait") return `wait ${a.ms} ms`;
  return `${a.type.replace("_", " ")} at ${a.x},${a.y}`;
}

/**
 * The host. Options:
 * - native: a createNativeWorker() (or the same interface)
 * - spool: a media spool (media.mjs createMediaSpool) frames go to, or null
 * - settings(): { enabled, real_screen, real_screen_opt_in, real_screen_on, virtual_display, limits, deny_apps } now
 *   (real_screen and real_screen_opt_in: the server allows the person's screen for this account, both true;
 *   real_screen_on: this device's own setting is on)
 * - refresh(): read the server's switch and limits now (before each open and each discover of the person's
 *   screen); true only when the server answered. Without it, or when it is false or throws, the person's
 *   screen is refused (the agent's own display is not affected)
 * - context(): { ancestors, self_pids, tty_pids, chat } now, or a promise of it (chat: per-chat counts)
 * - virtual: a createVirtualHost() for app_id "virtual:<name>", or null
 * - onEvent(event): `computer.session` events; onState(states); onFrame(frame)
 */
export async function createPortableHost({ native: firstNative, nativeFactory = null, spool = null, settings, context = () => ({}), refresh = null, virtual = null, onEvent = () => {}, onState = () => {}, onFrame = () => {}, platform = process.platform }) {
  let native = firstNative;
  let availability = await native.ready;
  let nativeDown = false;
  const restarts = [];
  let restartedNote = false;
  const os = osName(platform);
  const sessions = new Map();
  const shots = new Map();
  const virtualOwners = new Map(); // virtual session id → the connection that opened it
  let lease = null;
  let closed = false;

  const limits = () => normalizeLimits(settings()?.limits);
  const emit = (s, event, by, extra = {}) => {
    try {
      onEvent({ event, by, session_id: s.id, app_id: s.window.appId, app_name: s.window.appName, bundle_or_exe: s.window.executable, mode: s.mode, ...(s.approval ? { approval: s.approval } : {}), ...(s.screen ? { screen: s.screen } : {}), os, ...extra });
    } catch { /* the trace is best effort */ }
  };
  const publicState = () => [...sessions.values()].map((s) => ({
    id: s.id, phase: s.phase, mode: s.mode, purpose: s.purpose, status: s.status ?? "",
    app: { app_id: s.window.appId, app_name: s.window.appName, bundle_or_exe: s.window.executable, pid: s.window.pid },
    window: { id: String(s.window.id), title: s.window.title }, windows: (s.windows ?? []).map((w) => ({ id: String(w.id), title: w.title })),
    approval: s.approval ?? null, actions: s.actions, actions_left: Math.max(0, limits().actions_per_session - s.actions),
    seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)), can_continue: s.phase === "paused", log: s.log.slice(-30),
    frame: s.lastFrame ?? null, chat: s.chat ?? null, hidden: s.hidden === true, window_bounds: { ...s.window.bounds },
  }));
  const update = () => { try { onState(publicState()); } catch { /* UI is best effort */ } };

  async function stop(s, event, by, extra) {
    if (!sessions.has(s.id)) return;
    sessions.delete(s.id);
    s.phase = "ended"; s.observation = null;
    if (lease === s.id) lease = null;
    await native.call("stop").catch(() => {});
    emit(s, event, by, extra);
    update();
  }
  function pause(s, by, status) {
    if (s.phase !== "working") return;
    s.phase = "paused"; s.observation = null; s.status = status;
    void native.call("stop").catch(() => {});
    emit(s, "paused", by);
    update();
  }
  const attach = (worker) => worker.onEvent((value) => {
    if (worker !== native) return;
    if (value.event === "person-input") for (const s of sessions.values()) if (s.phase === "working" && s.mode === "control") pause(s, "person", "Your input paused computer use. Choose Continue when you are ready.");
    if (value.event === "unavailable") {
      nativeDown = true;
      for (const s of [...sessions.values()]) void stop(s, "stopped", "limit", { reason: "native_unavailable" });
    }
  });
  attach(native);
  /**
   * After the native worker died (its held input already released by createNativeWorker), a new one
   * is started on the next call, at most 3 times in 10 minutes. Whether the host can work now.
   */
  async function ensureNative() {
    if (!nativeDown || !nativeFactory || closed) return !nativeDown;
    const now = Date.now();
    while (restarts.length && now - restarts[0] > 600_000) restarts.shift();
    if (restarts.length >= 3) return false;
    restarts.push(now);
    try {
      const next = nativeFactory();
      const ready = await next.ready;
      native = next; availability = ready; nativeDown = !ready.supported; restartedNote = ready.supported;
      attach(next);
    } catch {
      return false;
    }
    return !nativeDown;
  }
  const ticker = setInterval(() => {
    const l = limits();
    // The server no longer allows the person's screen for this account (its latest answer): those sessions end.
    if (sessions.size && !serverRealScreen(settings() ?? {})) { void stopRealScreen("server"); return; }
    for (const s of [...sessions.values()]) {
      if (Date.now() >= s.expiresAt && s.phase !== "approval") void stop(s, "expired", "limit");
      else if (s.phase === "working" && Date.now() - s.lastOperation > l.idle_pause_seconds * 1000) pause(s, "limit", "Paused after a while without work. Choose Continue when ready.");
    }
  }, 1000);
  ticker.unref?.();

  // Windows: the agent's ancestor processes (its terminal or IDE) read natively when the app passed none.
  let nativeAncestors = null;
  async function ancestorsOf(ctx) {
    if (Array.isArray(ctx.ancestors) && ctx.ancestors.length) return ctx.ancestors;
    if (platform !== "win32") return ctx.ancestors ?? [];
    if (!nativeAncestors) nativeAncestors = await native.call("ancestors", { pid: process.pid }).catch(() => null);
    return nativeAncestors ?? [];
  }

  async function screenOf(w) {
    const screens = await native.call("screens").catch(() => []);
    const cx = w.bounds.x + w.bounds.width / 2, cy = w.bounds.y + w.bounds.height / 2;
    const screen = screens.find((s) => cx >= s.x && cx < s.x + s.width && cy >= s.y && cy < s.y + s.height) ?? screens.find((s) => s.primary) ?? screens[0];
    return screen ? { id: String(screen.id), width: screen.width, height: screen.height, scale: w.scale ?? screen.scale ?? 1 } : null;
  }

  async function capture(w) {
    const l = limits();
    let frame = await native.call("capture", { window: w }, 15_000);
    let grid = luminanceGrid(fitLongSide(frame, l.model_long_side));
    // Wait (briefly) for the window to settle: until two frames match (a blinking caret may never let it; then the last frame).
    for (let attempt = 0; attempt < 4; attempt += 1) {
      await sleep(attempt === 0 ? 60 : 90);
      const next = await native.call("capture", { window: w }, 15_000);
      const nextGrid = luminanceGrid(fitLongSide(next, l.model_long_side));
      const same = compareGrids(grid, nextGrid).changed === 0;
      frame = next; grid = nextGrid;
      if (same) break;
    }
    return frame;
  }

  async function observation(s, w, { store = true } = {}) {
    const l = limits();
    const raw = await capture(w);
    const masked = paintBoxes(raw, raw.secure ?? []);
    const model = fitLongSide(raw, l.model_long_side);
    const tiny = model.width * model.height <= 160 * 160;
    const bytes = tiny ? encodePng(model) : encodeJpeg(model, { quality: 80 });
    const mime = tiny ? "image/png" : "image/jpeg";
    const sha = sha256(bytes);
    let full = null;
    if (Math.max(raw.width, raw.height) > Math.max(model.width, model.height)) {
      const big = fitLongSide(raw, l.full_long_side);
      const png = encodePng(big, { level: 3 });
      full = { bytes: png, width: big.width, height: big.height, sha256: sha256(png) };
    }
    const ocr = raw.ocr === "client" ? "client" : "none";
    if (store && spool) {
      try { spool.put({ bytes, mime, width: model.width, height: model.height, full, thumb: encodeJpeg(fitLongSide(model, 320), { quality: 70 }), ocr, masked }); } catch { /* the archive copy is best effort */ }
    }
    const ratio = model.width / raw.width;
    const scaled = (r) => ({ x: r.x * ratio, y: r.y * ratio, width: r.width * ratio, height: r.height * ratio });
    const caret = raw.caret ? scaled(raw.caret) : null;
    // What the host can read on screen (macOS: AX / text recognition), for the per-act rules only: never sent to the model.
    const texts = Array.isArray(raw.texts) ? raw.texts.filter((t) => t && typeof t.text === "string" && [t.x, t.y, t.width, t.height].every(Number.isFinite)).slice(0, 2000).map((t) => ({ ...scaled(t), text: t.text.slice(0, 200) })) : null;
    const secureFields = (Array.isArray(raw.secure_fields) ? raw.secure_fields : raw.secure ?? []).filter((r) => r && [r.x, r.y, r.width, r.height].every(Number.isFinite)).map(scaled);
    const id = randomUUID();
    const value = {
      id, at: Date.now(), window: w, sha, mime, bytes,
      model: { width: model.width, height: model.height }, grid: luminanceGrid(model), caret, texts, secureFields,
      ref: { id, width: model.width, height: model.height, sha256: sha, mime, ...(full ? { full: { width: full.width, height: full.height, sha256: full.sha256 } } : {}), ocr, masked },
    };
    s.lastFrame = { sha, width: model.width, height: model.height, mime, data_url: `data:${mime};base64,${bytes.toString("base64")}`, at: now() };
    try { onFrame({ session_id: s.id, ...s.lastFrame }); } catch { /* live view is best effort */ }
    return value;
  }

  function countShot(s) {
    const key = s.chat ?? "";
    const n = (shots.get(key) ?? 0) + 1;
    shots.set(key, n);
    return n <= limits().screenshots_per_chat;
  }

  async function observeTool(s, includeImage) {
    if (s.phase !== "working") return mcpResult(failure("paused_by_person", s.status || "Paused. Wait for the person to choose Continue.", { next: "computer_session_status" }));
    if (!countShot(s)) return mcpResult(failure("screenshot_limit", "This chat reached its screenshot limit. Finish the task another way."));
    const w = await native.call("window", { id: s.window.id });
    if (w.identity !== s.window.identity) { await stop(s, "closed", "limit", { reason: "window_unavailable" }); return mcpResult(failure("window_unavailable", "The session's app closed or restarted. Start a new session.")); }
    let obs = s.resumeObservation && Date.now() - s.resumeObservation.at < limits().observation_ttl_seconds * 1000 && s.resumeObservation.window.identity === w.identity ? s.resumeObservation : null;
    s.resumeObservation = null;
    if (!obs) obs = await observation(s, w);
    const state = await native.call("state");
    if (s.phase !== "working" || !state.available) return mcpResult(failure("paused_by_person", "Paused. Wait for the person to choose Continue."));
    s.window = w; s.lastOperation = Date.now(); s.observation = { ...obs, generation: s.generation };
    update();
    const l = limits();
    return mcpResult(observeResult({ session_id: s.id, observation: obs.ref, window: windowRef(w), screen: s.screen, scale: w.bounds.width / obs.model.width, dpi: w.dpi, monitor_id: w.monitor, cursor: await native.call("cursor").catch(() => null), os, expires_in_ms: l.observation_ttl_seconds * 1000, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)) }), includeImage ? { data: obs.bytes, mime: obs.mime } : null);
  }

  async function actTool(s, args) {
    const l = limits();
    const started = Date.now();
    const base = { session_id: s.id, request_id: args.request_id, intent: args.intent, started_at: new Date(started).toISOString(), os };
    const finish = (fields, image) => {
      const ended = Date.now();
      const result = actResult({ ...base, ...fields, ended_at: new Date(ended).toISOString(), duration_ms: ended - started });
      s.log.push({ at: base.started_at, action: describeAction(fields.action ?? args.action), intent: args.intent, ok: result.ok, code: result.code ?? null });
      if (s.log.length > 200) s.log.splice(0, s.log.length - 200);
      update();
      return mcpResult(result, image);
    };
    if (s.mode !== "control") return finish({ ok: false, code: "scope_denied", message: "This session is read-only (mode observe)." });
    if (s.phase !== "working") return finish({ ok: false, code: "paused_by_person", message: s.status || "Paused. Wait for the person to choose Continue.", next: "computer_session_status" });
    if (s.requests.has(args.request_id)) return finish({ ok: false, code: "duplicate_request", message: "Use a new request_id. Observe instead of repeating input." });
    s.requests.add(args.request_id);
    if (s.actions >= l.actions_per_session) { await stop(s, "stopped", "limit", { reason: "action_limit" }); return finish({ ok: false, code: "action_limit", message: `Start a new session after ${l.actions_per_session} actions.` }); }
    const observed = s.observation; s.observation = null;
    if (!observed || observed.id !== args.observation_id || Date.now() - observed.at > l.observation_ttl_seconds * 1000) return finish({ ok: false, code: "stale_observation", message: "Observe the window again before acting.", next: "computer_observe" });
    const valid = validateAction(args.action, observed.model);
    if (!valid.ok) return finish({ ok: false, code: valid.code, message: valid.message, frame_before: { sha256: observed.sha } });
    const action = valid.action;
    // At most actions_per_second.
    const gap = 1000 / l.actions_per_second - (Date.now() - s.lastAct);
    if (gap > 0) await sleep(gap);
    const current = await native.call("window", { id: s.window.id });
    const common = { action, frame_before: { sha256: observed.sha }, window: windowRef(current), screen: s.screen, dpi: current.dpi, monitor_id: current.monitor, scale: current.bounds.width / observed.model.width };
    if (current.identity !== observed.window.identity || JSON.stringify(current.bounds) !== JSON.stringify(observed.window.bounds)) return finish({ ...common, ok: false, code: "window_changed", message: "The window moved or changed. Observe again.", next: "computer_observe" });
    // Native coordinates; every point must hit the session's window (never another app's).
    const mapped = action.type === "drag" ? action.path.map((p) => mapPoint(p, observed.model, current.bounds)) : action.x !== undefined ? [mapPoint(action, observed.model, current.bounds)] : [];
    const hitAll = async (w) => !mapped.length || (await native.call("hit", { window: w, points: mapped.map((m) => m.screen) })).inside.every(Boolean);
    if (!(await hitAll(current))) return finish({ ...common, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the session's window. Observe again." });
    // Tolerant stale check (caret masked): the action targets what the model saw.
    if (action.type !== "wait") {
      const fresh = await native.call("capture", { window: current }, 15_000);
      // The observation's grid was taken after masking: mask the fresh frame the same way.
      paintBoxes(fresh, fresh.secure ?? []);
      const grid = luminanceGrid(fitLongSide(fresh, l.model_long_side));
      const compare = compareGrids(observed.grid, grid, { mask: observed.caret ? [observed.caret] : [] });
      if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
    }
    // No question: the person turned the real screen on, which is the consent (sessionRules).
    const approval = { asked: false, by: "setting" };
    const cursorBefore = await native.call("cursor").catch(() => null);
    // The engine gave up on this call: no input with nobody waiting for it.
    if (s.connection.cancelled) return finish({ ...common, approval, ok: false, code: "cancelled", message: "The request was cancelled.", may_have_acted: false });
    const point = mapped.length === 1 ? { model: mapped[0].model, screen: mapped[0].screen } : undefined;
    const path = action.type === "drag" ? mapped.map((m) => ({ model: m.model, screen: m.screen })) : undefined;
    const screenAction = action.type === "drag" ? { ...action, path: mapped.map((m) => m.screen) } : mapped.length ? { ...action, x: mapped[0].screen.x, y: mapped[0].screen.y } : action;
    s.actions += 1; s.lastAct = Date.now();
    try {
      if (action.type === "wait") await sleep(action.ms);
      else await native.call("act", { window: current, generation: s.generation, action: screenAction, deadline: s.expiresAt }, 30_000);
    } catch (error) {
      const code = error.code === "paused_by_person" ? "paused_by_person" : error.code ?? "input_interrupted";
      if (code === "paused_by_person") pause(s, "person", "Your input paused computer use. Choose Continue when you are ready.");
      return finish({ ...common, approval, point, path, cursor_before: cursorBefore, ok: false, code, message: error.message, may_have_acted: action.type !== "wait", next: "computer_session_status" });
    }
    s.lastOperation = Date.now();
    const cursorAfter = await native.call("cursor").catch(() => null);
    const done = { ...common, approval, point, path, cursor_before: cursorBefore, cursor_after: cursorAfter, ok: true, may_have_acted: action.type !== "wait" };
    if (!args.observe_after) return finish({ ...done, next: "computer_observe" });
    await sleep(action.type === "wait" ? 0 : 150);
    if (s.phase !== "working" || !countShot(s)) return finish({ ...done, next: "computer_observe" });
    try {
      const w = await native.call("window", { id: s.window.id });
      const after = await observation(s, w);
      s.window = w; s.observation = { ...after, generation: s.generation };
      return finish({ ...done, frame_after: { sha256: after.sha }, observation: after.ref, window: windowRef(w), next: "computer_act" }, { data: after.bytes, mime: after.mime });
    } catch (error) {
      return finish({ ...done, next: "computer_observe", message: `The action was done, but the window could not be observed: ${error.message}` });
    }
  }

  async function discover() {
    const ctx = (await context()) ?? {};
    const config = settings() ?? {};
    const windows = await native.call("list");
    const ancestors = await ancestorsOf(ctx);
    const apps = new Map();
    for (const w of windows) {
      const denied = deniedApp(appIdentity(w), { ancestors, self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });
      const key = `${w.appId}\0${w.pid}`;
      if (!apps.has(key)) apps.set(key, { app_id: w.appId, name: w.appName, pid: w.pid, ...(denied ? { available: false, reason: denied.reason } : {}), windows: [] });
      if (!denied) apps.get(key).windows.push({ id: String(w.id), title: w.title });
    }
    const screens = await native.call("screens").catch(() => []);
    const virtualOk = Boolean(virtual) && config.virtual_display !== false;
    return mcpResult({ ok: true, os, contract: CONTRACT_VERSION, screens: screens.map((s) => ({ id: String(s.id), width: s.width, height: s.height, scale: s.scale, primary: Boolean(s.primary) })), apps: [...apps.values()], modes: ["observe", "control"], real_screen: true, virtual_display: virtualOk, ...(virtualOk ? { virtual_hint: virtual.hint } : {}) });
  }

  /** The server's switch read just now: true only when it answered (no refresh, a false or a throw: not read). */
  async function freshRead() {
    if (typeof refresh !== "function") return false;
    return Promise.resolve().then(refresh).then((ok) => ok === true, () => false);
  }

  async function openSession(connection, args, verified) {
    // The server's switch and limits as of now (`call` read them), so a stricter value (off) applies at once.
    const config = settings() ?? {};
    const ctx = (await context()) ?? {};
    if (!config.enabled) return mcpResult(failure("disabled", config.message || "Computer use is turned off."));
    const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
    if (!rules.open) return mcpResult(failure(rules.code, rules.message));
    if (lease) return mcpResult(failure("session_busy", "Another computer use session is open. Close it first."));
    const windows = (await native.call("list")).filter((w) => w.appId === args.app_id && (args.pid === undefined || w.pid === args.pid));
    if (!windows.length) return mcpResult(failure("app_unavailable", "That app has no visible window. Open it yourself, then call computer_discover again."));
    const denied = deniedApp(appIdentity(windows[0]), { ancestors: await ancestorsOf(ctx), self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });
    if (denied) return mcpResult(failure(denied.code, denied.reason));
    const preferred = args.window_id !== undefined ? windows.find((w) => String(w.id) === args.window_id) : windows[0];
    if (!preferred) return mcpResult(failure("window_unavailable", "That window is not open. Call computer_discover again."));
    if (connection.cancelled) return mcpResult(failure("cancelled", "The request was cancelled."));
    const l = limits();
    const s = {
      id: randomUUID(), connection, window: preferred, windows, mode: args.mode, purpose: args.purpose, phase: "working", status: "",
      approval: "setting", chat: connection.internal === false ? null : ctx.chat ?? null, actions: 0, requests: new Set(), observation: null, resumeObservation: null, log: [],
      lastOperation: Date.now(), lastAct: 0, generation: 0, expiresAt: Date.now() + l.session_seconds * 1000, screen: await screenOf(preferred),
    };
    try {
      const current = await native.call("window", { id: preferred.id });
      if (current.identity !== preferred.identity) throw Object.assign(new Error("The app restarted. Start a new session."), { code: "window_unavailable" });
      if (s.mode === "control") await native.call("focus", { window: current });
      const started = await native.call("start");
      s.generation = started?.generation ?? 0;
      s.window = current; s.screen = await screenOf(current);
    } catch (error) {
      return mcpResult(failure(error.code ?? "window_unavailable", error.message));
    }
    lease = s.id; sessions.set(s.id, s);
    s.approved = { window_id: String(s.window.id), app_id: s.window.appId };
    emit(s, "opened", "agent", restartedNote ? { reason: "worker_restarted" } : {});
    restartedNote = false;
    // The person's real-screen setting is the approval: no question was asked.
    emit(s, "approved", "agent", { by_setting: true });
    update();
    return mcpResult({ ok: true, session_id: s.id, state: "active", mode: s.mode, approval: s.approval, window: windowRef(s.window), screen: s.screen, os, expires_in_seconds: l.session_seconds, actions_left: l.actions_per_session });
  }

  async function call(connection, name, rawArgs) {
    if (!TOOLS.includes(name)) return mcpResult(failure("unknown_tool", "Unknown computer use tool."));
    const valid = validateCall(name, rawArgs);
    if (!valid.ok) return mcpResult(failure(valid.code, valid.message));
    const args = valid.args;
    // The agent's own display: no native access to the person's screen is involved.
    if (virtual && name === "computer_open_session" && args.app_id.startsWith("virtual:")) {
      // The server's switch as of now, as for the person's screen.
      if (typeof refresh === "function") await Promise.resolve().then(refresh).catch(() => false);
      if (!(settings() ?? {}).enabled) return mcpResult(failure("disabled", settings()?.message || "Computer use is turned off."));
      const opened = await virtual.open(args);
      if (opened.ok) virtualOwners.set(opened.session_id, connection);
      return mcpResult(opened);
    }
    if (virtual && args.session_id && virtual.owns(args.session_id)) {
      if (virtualOwners.get(args.session_id) !== connection) return mcpResult(failure("session_unavailable", "This session has ended. Open a new one."));
      return virtual.call(name, args);
    }
    // The person's screen is off (the default): refused (and its windows not listed) before any desktop access.
    // It needs the server's yes for this account read just now (a cached answer, or none, is not enough).
    let verified = false;
    if (name === "computer_open_session" || name === "computer_discover") verified = await freshRead();
    if (name === "computer_open_session") {
      const config = settings() ?? {};
      const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
      if (!rules.open) return mcpResult(failure(rules.code, rules.message));
    }
    if (name === "computer_discover") {
      const config = settings() ?? {};
      const rules = sessionRules({ display: "real", server: config, optIn: config.real_screen_on === true, verified });
      if (rules.open && nativeDown) await ensureNative();
      const why = !rules.open ? rules.message : !availability.supported || nativeDown ? availability.error ?? "Desktop access is unavailable." : null;
      if (why) return mcpResult({ ok: true, os, contract: CONTRACT_VERSION, real_screen: false, real_screen_unavailable: why, screens: [], apps: [], modes: ["observe", "control"], virtual_display: Boolean(virtual) && config.virtual_display !== false, ...(virtual ? { virtual_hint: virtual.hint } : {}) });
    }
    if (nativeDown && !(await ensureNative())) return mcpResult(failure("native_unavailable", "Desktop access stopped. Try again in a few minutes, or restart the app."));
    if (!availability.supported) return mcpResult(failure(availability.code ?? "desktop_unavailable", availability.error ?? "Computer use is unavailable on this desktop."));
    if (name === "computer_discover") return discover();
    if (name === "computer_open_session") return openSession(connection, args, verified);
    const s = sessions.get(args.session_id);
    if (!s || s.connection !== connection) return mcpResult(failure("session_unavailable", "This session has ended. Open a new one."));
    // The server's latest answer no longer allows the person's screen: the session ends before anything else.
    if (!serverRealScreen(settings() ?? {}) && name !== "computer_close_session") {
      await stop(s, "stopped", "server");
      return mcpResult(failure("real_screen_unavailable", REAL_SCREEN_NOT_OPTED_IN_MESSAGE));
    }
    if (name === "computer_close_session") { await stop(s, "closed", "agent"); return mcpResult({ ok: true, session_id: s.id, state: "closed" }); }
    if (name === "computer_session_status") {
      const l = limits();
      return mcpResult({ ok: true, session_id: s.id, state: s.phase === "working" ? "active" : s.phase, mode: s.mode, approval: s.approval, message: s.status || undefined, actions_left: Math.max(0, l.actions_per_session - s.actions), seconds_left: Math.max(0, Math.ceil((s.expiresAt - Date.now()) / 1000)), window: windowRef(s.window), os });
    }
    if (name === "computer_observe") return observeTool(s, args.include_image);
    return actTool(s, args);
  }

  /** One MCP client (an engine connection): JSON-RPC in, JSON-RPC out. */
  /**
   * One MCP connection. `internal` false: a connection from outside the app's own engine (the copied CLI
   * connection), whose calls belong to no chat of this app.
   */
  function connect({ internal = true } = {}) {
    const connection = { id: randomUUID(), busy: false, closed: false, internal: internal !== false };
    return {
      id: connection.id,
      async handle(message) {
        if (!record(message) || message.jsonrpc !== "2.0" || !RPC.has(message.method)) return { jsonrpc: "2.0", id: message?.id ?? null, error: { code: -32601, message: "Method not available." } };
        if (message.method === "notifications/initialized") return null;
        if (message.method === "notifications/cancelled") {
          // The engine gave up on the call in flight (its timeout, or the person stopped the turn): a question
          // it is waiting on ends unanswered, so nothing opens or acts after it.
          const id = message.params?.requestId;
          if (id !== undefined && id !== null && id === connection.inflight) {
            connection.cancelled = true;
          }
          return null;
        }
        let result;
        try {
          if (message.method === "initialize") result = { protocolVersion: typeof message.params?.protocolVersion === "string" ? message.params.protocolVersion : "2025-06-18", capabilities: { tools: {} }, serverInfo: { name: SERVER_NAME, version: CONTRACT_VERSION }, instructions: INSTRUCTIONS };
          else if (message.method === "ping") result = {};
          else if (message.method === "tools/list") result = { tools: toolDefinitions({ limits: limits() }) };
          else {
            const name = message.params?.name, args = message.params?.arguments ?? {};
            const light = name === "computer_session_status" || name === "computer_close_session";
            if (connection.busy && !light) result = mcpResult(failure("busy", "Wait for the previous computer use call to finish."));
            else {
              if (!light) { connection.busy = true; connection.inflight = message.id ?? null; connection.cancelled = false; }
              try { result = await call(connection, name, args); } finally { if (!light) { connection.busy = false; connection.inflight = null; } }
            }
          }
        } catch (error) {
          result = mcpResult(failure(error.code ?? "desktop_error", error.message ?? "Computer use failed."));
        }
        return message.id === undefined ? null : { jsonrpc: "2.0", id: message.id, result };
      },
      async close() {
        connection.closed = true;
        for (const s of [...sessions.values()]) if (s.connection === connection) await stop(s, "closed", "agent", { reason: "disconnected" });
        for (const [id, owner] of [...virtualOwners]) if (owner === connection) { virtualOwners.delete(id); if (virtual?.owns(id)) await virtual.control({ session_id: id, action: "stop" }).catch(() => {}); }
      },
    };
  }

  /** Desktop transport: a loopback socket with a token, found through `<dir>/engine-connection.json` (the bundled relay reads it; the copied CLI connection reads `<dir>/cli-connection.json`). */
  async function listen(dir) {
    await mkdir(dir, { recursive: true, mode: 0o700 });
    // Two tokens: the app's own engine relay reads engine-connection.json; the copied CLI connection reads
    // cli-connection.json, and its connections are external (no chat). (A CLI connection copied
    // before these names pointed at connection.json, which is no longer written: it fails to connect.)
    const discoveryPath = path.join(dir, "engine-connection.json");
    const externalDiscoveryPath = path.join(dir, "cli-connection.json");
    const token = randomBytes(32).toString("hex");
    const externalToken = randomBytes(32).toString("hex");
    const sockets = new Set();
    const server = createServer((socket) => {
      if (closed || sockets.size >= 16) { socket.destroy(); return; }
      sockets.add(socket);
      let connection = null;
      let incoming = "", authenticated = false;
      const authTimer = setTimeout(() => socket.destroy(), 5000);
      socket.setEncoding("utf8");
      socket.once("close", () => { clearTimeout(authTimer); sockets.delete(socket); void connection?.close(); });
      socket.on("error", () => socket.destroy());
      socket.on("data", (chunk) => {
        incoming += chunk;
        if (incoming.length > 1_048_576) { socket.destroy(); return; }
        let end;
        while ((end = incoming.indexOf("\n")) >= 0) {
          const line = incoming.slice(0, end); incoming = incoming.slice(end + 1);
          let message; try { message = JSON.parse(line); } catch { socket.destroy(); return; }
          if (!authenticated) {
            const candidate = typeof message?.token === "string" ? Buffer.from(message.token) : Buffer.alloc(0);
            const matches = (expected) => candidate.length === expected.length && timingSafeEqual(candidate, Buffer.from(expected));
            const internal = matches(token);
            if (!internal && !matches(externalToken)) { socket.destroy(); return; }
            connection = connect({ internal });
            authenticated = true; clearTimeout(authTimer); continue;
          }
          void connection.handle(message).then((reply) => { if (reply && !socket.destroyed) socket.write(`${JSON.stringify(reply)}\n`); });
        }
      });
    });
    await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => resolve(undefined)); });
    const address = server.address();
    if (!address || typeof address === "string") throw new Error("Computer use could not open its local connection.");
    for (const [file, value] of [[discoveryPath, token], [externalDiscoveryPath, externalToken]]) {
      const temp = `${file}.${randomUUID()}`;
      await writeFile(temp, JSON.stringify({ version: 1, host: "127.0.0.1", port: address.port, token: value }), { mode: 0o600, flag: "wx" });
      await rename(temp, file);
    }
    return { discoveryPath, externalDiscoveryPath, close: async () => { for (const socket of sockets) socket.destroy(); server.close(); await unlink(discoveryPath).catch(() => {}); await unlink(externalDiscoveryPath).catch(() => {}); } };
  }

  /** Controls from the person (host channel only): stop | takeover | continue | hide | show. */
  async function control({ session_id, action }) {
    if (virtual?.owns(session_id)) return virtual.control({ session_id, action });
    const s = sessions.get(session_id);
    if (!s) throw new Error("This computer use session has ended.");
    if (action === "stop") return stop(s, "stopped", "person");
    if (action === "takeover") return pause(s, "person", "You took over. Choose Continue when you are ready.");
    if (action === "hide" || action === "show") { s.hidden = action === "hide"; update(); return undefined; }
    if (action !== "continue" || s.phase !== "paused") throw new Error("This action is unavailable.");
    const current = await native.call("window", { id: s.window.id });
    if (current.identity !== s.window.identity) { await stop(s, "closed", "limit", { reason: "window_unavailable" }); throw new Error("The app closed or restarted. Start a new session."); }
    if (s.mode === "control") await native.call("focus", { window: current });
    const started = await native.call("start");
    s.generation = started?.generation ?? s.generation;
    s.window = current; s.phase = "working"; s.status = ""; s.lastOperation = Date.now();
    emit(s, "continued", "person");
    // One observation now, so what the person changed is in the next result.
    try { s.resumeObservation = await observation(s, current); } catch { s.resumeObservation = null; }
    update();
    return undefined;
  }

  /** Ends every session (server switch off, app quitting). */
  async function stopAll(by = "server", event = "stopped") {
    for (const s of [...sessions.values()]) await stop(s, event, by);
    await virtual?.stopAll(by, event);
  }

  /** Ends the sessions on the person's screen only (the agent's own display keeps running). */
  async function stopRealScreen(by = "server", event = "stopped") {
    for (const s of [...sessions.values()]) await stop(s, event, by);
  }

  /** Whether the latest press (key or button) within `withinMs` was synthetic (injected by a program). */
  /**
   * Whether a physical key or button press (not injected by a program, not a remote-debugging
   * click that makes no OS input) came within `withinMs`: the person's own answer.
   */
  async function physicalInput(withinMs = 1500, { foregroundPid = null, rect = null, keys = ["Enter", "Space"] } = {}) {
    const value = await native.call("input-source").catch(() => null);
    if (!value || !value.at || value.synthetic || Date.now() - value.at >= withinMs) return false;
    // In the answering app's window (when the worker knows which app was in front).
    if (foregroundPid && Number.isInteger(value.foreground_pid) && value.foreground_pid !== foregroundPid) return false;
    // A pointer press must land on the pressed button (screen pixels); a key press must be one that presses a button.
    if (value.kind === "pointer" && rect) {
      if (!Number.isFinite(value.x) || !Number.isFinite(value.y)) return false;
      return value.x >= rect.x - 4 && value.x <= rect.x + rect.width + 4 && value.y >= rect.y - 4 && value.y <= rect.y + rect.height + 4;
    }
    if (value.kind === "key") return keys.includes(value.key);
    return !rect;
  }

  async function syntheticInput(withinMs = 5000) {
    const value = await native.call("input-source").catch(() => null);
    return Boolean(value && value.synthetic && Date.now() - value.at < withinMs);
  }

  /** Starts input watching for a pending approval (Windows hooks run only while needed). */
  async function watchInput() { await native.call("watch").catch(() => {}); }

  async function close() {
    if (closed) return;
    closed = true; clearInterval(ticker);
    await stopAll("agent", "closed");
    await virtual?.close();
    await native.close();
  }

  update();
  const allState = () => [...publicState(), ...(virtual ? virtual.state() : [])];
  return { get availability() { return availability; }, os, connect, listen, control, stopAll, stopRealScreen, syntheticInput, physicalInput, watchInput, state: allState, close };
}