Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

80.4 KB
Raw Blame
import { mkdir, rm, writeFile } from "node:fs/promises";
import { setDefaultAutoSelectFamily } from "node:net";
import { join } from "node:path";
import { type ServerWebSocket, serve } from "bun";
import type { Page } from "playwright";
import { downloadHeaders } from "../../shared/file-download";
import {
  isPersonBrowserId,
  RECORDER_DIR_ENV,
} from "../../shared/openbot-browser";
import {
  cutAtCodeUnits,
  parseAriaSnapshot,
  type SnapshotElement,
} from "./aria-snapshot";
import {
  actsOnTheComputer,
  isOpenPath,
  matchesToken,
  mutatesBrowser,
  offeredToken,
} from "./authorisation";
import { createBotBrowserHost, isBotBrowserPath } from "./bot-browser-host";
import { isPlainBotId, seatIdFor } from "./bot-id";
import { createBrowserHost, isBrowserPath } from "./browser-host";
import { browserRuntimeFromEnv } from "./browser-runtime";
import { detectChallenge } from "./challenge";
import {
  ControlError,
  ControlRequestError,
  NO_SECRET_PENDING,
  SnapshotRequiredError,
  TAKE_CONTROL_FIRST,
} from "./control";
import { handleEgressPolicyRequest, startEgressFilter } from "./egress";
import { enterText, FieldValueError } from "./enter-text";
import { identity } from "./identity";
import {
  assertPageAccess,
  navigateWebPage,
  type PagePurpose,
} from "./navigation";
import {
  computerOwner,
  NOT_ITS_OWN_COMPUTER_RECORD,
  PERSONS_OWN_BROWSER,
  personHoldsTheWheel,
  personMayUse,
  refusedHere,
} from "./person-access";
import { createProfiles, numberFromEnv, VIEWPORT } from "./profiles";
import { createRealBrowserHost, isRealBrowserPath } from "./real-browser";
import {
  parseExecTimeout,
  parseInputMessage,
  parseNavigateUrl,
  parseScrollDelta,
} from "./request-validation";
import { type InputMessage, startScreencast } from "./screencast";
import {
  HIDDEN_FIELD_SELECTOR,
  hiddenValues,
  inputFills,
  keyFills,
  MASKED_VALUE,
  markElementSecret,
  markPersonFilled,
  maskSensitiveValues,
  PERSON_FILLED_ATTRIBUTE,
  PERSON_FILLED_VALUE,
  SECRET_ATTRIBUTE,
  unmarkPersonFilled,
} from "./secret-masking";
import { type BotSession, createSessions } from "./sessions";
import {
  fillSignIn,
  fillSignInCode,
  parseSignInCode,
  parseSignInFill,
} from "./sign-in";
import { startVirtualDisplay } from "./virtual-display";
import { applyWebBotAuthPush } from "./web-bot-auth";
import {
  WorkspaceFileError,
  WorkspaceFileNotFoundError,
  WorkspaceFileTooLargeError,
  WorkspacePathError,
} from "./workspace";
import { createWorkspaces } from "./workspaces";

/**
 * The Bot's computer: one long-lived browser, reachable over HTTP.
 *
 * Acting on a page lives in this process because only this process holds the browser. In the
 * intended deployment path, the server gateway decides whether an action may run and records the
 * audit row before calling this process. This process has no policy engine and no audit trail of its
 * own; its direct-port boundary is the computer token.
 *
 * `/files/read`, `/files/download` and `/files/write` reach the durable workspace volume, confined
 * to it by workspace.ts. Reading and writing are the two operations a Bot needs to keep notes between
 * turns; download returns the exact bytes so a PDF, image or archive is not damaged by text decoding.
 *
 * Elements are addressed by reference, not by pixel. `/snapshot` stamps every interactive element
 * with a ref and hands back a compact list; `/click` and `/type` take one of those refs. That is the
 * accessibility-tree-first driver this is built on, and it is why filling in a form needs no
 * vision model at all: the Bot reads a list of fields rather than squinting at a picture and guessing
 * coordinates. Pixels remain the eventual fallback for canvas-style pages that expose no elements.
 *
 * One browser stays open so state survives between
 * turns: a session it signed into an hour ago is still signed in now. Launching per request would
 * make every task start from a cold, logged-out browser, which is the behaviour we are specifically
 * trying not to have.
 *
 * It authenticates its caller: every request must present the secret below, and the process refuses
 * to start without one. That is a lock on the door rather than a reason to put the door somewhere
 * public. It still belongs on the deployment network, behind the server that decides who is asking.
 */

/**
 * The secret every caller must present.
 *
 * This process drives a browser that holds real logins. Policy, audit, actor identity and SPIFFE
 * identity live in the server and are not on the direct computer port.
 *
 * Refusing to start without a token makes missing authentication a deployment failure, never an open
 * computer.
 */
const COMPUTER_TOKEN = process.env.COMPUTER_TOKEN?.trim();
if (!COMPUTER_TOKEN) {
  console.error(
    "COMPUTER_TOKEN is not set. This process drives a browser holding real logins and will not start without the secret its caller must present.",
  );
  process.exit(1);
}

/*
 * No multi-address connect anywhere in this process. Bun turns it on by default, and its timer can
 * throw from `internalConnectMultipleTimeout` after the connection is gone, which exits the process
 * (oven-sh/bun#25633, fixed in Bun 1.4.0, kept as defence in depth). The egress filter already
 * connects one checked address at a time (see `connectInTurn`); this covers every other connect here,
 * such as reaching an upstream proxy by name.
 */
setDefaultAutoSelectFamily(false);

const RUNTIME = browserRuntimeFromEnv(process.env);
/** Whose this computer is, which decides whether a person's browser may run on it. See person-access.ts. */
const OWNER = computerOwner(process.env);
const BROWSER_MODE = RUNTIME.mode;
/*
 * The display this process was started on, kept before the virtual one replaces it. A person's
 * native browser on OpenBot Desktop opens on the person's own desktop, never on the Xvfb display that
 * a Bot's headed browser uses on Linux. See `personBrowserEnv`.
 */
const PERSON_DISPLAY = {
  DISPLAY: process.env.DISPLAY,
  WAYLAND_DISPLAY: process.env.WAYLAND_DISPLAY,
};
const VIRTUAL_DISPLAY = await startVirtualDisplay(
  RUNTIME.useVirtualDisplay ? "headed" : "headless",
);
if (VIRTUAL_DISPLAY) process.env.DISPLAY = VIRTUAL_DISPLAY.name;
console.info(
  JSON.stringify({
    type: "computer-browser-mode",
    mode: BROWSER_MODE,
    display: VIRTUAL_DISPLAY?.name ?? null,
  }),
);

/*
 * A whole port in range, or the default: a fraction never binds and an out-of-range one
 * misbinds at boot, which is a deployment failure instead of the documented fallback.
 */
const PORT = numberFromEnv("PORT", 4100, { min: 1, max: 65535 });
const NAVIGATION_TIMEOUT_MS = numberFromEnv("NAVIGATION_TIMEOUT_MS", 30000);

/**
 * How long one action waits for its element.
 *
 * Much shorter than a navigation. Playwright waits for a control to become clickable, which is the
 * behaviour we want, but a ref that no longer resolves would otherwise hang for the full navigation
 * timeout before saying so, and the person is sitting watching a screen that is not changing.
 */
const ACTION_TIMEOUT_MS = numberFromEnv("ACTION_TIMEOUT_MS", 10000);

/**
 * How much page text a navigation hands back.
 *
 * Bounded because a page can be megabytes and the text goes into a model's context, where a single
 * unbounded page can push the rest of the conversation out. Generous enough that the visible part of
 * an ordinary page arrives whole, which is what the answer is usually made of.
 */
const TEXT_EXTRACT_LIMIT = 6000;

/**
 * Which snapshot the caller's refs came from.
 *
 * Kept as a caller-facing guard even though Playwright enforces the real thing underneath. The
 * published tool contract says an action carries the `snapshotId` it got, and a mismatch is answered
 * with "take a new snapshot", which is a clearer message for a model than an element that merely fails
 * to resolve. Playwright's `aria-ref` engine is the runtime enforcement: it resolves a ref only against
 * the most recent snapshot, only while the element is still connected to the document, and it mints a
 * new ref if an element's role or accessible name changed, so a recycled node cannot inherit an old one.
 *
 * The counter, the run it belongs to, and the sessions holding both live in their own module: the
 * rules about when a run changes are testable there, and not here, because this file launches a
 * browser the moment it is imported.
 */
const sessions = createSessions({
  isLive: (botId) => profiles.isLive(botId),
  profilesDirectory: process.env.PROFILES_DIR?.trim() || "/profiles",
  // A recording in the Bot's browser is the person's who holds the wheel, and ends the moment they no longer do.
  onWheelLeft: (botId) => botBrowser.wheelLeft(botId),
});

/**
 * Which Bot a request is for, and whose seat on this computer it reaches.
 *
 * The Bot is sent by the server as a header on every call. Absent means the caller does not know or
 * does not care, such as a health check, and that gets the default computer rather than an error,
 * because refusing it would make the container undemonstrable on its own.
 *
 * The person is sent only to a computer many people share (Helm's `computers.mode: shared`), and
 * there the seat is the pair: each person's profile, folder and session for that Bot are their own,
 * so one person's logins never reach another. A computer of one person's own is sent the Bot alone,
 * and its seat is the Bot. See `seatIdFor`.
 *
 * `seat` is undefined when either half is not a usable id, which the caller refuses.
 */
function seatOf(
  request: Request,
  query?: URLSearchParams,
): { bot: string; seat: string | undefined } {
  const bot =
    request.headers.get("x-openbot-bot-id")?.trim() ||
    query?.get("bot")?.trim() ||
    DEFAULT_BOT_ID;
  const person =
    request.headers.get("x-openbot-person-id")?.trim() ||
    query?.get("person")?.trim() ||
    undefined;
  try {
    return { bot, seat: seatIdFor(bot, person) };
  } catch {
    return { bot, seat: undefined };
  }
}

/**
 * The Bot's durable files.
 *
 * Rooted at WORKSPACE_DIR, which the image creates and docker-compose mounts as a volume, so what a
 * Bot saves outlives the container. Each folder is built the first time its Bot uses it and kept, so
 * a request adds no syscall after that. Which folder a Bot gets lives in workspaces.ts; everything
 * about why confinement is harder than it looks lives in workspace.ts.
 */
/**
 * The two directories this process cannot do its job without, checked before it says it is ready.
 *
 * WHY A CHECK AND NOT A CRASH LATER. Neither of these fails loudly on its own. A persistent
 * Chromium profile it cannot write is not an error to Chromium: `launchPersistentContext` falls
 * back to a throwaway profile, so the Bot is signed out of everything and the container is healthy.
 * `profiles.ts` swallows the EACCES on cleanup for its own good reasons. So the failure that
 * actually happens in the field is a green pod and an empty profile, which is the worst shape a
 * failure can take: nothing to read, and a week of logins gone.
 *
 * WHEN IT HAPPENS. The volume is created root-owned and something has to hand it over. In Docker
 * the image does it at build time; on Kubernetes the kubelet does it from `fsGroup`, which it
 * applies only where the volume plugin reports it can. `hostPath` reports it cannot, and that is
 * what rancher/local-path-provisioner hands out by default, which is the default StorageClass on
 * k3s. There, uid 1001 gets a directory it cannot even list.
 *
 * Exiting non-zero turns that into a CrashLoopBackOff with a readable reason, which an operator can
 * act on: chown the directory, or set `computers.podSecurityContext: null`.
 */
async function assertWritable(label: string, directory: string) {
  const probe = join(directory, `.openbot-write-probe-${process.pid}`);
  try {
    await mkdir(directory, { recursive: true });
    await writeFile(probe, "");
    await rm(probe, { force: true });
  } catch (error) {
    console.error(
      `${label} at ${directory} is not writable by uid ${process.getuid?.() ?? "unknown"}: ${String(error)}. ` +
        "A Bot's files and browser profile live here, and a profile this process cannot write is one Chromium silently replaces with a throwaway, so this refuses to start instead. " +
        "On Kubernetes this usually means the volume's storage class does not apply fsGroup, which hostPath-backed provisioners such as local-path do not: chown it to the pod's uid, or set computers.podSecurityContext to null.",
    );
    process.exit(1);
  }
}

await assertWritable(
  "The workspace",
  process.env.WORKSPACE_DIR?.trim() || "/workspace",
);
await assertWritable(
  "The browser profiles directory",
  process.env.PROFILES_DIR?.trim() || "/profiles",
);

// One folder per Bot where one process serves many; see workspaces.ts.
const workspaces = createWorkspaces(
  process.env.WORKSPACE_DIR?.trim() || "/workspace",
  process.env.COMPUTER_SEPARATE_WORKSPACES?.trim().toLowerCase() === "on",
);

/**
 * Who has the wheel, as a state machine in its own module.
 *
 * The state machine lives in `control.ts` so it can be tested without importing Playwright.
 */

/**
 * The Bot's browser and the profile that outlives it. See profiles.ts.
 *
 * `chromium.launch()` gives a fresh anonymous profile every time. Persistent profiles live on a
 * mounted volume so sign-in state survives the container.
 */
/**
 * The browsers, and what a closing one takes with it.
 *
 * Every close announces itself here, whether it came from a request, the cap, or the idle sweep, and
 * the live screen watching that Bot comes down with it. Hanging this off the close rather than
 * calling it from the stop and reset handlers is what covers the two closes no request makes: a
 * viewer that outlived one of those kept a 1Hz loop asking for a page, which starts a browser, so
 * the Bot was immune to the idle timeout and came straight back after a cap eviction.
 *
 * `sessions.get`, never `sessions.for`: a Bot with no session has nobody watching, and inventing one
 * here would put an entry in the map on the path that closes browsers, which is where the map is
 * meant to shrink.
 */
// Every browser and shell launched below goes through the policy filter, which still chains to the
// per-Bot upstream proxy. Started before any browser exists so nothing escapes it.
// Only the running computer points its shell at it: a test that imports this module shares its
// process with every later test.
await startEgressFilter({ forShell: import.meta.main });
const profiles = createProfiles(
  process.env.PROFILES_DIR?.trim() || "/profiles",
  async (botId) => {
    // First, while the browser is still there: a recording in it ends, and its Recorder lets the person go.
    await botBrowser.browserClosing(botId);
    if (sessions.get(botId)) sessions.renewRun(botId);
    await sessions.get(botId)?.viewer.releaseAll(COMPUTER_STOPPED);
  },
  {
    personDisplay: PERSON_DISPLAY,
    // A Bot's browser a person is working in, or has been asked to, is never closed for memory, and
    // none of its tabs are closed for being idle. A person's own browser has no Bot's wheel to hold:
    // it is never closed for memory whoever is at it (`chooseForMemory`), and its tabs are never swept.
    heldByPerson: (botId) =>
      !isPersonBrowserId(botId) && sessions.personHasTheWheel(botId),
  },
);

/**
 * The OpenBot Browser: a person's own browser with the Recorder in it, opened for them by the
 * server. See browser-host.ts, and person-access.ts for what a person's browser id may do here.
 */
/**
 * A browser being cleared by the person working it is forgotten as a closed one is (a new run, no page), with the wheel
 * left where it is: the browser starts again under them (profiles.ts, `clear`).
 */
const forgetClearedBrowser = (botId: string) => {
  if (sessions.get(botId)) sessions.renewRun(botId, { keepWheel: true });
};

const browserHost = createBrowserHost({
  profiles,
  recorderDir: () => process.env[RECORDER_DIR_ENV],
  navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
  forget: forgetClearedBrowser,
});

/**
 * A Bot's browser worked by a person holding its wheel: the bar above its screen, and the Recorder loaded in it,
 * which records that person and nobody else. See bot-browser-host.ts.
 */
const botBrowser = createBotBrowserHost({
  profiles,
  control: (botId) => sessions.for(botId).control,
  // The bar is driving, as the person's input on the screen is, so a page restore never moves a page under them.
  drove: (botId) => {
    sessions.for(botId).humanInputs += 1;
  },
  forget: forgetClearedBrowser,
  navigationTimeoutMs: NAVIGATION_TIMEOUT_MS,
  ...(OWNER.kind === "shared"
    ? { recordRefused: NOT_ITS_OWN_COMPUTER_RECORD }
    : {}),
});

/**
 * The person's own everyday Chrome or Edge, on OpenBot Desktop's computer only: brought forward, or
 * handed the Recorder's store page. See real-browser.ts.
 */
const realBrowser = createRealBrowserHost({
  backend: RUNTIME.backend,
  display: PERSON_DISPLAY,
});

/**
 * The id normally arrives as a header on every request. This is the fallback for a caller that has no
 * Bot to name, such as a health check, so the container stays demonstrable on its own rather than
 * refusing everything that is not the server.
 */
const DEFAULT_BOT_ID = (() => {
  const configured = process.env.COMPUTER_BOT_ID ?? "shared";
  // At boot rather than per request. This is the id every unheadered call falls back to, so a value
  // the path rules refuse would answer 400 to everything and read as a broken computer.
  if (!isPlainBotId(configured)) {
    throw new Error(
      "COMPUTER_BOT_ID may contain only letters, digits, hyphen and underscore, and must start with a letter or digit.",
    );
  }
  return configured;
})();

async function currentPage(
  botId: string,
  purpose: PagePurpose = "read",
): Promise<Page> {
  const session = sessions.for(botId);
  const page = await profiles.page(botId);
  sessions.observeBrowser(botId, page.context());
  // A Bot's tools read web pages only. A person's browser is theirs, on whatever page they are on.
  if (!isPersonBrowserId(botId))
    assertPageAccess(RUNTIME.backend, page.url(), purpose);
  // A ref names an element on the page it was taken from, so moving to a window the site opened has
  // to retire the outstanding ones exactly as a navigation does, or a click lands on the wrong document.
  if (session.livePage && session.livePage !== page) session.snapshotId += 1;
  session.livePage = page;
  return page;
}

/** A browser that has not been sent anywhere, which is what a restarted one shows. */
function isBlankPage(url: string): boolean {
  return (
    url === "" ||
    url === "about:blank" ||
    url.startsWith("chrome://newtab") ||
    url.startsWith("chrome://new-tab-page")
  );
}

/**
 * Put the page a handoff is about back in front of the person, when the browser came back blank.
 *
 * A browser restarts underneath a waiting request more often than it should: Chromium crashing, the
 * idle sweep or the cap closing it, the whole computer restarting. Each of those relaunches onto
 * `about:blank`, and before this a person who took control was handed a blank screen with nothing to
 * solve while the Bot waited for them to solve it. The request carries the page it was raised on, so
 * the computer opens it again first. Through the same navigation the Bot uses, so the egress filter
 * and the page-access rule still decide whether it may load.
 *
 * Only a blank browser is touched. A person or a site that moved the page somewhere else meant to.
 * Best effort: a page that will not load is logged and the handoff goes ahead, because a person in
 * front of a blank browser can still type the address, and a refused handoff helps nobody.
 */
const restoringPages = new Set<string>();
async function restoreHandoffPage(botId: string): Promise<void> {
  // One at a time per Bot: the help call and "Take control" both start one, and the second would
  // navigate again over the first while it is still loading.
  if (restoringPages.has(botId)) return;
  const session = sessions.for(botId);
  const want = session.control.pageToRestore();
  if (!want) return;
  // Run in the background, so the help call and "Take control" answer at once. Starting the browser
  // can take a while; by the time it is up the person may already be driving, or the request over.
  const inputsBefore = session.humanInputs;
  restoringPages.add(botId);
  try {
    const target = await currentPage(botId);
    if (!isBlankPage(target.url())) return;
    if (
      session.humanInputs !== inputsBefore ||
      session.control.pageToRestore() !== want
    )
      return;
    await navigateWebPage(target, want, RUNTIME.backend, NAVIGATION_TIMEOUT_MS);
    // A new document, so every ref handed out before it is retired, as after any navigation.
    session.snapshotId += 1;
    console.info(
      JSON.stringify({
        type: "computer-handoff-page-restored",
        botId,
        origin: new URL(want).origin,
      }),
    );
  } catch (error) {
    console.info(
      JSON.stringify({
        type: "computer-handoff-page-not-restored",
        botId,
        reason: error instanceof Error ? error.message : String(error),
      }),
    );
  } finally {
    restoringPages.delete(botId);
  }
}

/**
 * The page as text, the way a reader sees it.
 *
 * Script and style bodies are dropped rather than included: they are the bulk of a modern page and
 * none of it is what anybody asked about, so leaving them in spends the extract on noise and pushes
 * the actual article past the limit.
 */
async function readablePageText(
  target: Page,
): Promise<{ text: string; truncated: boolean }> {
  const raw = await target.evaluate(
    (hidden) => {
      const clone = document.body?.cloneNode(true) as HTMLElement | undefined;
      if (!clone) return "";
      for (const node of clone.querySelectorAll(
        "script, style, noscript, svg",
      )) {
        node.remove();
      }
      /*
       * An editor's words are page text, where a field's value is not, so an editor a secret was typed
       * into, or a person filled in while they held the wheel, says it holds something and not what
       * (see secret-masking.ts).
       */
      for (const { selector, instead } of hidden)
        for (const node of clone.querySelectorAll(selector))
          if (
            !(node instanceof HTMLInputElement) &&
            !(node instanceof HTMLTextAreaElement) &&
            !(node instanceof HTMLSelectElement)
          )
            node.textContent = instead;
      return clone.innerText ?? "";
    },
    [
      { selector: `[${SECRET_ATTRIBUTE}]`, instead: MASKED_VALUE },
      {
        selector: `[${PERSON_FILLED_ATTRIBUTE}]:not([${SECRET_ATTRIBUTE}])`,
        instead: PERSON_FILLED_VALUE,
      },
    ],
  );

  const collapsed = raw.replace(/\n{3,}/g, "\n\n").trim();
  return {
    /*
     * Cut between characters, not through one. #539 fixed this for a control's name and value in the
     * snapshot and left the page text, which is the same bug at thirty times the length: `slice`
     * counts UTF-16 code units, an emoji is two, and a limit landing between the halves hands the Bot
     * a lone high surrogate that reads as U+FFFD — a character that is not on the page.
     *
     * More likely to bite here than there, for the reason the limit is bigger. A 200-unit control
     * name rarely reaches an emoji; 6000 units of somebody's page usually passes through several, and
     * whether the cut lands mid-character is decided by whatever was above it.
     */
    text: cutAtCodeUnits(collapsed, TEXT_EXTRACT_LIMIT),
    // Unaffected by the line above: dropping one more unit cannot make an over-limit string fit.
    truncated: collapsed.length > TEXT_EXTRACT_LIMIT,
  };
}

/**
 * Describe everything on the page a Bot can act on.
 *
 * Uses Playwright's AI snapshot rather than stamping attributes into the DOM. `ariaSnapshot` keeps
 * refs outside the page, survives framework re-renders, resolves accessible names, reports current
 * values and checked state, filters to actionable elements and descends into iframes.
 */
async function snapshotPage(
  session: BotSession,
  target: Page,
): Promise<{
  snapshotId: number;
  url: string;
  title: string;
  elements: SnapshotElement[];
  truncated: boolean;
}> {
  const snapshotId = ++session.snapshotId;
  const before = session.control.get();
  const yaml = await target.ariaSnapshot({ mode: "ai" });
  const title = await target.title();
  if (
    session.snapshotId === snapshotId &&
    before.holder === "bot" &&
    !before.requested
  )
    session.control.snapshotTaken();
  const parsed = parseAriaSnapshot(yaml);
  return {
    snapshotId,
    url: target.url(),
    title,
    truncated: parsed.truncated,
    // A password field's value is in the aria snapshot in plain text, and so is anything a person
    // typed while they held the wheel. See secret-masking.ts.
    elements: maskSensitiveValues(
      parsed.elements,
      await hiddenValues(target, parsed.elements),
    ),
  };
}

/**
 * Resolve a ref to a locator, refusing anything from a superseded snapshot.
 *
 * `aria-ref=` is a first-party Playwright selector engine, and it is the same one its MCP server uses.
 * The generation check here is the caller-facing half; see the note on `snapshotId` for why both exist.
 */
function locateRef(
  session: BotSession,
  target: Page,
  ref: string,
  expectedSnapshotId: number | undefined,
) {
  if (
    expectedSnapshotId !== undefined &&
    expectedSnapshotId !== session.snapshotId
  ) {
    throw new StaleSnapshotError(
      `That list of elements is out of date: it was taken for snapshot ${expectedSnapshotId} and the page is now at ${session.snapshotId}. Take a new snapshot and use the refs from it.`,
    );
  }
  return target.locator(`aria-ref=${ref}`);
}

/**
 * The element, or a refusal that says what to do about it.
 *
 * A generation check is not an existence check. A ref from
 * the current snapshot that names nothing on the page, because a model invented it or because the
 * page moved on without a new snapshot being taken, passes `locateRef` and then simply waits. The
 * action times out, and the caller gets a generic failure carrying Playwright's internal call log
 * instead of the actionable answer: take a fresh snapshot.
 *
 * `count()` resolves immediately rather than waiting, so a ref that names nothing is refused in
 * milliseconds instead of holding the action open for the full timeout.
 */
async function resolveRef(
  session: BotSession,
  target: Page,
  ref: string,
  expectedSnapshotId: number | undefined,
) {
  const locator = locateRef(session, target, ref, expectedSnapshotId);
  if ((await locator.count()) === 0) {
    throw new StaleSnapshotError(
      `Nothing on this page has the ref ${ref}. Take a new snapshot and use the refs it returns.`,
    );
  }
  return locator;
}

class StaleSnapshotError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "StaleSnapshotError";
  }
}

function json(body: unknown, status = 200): Response {
  return new Response(JSON.stringify(body), {
    status,
    headers: { "content-type": "application/json" },
  });
}

/** Since when a person's browser has been theirs, which is always, so since this process started. */
const PROCESS_STARTED_AT = new Date().toISOString();

/** What a person watching is told when the browser they were watching went away. */
const COMPUTER_STOPPED =
  "This computer stopped, so the screen ended. Start it again to carry on watching.";

/** What a person is told when their screen is still opening and they have already started typing. */
const SCREEN_STILL_STARTING =
  "This screen is still starting. Try that again in a moment.";

/** What a person is told when they act on a screen that is no longer theirs, or no longer anything. */
const SCREEN_NO_LONGER_LIVE =
  "This screen is no longer live. Reopen it to carry on watching.";

/** How often the cast checks that it is still showing the page the Bot is on. */
const FOLLOW_INTERVAL_MS = 1_000;

/**
 * After a person's input on a Bot's screen lands: the field it filled is theirs, and the Bot reads only
 * that it was filled (see secret-masking.ts). A person's own browser has no Bot reading it. Never fails
 * the input, which has landed: a field that could not be marked is said in the log.
 */
async function notePersonFilled(
  botId: string,
  fills: "typed" | "chose" | null,
): Promise<void> {
  if (!fills || isPersonBrowserId(botId)) return;
  try {
    await markPersonFilled(await currentPage(botId), fills === "typed");
  } catch (error) {
    console.error(
      JSON.stringify({
        type: "computer-person-filled-not-marked",
        botId,
        error: String(error),
      }),
    );
  }
}

/** What a live-screen socket carries: the Bot whose screen it is showing. */
type StreamData = {
  botId: string;
  /** The person's input so far, applied one message after another. */
  inputs?: Promise<void>;
};

/**
 * Apply one thing the person did on the live screen, after asking whether the screen is theirs and
 * whether they may drive. See the socket's `message` below for why one at a time.
 */
async function applyInput(
  ws: ServerWebSocket<StreamData>,
  raw: string | Buffer,
): Promise<void> {
  const session = sessions.for(ws.data.botId);
  /*
   * Whose screen this is, asked before anything is done with the input.
   *
   * A superseded socket used to dispatch through whatever the session held, so a replaced
   * window's typing landed in the page the current viewer was watching. It heard nothing about
   * it either, because the old missing-viewer check returned before reaching anything that could
   * report, which is why this answers the sender rather than returning quietly.
   *
   * Starting and gone are told apart deliberately. Both own no cast, and answering them the same
   * way tells somebody whose screen is still opening that their session ended.
   */
  const standing = session.viewer.standingOf(ws);
  if (standing.state !== "casting") {
    ws.send(
      JSON.stringify({
        type: "error",
        error:
          standing.state === "starting"
            ? SCREEN_STILL_STARTING
            : SCREEN_NO_LONGER_LIVE,
      }),
    );
    return;
  }
  let message: InputMessage;
  try {
    const parsed: unknown = JSON.parse(String(raw));
    const validated = parseInputMessage(parsed);
    if (!validated.ok) {
      ws.send(JSON.stringify({ type: "error", error: validated.error }));
      return;
    }
    message = validated.message;
  } catch {
    // The validated-but-wrong branch above sends an error frame; unparseable input used to
    // be dropped silently, so a buggy surface saw input "ignored" with no diagnostic.
    ws.send(JSON.stringify({ type: "error", error: "Input is not JSON." }));
    return;
  }
  // A person's input is accepted only while they hold the wheel. The socket being open is not permission:
  // without this check, anything that could reach this port could drive the browser while a Bot
  // was working, which is the one thing the control state exists to prevent.
  //
  // Owning the screen is not permission either, which is why this stands after the ownership
  // question above and not instead of it: the two refuse different things, and the one asked
  // first only decides whether the input has anywhere to land.
  //
  // Refuse with an error so the surface can explain why input is ignored.
  //
  // A person's own browser needs no handover: nobody else works in it, so it starts, and stays,
  // with the person holding the wheel.
  if (!isPersonBrowserId(ws.data.botId) && !session.control.humanMayDrive()) {
    ws.send(JSON.stringify({ type: "error", error: TAKE_CONTROL_FIRST }));
    return;
  }
  session.humanInputs += 1;
  try {
    await standing.cast.send(message);
    await notePersonFilled(ws.data.botId, inputFills(message));
  } catch (error) {
    // Reported rather than swallowed. A dispatch that fails means the person's input did nothing,
    // and they must not be left believing it landed.
    console.error(
      JSON.stringify({
        type: "screencast-input-error",
        message: message.type,
        error: String(error),
      }),
    );
    ws.send(
      JSON.stringify({
        type: "error",
        error: describe(error, "That input could not be applied."),
      }),
    );
  }
}

serve<StreamData>({
  port: PORT,
  ...(RUNTIME.hostname ? { hostname: RUNTIME.hostname } : {}),
  idleTimeout: 120,
  /**
   * The live screen, pushed by Chrome rather than polled.
   *
   * Upgraded here rather than served as HTTP because the whole point is that frames arrive when the
   * page changes and input goes back over the same connection. See screencast.ts for why polling was
   * not good enough once a person had to type into this.
   */
  websocket: {
    async open(ws) {
      const session = sessions.for(ws.data.botId);
      /*
       * Claimed before anything is awaited, and that order is the fix.
       *
       * Below this line the Bot's browser may have to be launched, which takes long enough for a
       * client to connect and go away inside it. A close arriving in that window used to find nothing
       * installed and so did nothing, while this function carried on to install a cast and a 1Hz
       * interval for a socket that had already gone: no second close ever came, and the interval went
       * on relaunching a browser somebody had stopped. With the claim taken first there is always
       * something for that close to release, and everything below goes through the claim and is
       * refused once it is gone.
       */
      const claim = session.viewer.claim(ws, (reason) => {
        try {
          ws.send(JSON.stringify({ type: "error", error: reason }));
        } catch {
          // Best effort. The socket may already be gone, which is not a reason to stop tearing down.
        }
      });
      try {
        const send = (frame: unknown) => {
          // A closed socket starts a fresh cast on the next connection.
          try {
            ws.send(JSON.stringify(frame));
          } catch {
            void session.viewer.release(ws);
          }
        };

        /*
         * The cast follows the Bot's current page. Re-checking also handles a page being closed
         * underneath us without a listener per page.
         */
        let casting: Page | undefined;
        const attach = async () => {
          const target = await currentPage(ws.data.botId);
          if (target === casting) return;
          const cast = await startScreencast(target, send);
          // The claim stops a cast it refuses, so a launch that lost the screen leaks nothing.
          if (!(await claim.install(cast))) return;
          casting = target;
        };

        await attach();
        const follow = setInterval(() => {
          void attach().catch(() => undefined);
        }, FOLLOW_INTERVAL_MS);
        if (!claim.setFollow(() => clearInterval(follow))) return;
      } catch (error) {
        /*
         * Released before the socket is told, because this path is reachable in exactly the timing
         * the claim exists for. A claim left behind here would keep the session occupied for the life
         * of the process, and the sweep in sessions.ts could never take it: the unbounded growth that
         * function was written to stop, reintroduced by the error path of the fix for it.
         */
        await session.viewer.release(ws);
        ws.send(
          JSON.stringify({
            type: "error",
            error: describe(error, "The screen could not be started."),
          }),
        );
        ws.close();
      }
    },

    /*
     * One message at a time, in the order the person sent them. Applying one is a trip to the
     * browser, and a message whose trip is shorter than the one before it must still land after it:
     * applied as each was ready, a key came up before it went down (every key up of a word before
     * any of its key downs, measured with a page busy 20 ms per key down). Nothing else here holds a
     * message, and a message refused or failed says so and lets the next one through.
     */
    message(ws, raw) {
      ws.data.inputs = (ws.data.inputs ?? Promise.resolve())
        .then(() => applyInput(ws, raw))
        .catch((error: unknown) => {
          console.error(
            JSON.stringify({
              type: "screencast-input-error",
              error: String(error),
            }),
          );
        });
    },

    async close(ws) {
      // Names the socket, so it can only ever give up its own screen. A superseded socket closing
      // after its replacement has started releases nothing; see viewer.ts.
      //
      // `get`, not `for`: a socket closing for a Bot with no session has nothing to release,
      // and creating one here would add a map entry on a teardown path, which is the direction the
      // map is meant to shrink in.
      await sessions.get(ws.data.botId)?.viewer.release(ws);
    },
  },
  async fetch(request, server) {
    const url = new URL(request.url);

    /*
     * Nothing below this line happens for an untrusted caller.
     *
     * `/health` is the single exception: it names no Bot, touches no browser and reports nothing but
     * whether this process is up, and a container orchestrator has to be able to ask that without
     * holding a secret.
     *
     * The websocket upgrade is checked here too. A browser cannot set headers on an upgrade, so the
     * stream carries the token as a query
     * parameter the same way it already carries the Bot.
     */
    if (
      !isOpenPath(url.pathname) &&
      !matchesToken(COMPUTER_TOKEN, offeredToken(request.headers, url))
    ) {
      // Says nothing about what is here. A refusal that describes the endpoint it is protecting is a
      // directory listing for whoever is knocking.
      return json({ error: "Not authorised." }, 401);
    }

    // Resolved once per request. Everything below that touches a browser, a takeover or a snapshot
    // goes through this seat's session, so there is no path where one Bot's call reaches another's,
    // nor one person's call another person's seat on a computer they share.
    const named = seatOf(request);
    /** The Bot, for what is decided per Bot: its network policy. */
    const requestBot = named.bot;
    /** The seat, for everything else. Named `botId` because on most computers it is exactly that. */
    const botId = named.seat ?? named.bot;

    // Refused here rather than deeper down, because the id names a directory and the API server
    // forwards whatever URL segment a caller typed. `reset` deletes that directory as root.
    // `/health` is exempt for the same reason it is exempt from the token: it names no Bot, and an
    // orchestrator's probe must not fail on a header it never meant to send.
    if (!isOpenPath(url.pathname) && named.seat === undefined) {
      return json({ error: "That is not a usable bot id." }, 400);
    }
    // A person's browser only on a computer of its own, and nothing else on one: before the id reaches a profile, a
    // policy or a shell. Asked of the id named, never of the seat made from it: on a computer people share, a person's
    // browser id sent with a person would make a seat that no longer reads as a person's browser. The live screen
    // names its browser in the query, which is the one it would show. See person-access.ts.
    const refused = isOpenPath(url.pathname)
      ? null
      : refusedHere(
          OWNER,
          url.pathname === "/stream"
            ? seatOf(request, url.searchParams).bot
            : requestBot,
        );
    if (refused) return json({ error: refused.error }, refused.status);
    // The admin network policy, pushed by the server on every change; applied without a restart.
    // The deployment's Web Bot Auth key rides the same push (see `web-bot-auth.ts`).
    if (url.pathname === "/egress-policy" && request.method === "PUT") {
      const body = (await request
        .clone()
        .json()
        .catch(() => null)) as { webBotAuth?: unknown } | null;
      const applied = await handleEgressPolicyRequest(requestBot, request);
      if (applied.ok) await applyWebBotAuthPush(body?.webBotAuth);
      return applied;
    }
    // A Bot's browser as a person holding its wheel works it. Ahead of the person's-browser rule below, so a
    // person's browser id sent here is told it has the wrong routes rather than that it may not use them.
    if (isBotBrowserPath(url.pathname))
      return botBrowser.handle(request, url.pathname, botId);
    // A person's browser id reaches the person's browser routes, the live screen and the computer's
    // housekeeping, and nothing a Bot acts or reads through. See person-access.ts.
    if (isPersonBrowserId(botId) && !personMayUse(url.pathname))
      return json({ error: PERSONS_OWN_BROWSER }, 403);
    if (isBrowserPath(url.pathname))
      return browserHost.handle(request, url.pathname, botId);
    if (isRealBrowserPath(url.pathname))
      return realBrowser.handle(request, url.pathname, botId);
    const session = sessions.for(botId);

    /*
     * The wheel, asked once for everything that acts.
     *
     * Refused here rather than inside each handler because the handler that forgets is the whole
     * defect: the shell shipped without this check and ran commands underneath a person who had taken
     * the browser at a login wall. `actsOnTheComputer` is the list, and a new acting endpoint is
     * refused by being added to it rather than by remembering to repeat this.
     */
    let releaseAction: (() => void) | undefined;
    try {
      if (actsOnTheComputer(url.pathname)) {
        releaseAction = session.control.admitBotAction(
          mutatesBrowser(url.pathname),
        );
      }

      if (url.pathname === "/stream") {
        /*
         * The socket carries the Bot in the query because it cannot do it in a header. Every other call here names
         * its Bot in `x-openbot-bot-id`, but a websocket client sends no custom headers on the upgrade,
         * so the stream, and only the stream, also accepts the Bot as a query parameter. The header
         * still wins where there is one.
         */
        const streamBotId = seatOf(request, url.searchParams).seat;
        if (streamBotId === undefined) {
          return json({ error: "That is not a usable bot id." }, 400);
        }
        if (server.upgrade(request, { data: { botId: streamBotId } }))
          return undefined as unknown as Response;
        return json({ error: "Expected a WebSocket upgrade." }, 400);
      }

      /*
       * `/live` stays absent. A page served by this process can only be opened by putting the secret in
       * a URL, where it lands in history and logs. The React app is the guarded way to watch a Bot.
       */

      // Who has the wheel. Polled by the surface alongside the screen, so the person sees the Bot ask
      // for help without having to reload anything.
      if (url.pathname === "/control" && request.method === "GET") {
        if (isPersonBrowserId(botId))
          return json(personHoldsTheWheel(PROCESS_STARTED_AT));
        return json(
          session.control.get(url.searchParams.get("requestId") ?? undefined),
        );
      }

      // The Bot asking for help, or a person asking to take the wheel themselves (the first half of a
      // take). It does not take control: it says who is asking and why, and a person decides. A Bot
      // that could hand itself to a human could also hand a human a page they never asked to see.
      if (url.pathname === "/control/request" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          reason?: unknown;
          toolCallId?: unknown;
          source?: unknown;
        } | null;
        if (
          body?.toolCallId !== undefined &&
          typeof body.toolCallId !== "string"
        )
          return json({ error: "toolCallId must be a string." }, 400);
        // A person taking the wheel themselves (`manual`), or the Bot asking for one (`model`, the default). The
        // challenges the computer detects are recorded by the computer, never claimed by a caller.
        if (
          body?.source !== undefined &&
          body.source !== "model" &&
          body.source !== "manual"
        )
          return json({ error: "source must be model or manual." }, 400);
        // The page the Bot is stuck on, recorded with the request. A browser that will not start is
        // no reason to refuse the request: the person is the one who can see why.
        const stuckOn = await currentPage(botId).catch(() => undefined);
        const state = session.control.requestHelp(
          body?.reason,
          body?.toolCallId,
          body?.source ?? "model",
          stuckOn?.url(),
        );
        // Not awaited: reopening the page can take a navigation timeout plus a cold browser start,
        // and the Bot's help call must not wait on it. It catches its own errors.
        void restoreHandoffPage(botId);
        return json(state);
      }

      // The Bot asking for one value it must not be told. It has already focused the field.
      if (url.pathname === "/control/secret" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          label?: unknown;
          ref?: unknown;
          snapshotId?: unknown;
        } | null;
        try {
          return json(session.control.requestSecret(body ?? {}));
        } catch (error) {
          if (error instanceof ControlRequestError) {
            return json({ error: error.message }, 400);
          }
          throw error;
        }
      }

      /**
       * A person supplying that value.
       *
       * Scoped by the pending request rather than by a control handover: it is usable only while the Bot
       * has actually asked for a secret, and the request is cleared the moment it is answered, so this
       * cannot be used as a general back door to type into the page.
       *
       * The value is typed and forgotten. Not stored on `control`, not returned in the response, not
       * logged. The response says how many characters arrived, which is enough for the surface to
       * confirm something was sent and useless to anybody reading it later.
       *
       * It types and does not submit. Committing a form is a separate action through the gateway and
       * audit trail; secret entry only places the value in the named field.
       */
      if (url.pathname === "/human/secret" && request.method === "POST") {
        const pending = session.control.pendingSecret();
        if (!pending) {
          return json({ error: NO_SECRET_PENDING }, 409);
        }
        const body = (await request.json().catch(() => null)) as {
          text?: unknown;
        } | null;
        if (typeof body?.text !== "string" || !body.text) {
          return json({ error: "A value is required." }, 400);
        }
        try {
          const target = await currentPage(botId);
          // Focus the field the Bot named, and let this throw if it cannot be found. A secret must not
          // be reported as delivered unless a field receives it.
          //
          // No generation check here: a Bot may take another snapshot after asking for a secret, while
          // the ref remains protected by Playwright's `aria-ref` rules.
          //
          // `aria-ref` resolves a ref only against the most recent snapshot, only while the element is
          // still connected, and mints a
          // new ref when an element's role or accessible name changes, so a recycled node cannot
          // inherit an old one. If the ref resolves, it is the field the Bot meant. If it does not,
          // nothing is typed, which is the outcome the generation check existed to guarantee.
          const field = locateRef(session, target, pending.ref, undefined);
          await field.click({ timeout: ACTION_TIMEOUT_MS });
          await field.fill(body.text, { timeout: ACTION_TIMEOUT_MS });
          // Marked so the snapshot, screenshots and the live screen never show it back.
          await field
            .evaluate(markElementSecret, SECRET_ATTRIBUTE)
            .catch(() => undefined);
          const characters = body.text.length;
          // Cleared only after it actually landed, so a failure leaves the request open and the person
          // can try again rather than being told to start over.
          session.control.secretSupplied();
          return json({ supplied: true, characters, url: target.url() });
        } catch (error) {
          if (error instanceof StaleSnapshotError) {
            return json({ error: error.message, stale: true }, 409);
          }
          // The field is gone, which is unretryable, so the request is closed rather than left open.
          // Keeping it open is right for a mistyped value and wrong here: the person would retype their
          // password into the same dead ref for ever. Clearing it also unblocks the Bot, which can see
          // on its next turn that nothing is pending and ask again against a fresh snapshot.
          session.control.secretSupplied();
          return json(
            {
              error: describe(
                error,
                "That value could not be entered: the field is no longer on the page. Ask the assistant to request it again.",
              ),
            },
            502,
          );
        }
      }

      /**
       * A person's login, from the private sign-in form, typed by this process.
       *
       * On the acting list, so it is refused while a person holds the wheel. The body is read once,
       * handed to `fillSignIn`, and dropped: not logged, not stored, not returned. The answer says
       * whether a form was submitted and whether a password field is still showing, and every error
       * in it is `fillSignIn`'s own sentence rather than a browser's.
       */
      if (url.pathname === "/sign-in/fill" && request.method === "POST") {
        const input = parseSignInFill(await request.json().catch(() => null));
        if (typeof input === "string") return json({ error: input }, 400);
        const target = await currentPage(botId);
        const result = await fillSignIn(target, input);
        // The page moved on, so every ref handed out before it is retired, as after a navigation.
        if (result.submitted) session.snapshotId += 1;
        return json(result);
      }

      /**
       * The verification code a site asked for after the password, from the same private form.
       *
       * Handled exactly as the login above: read once, typed by `fillSignInCode` into the page that is
       * waiting for it, and dropped.
       */
      if (url.pathname === "/sign-in/code" && request.method === "POST") {
        const input = parseSignInCode(await request.json().catch(() => null));
        if (typeof input === "string") return json({ error: input }, 400);
        const target = await currentPage(botId);
        const result = await fillSignInCode(target, input);
        if (result.submitted) session.snapshotId += 1;
        return json(result);
      }

      if (
        ["/control/take", "/control/release", "/control/cancel"].includes(
          url.pathname,
        ) &&
        request.method === "POST"
      ) {
        const body = (await request.json().catch(() => null)) as {
          requestId?: unknown;
          by?: unknown;
        } | null;
        if (typeof body?.requestId !== "string" || !body.requestId.trim())
          return json({ error: "A requestId is required." }, 400);
        if (url.pathname === "/control/take") {
          // Who is taking it: the signed-in person's user id, which the server always sends. Checked by `take`.
          if (body.by !== undefined && typeof body.by !== "string")
            return json({ error: "by must be a string." }, 400);
          const taken = await session.control.take(body.requestId, body.by);
          // The browser may have restarted while the request waited; give the person the page. In the
          // background, so "Take control" answers at once; the live screen shows the page as it loads.
          void restoreHandoffPage(botId);
          return json(taken);
        }
        const changed =
          url.pathname === "/control/cancel"
            ? session.control.cancel(body.requestId)
            : session.control.release(body.requestId);
        // Answered once a recording the wheel took with it has ended, so what the caller reads next is true.
        await botBrowser.settled(botId);
        return json(changed);
      }

      // A person's input, by pixel. The Bot addresses elements by reference because it reads a list; a
      // person addresses them by pointing, because they are looking at a picture. Different problem,
      // different endpoint, and only usable while they hold the wheel.
      if (HUMAN_INPUT.has(url.pathname) && request.method === "POST") {
        if (!isPersonBrowserId(botId) && !session.control.humanMayDrive()) {
          return json({ error: TAKE_CONTROL_FIRST }, 409);
        }
        session.humanInputs += 1;
        const body = (await request.json().catch(() => null)) as Record<
          string,
          unknown
        > | null;
        if (url.pathname === "/human/scroll") {
          const parsed = parseScrollDelta(body?.deltaY);
          if (!parsed.ok) {
            return json({ error: parsed.error }, 400);
          }
        }
        try {
          const target = await currentPage(botId);
          const done = await performHumanInput(
            target,
            url.pathname,
            body ?? {},
          );
          await notePersonFilled(
            botId,
            url.pathname === "/human/type"
              ? "typed"
              : url.pathname === "/human/key" && typeof body?.key === "string"
                ? keyFills(body.key)
                : url.pathname === "/human/click"
                  ? "chose"
                  : null,
          );
          return json(done);
        } catch (error) {
          return json({ error: describe(error, "That did not work.") }, 502);
        }
      }

      if (url.pathname === "/health") {
        const [profile] = profiles.summary([botId]);
        return json({
          status: "ok",
          // `browser` kept as it was: it is in the published contract and start.sh reads it.
          browser: profile?.running ?? false,
          profile,
          // Which Bot this computer can prove it is, when the deployment runs SPIRE. Null is a
          // deployment without it, not a failure, and it is reported rather than omitted so the
          // difference between "no identity here" and "identity broken" is visible.
          identity: await identity(),
          browserMode: BROWSER_MODE,
          browserBackend: RUNTIME.backend,
          browserChannel: RUNTIME.channel,
        });
      }

      /**
       * Which run of this Bot's browser the caller is looking at.
       *
       * The server orders snapshots on `(run, generation)`, and the generation alone cannot carry it:
       * this process mints one at zero for every session that is new, so a restart, a redeploy, an
       * eviction from the idle sweep and a reset all produce a page at generation one that looks older
       * than the page the server still has. Ordering on the run as well is what lets the fresh one land
       * and the dead one stop resolving.
       *
       * A read, and deliberately not on the acting list: a person holding the wheel must not turn every
       * ref the server holds into an unanswerable question.
       *
       * Its own endpoint rather than a field on `/computers`, because that one reads the profile
       * directory and this is asked on the path of every governed action.
       */
      if (url.pathname === "/run" && request.method === "GET") {
        return json({ run: session.run });
      }

      /**
       * The computers this process holds. The shape is a list because the admin surface is a
       * list, and because a Bot that has a profile has a computer whether or not a browser is running
       * for it this second.
       */
      if (url.pathname === "/computers" && request.method === "GET") {
        return json({ computers: profiles.summary(await profiles.known()) });
      }

      /**
       * Stop the browser, keep what it knows.
       *
       * Closed gracefully so Chromium flushes its profile, and deliberately
       * not restarted here: the next request starts it again, which is the same path as a first ever
       * start, so there is no second way for a browser to come into existence.
       */
      if (url.pathname === "/computers/stop" && request.method === "POST") {
        const wasRunning = await profiles.stop(botId);
        session.control.interrupt(
          "The computer stopped; the previous task was interrupted.",
        );
        await botBrowser.settled(botId);
        return json({ stopped: true, wasRunning });
      }

      /**
       * Hand a private Bot's profile from before seats were per person to its owner's seat.
       *
       * Asked by the server, once per seat, before that seat's first use, and only for a private Bot
       * on its owner's seat: a public Bot's old profile had no single owner. Idempotent, because a
       * move leaves nothing to move again. See carry-over.ts.
       */
      if (
        url.pathname === "/computers/carry-over" &&
        request.method === "POST"
      ) {
        const outcome = await profiles.carryOver(requestBot, botId);
        console.info(
          JSON.stringify({
            type: "computer-profile-carried-over",
            bot: requestBot,
            seat: botId,
            outcome,
          }),
        );
        return json({ outcome });
      }

      /**
       * Forget everything and start over.
       *
       * Signs the computer out of everything by deleting the profile. Irreversible, which is why it is
       * its own endpoint rather than a flag on the one above: a person clicking "stop" must not be able
       * to discard a login by mistyping a parameter.
       */
      if (url.pathname === "/computers/reset" && request.method === "POST") {
        await profiles.reset(botId);
        session.control.interrupt(
          "The computer was reset; the previous task was interrupted.",
        );
        await botBrowser.settled(botId);
        /*
         * And a new run, because the browser this session described is gone.
         *
         * Nothing else here says so. The entry stays in the map and the generation counter carries on,
         * so a snapshot that was in flight when the wipe landed arrives at the server carrying the same
         * run and the same generation as the fresh browser's would: the server deletes its row on
         * reset, the late save inserts the wiped page straight back, and every ref on it goes on
         * resolving. A new run is what makes those two distinguishable at the far end.
         */
        sessions.renewRun(botId);
        return json({ reset: true, botId });
      }

      if (url.pathname === "/navigate" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          url?: unknown;
          toolCallId?: unknown;
        } | null;
        const parsed = parseNavigateUrl(body?.url);
        if (!parsed.ok) {
          return json({ error: parsed.error }, 400);
        }

        const startedAt = Date.now();
        try {
          const target = await currentPage(botId, "navigate");
          session.control.assertBotMayAct(true);
          const response = await navigateWebPage(
            target,
            parsed.url,
            RUNTIME.backend,
            NAVIGATION_TIMEOUT_MS,
          );
          // A new document wipes every stamp, so every ref handed out before now is meaningless.
          // Bumping the generation makes an action carrying one fail with "take a new snapshot" rather
          // than fall through to a selector that matches nothing and read as a missing element.
          session.snapshotId += 1;
          const extract = await readablePageText(target);
          return json({
            url: target.url(),
            title: await target.title(),
            text: extract.text,
            truncated: extract.truncated,
            elapsedMs: Date.now() - startedAt,
            challenge: await detectChallenge(target, session.control, {
              cfMitigated: response
                ? await response.headerValue("cf-mitigated")
                : undefined,
              toolCallId:
                typeof body?.toolCallId === "string"
                  ? body.toolCallId
                  : undefined,
            }),
          });
        } catch (error) {
          if (
            error instanceof ControlError ||
            error instanceof SnapshotRequiredError
          )
            throw error;
          // The page is the Bot's working surface, so a failed navigation is reported rather than
          // thrown: the transcript needs to say what happened, and the browser stays usable.
          return json(
            {
              error:
                error instanceof Error ? error.message : "Navigation failed.",
            },
            502,
          );
        }
      }

      if (url.pathname === "/screenshot" && request.method === "GET") {
        try {
          const target = await currentPage(botId);
          // Sensitive fields, and fields a person filled while they held the wheel, are masked in the
          // picture the Bot is handed. See secret-masking.ts.
          const buffer = await target.screenshot({
            type: "png",
            mask: [target.locator(HIDDEN_FIELD_SELECTOR)],
          });
          const size = target.viewportSize() ?? { width: 1280, height: 800 };
          return json({
            base64: buffer.toString("base64"),
            width: size.width,
            height: size.height,
            capturedAt: new Date().toISOString(),
            // Which page this is a picture of. A browser that has not been sent anywhere sits on
            // `about:blank`, and a screenshot of that is a valid, entirely white PNG, indistinguishable
            // from a real page to anything looking only at the bytes. The transcript needs to tell
            // those apart to avoid presenting a blank browser as though it were a loaded page.
            url: target.url(),
          });
        } catch (error) {
          return json(
            {
              error:
                error instanceof Error ? error.message : "Screenshot failed.",
            },
            502,
          );
        }
      }

      // The Bot's files. Confined to the workspace by workspace.ts. Nothing here decides whether a Bot
      // MAY touch a path: the gateway in front of this process does that.
      if (url.pathname === "/files/download" && request.method === "GET") {
        try {
          const { files } = await workspaces.for(botId);
          const file = await files.download(url.searchParams.get("path") ?? "");
          return new Response(file.body, {
            headers: downloadHeaders(file.name, file.bytes),
          });
        } catch (error) {
          return json(
            { error: describe(error, "The file could not be downloaded.") },
            fileStatus(error),
          );
        }
      }

      if (url.pathname === "/files/read" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          path?: unknown;
        } | null;
        try {
          const { files } = await workspaces.for(botId);
          return json(await files.read(String(body?.path ?? "")));
        } catch (error) {
          return json(
            { error: describe(error, "The file could not be read.") },
            fileStatus(error),
          );
        }
      }

      if (url.pathname === "/files/list" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          path?: unknown;
        } | null;
        try {
          const { files } = await workspaces.for(botId);
          return json(
            await files.list(
              typeof body?.path === "string" ? body.path : undefined,
            ),
          );
        } catch (error) {
          return json(
            { error: describe(error, "The folder could not be listed.") },
            fileStatus(error),
          );
        }
      }

      /*
       * A command on this computer.
       *
       * Nothing here decides whether it may run: the gateway already asked the deployment's policy and
       * wrote the audit row before this was called. Refusing again here would be a second, quieter
       * policy nobody configured.
       */
      if (url.pathname === "/exec" && request.method === "POST") {
        if (!RUNTIME.allowExec)
          return json(
            {
              error:
                "Shell execution is disabled for local Chrome. Use the approved host-access tools to run commands on this computer.",
            },
            403,
          );
        const body = (await request.json().catch(() => null)) as {
          command?: unknown;
          timeoutMs?: unknown;
        } | null;
        if (typeof body?.command !== "string" || !body.command.trim()) {
          return json({ error: "A command is required." }, 400);
        }
        const timeout = parseExecTimeout(body.timeoutMs);
        if (!timeout.ok) {
          return json({ error: timeout.error }, 400);
        }
        try {
          // Rooted in the same folder the file tools use, so a command and a written file see one
          // directory rather than two.
          const { shell } = await workspaces.for(botId);
          return json(
            await shell.run({
              command: body.command,
              ...(timeout.timeoutMs !== undefined
                ? { timeoutMs: timeout.timeoutMs }
                : {}),
              signal: request.signal,
            }),
          );
        } catch (error) {
          return json(
            { error: describe(error, "The command could not be run.") },
            500,
          );
        }
      }

      if (url.pathname === "/files/write" && request.method === "POST") {
        const body = (await request.json().catch(() => null)) as {
          path?: unknown;
          contents?: unknown;
          append?: unknown;
        } | null;
        if (typeof body?.contents !== "string") {
          return json({ error: "The contents to write are required." }, 400);
        }
        try {
          const { files } = await workspaces.for(botId);
          return json(
            await files.write(String(body?.path ?? ""), body.contents, {
              append: body.append === true,
            }),
          );
        } catch (error) {
          return json(
            { error: describe(error, "The file could not be written.") },
            fileStatus(error),
          );
        }
      }

      // The current page as text, without navigating anywhere.
      //
      // Reading must be available after actions too. Returning page text only from `/navigate` would be enough if
      // opening a page were the only way to change what is on screen. It is not: the Bot presses
      // "Submit order", the page becomes a confirmation, and it has no way to find out what the
      // confirmation said. "I clicked the button" is not an answer to what happened.
      if (url.pathname === "/read" && request.method === "GET") {
        try {
          const target = await currentPage(botId);
          const extract = await readablePageText(target);
          return json({
            url: target.url(),
            title: await target.title(),
            text: extract.text,
            truncated: extract.truncated,
            challenge: await detectChallenge(target, session.control),
          });
        } catch (error) {
          return json(
            { error: describe(error, "Reading the page failed.") },
            502,
          );
        }
      }

      // The list of things on the page a Bot can act on. POST rather than GET because it mutates the
      // page, stamping every element it describes, and a GET that changes the document is a lie that
      // caches and prefetchers eventually punish.
      if (url.pathname === "/snapshot" && request.method === "POST") {
        try {
          const target = await currentPage(botId);
          const snapshot = await snapshotPage(session, target);
          return json({
            ...snapshot,
            challenge: await detectChallenge(target, session.control),
          });
        } catch (error) {
          return json({ error: describe(error, "Snapshot failed.") }, 502);
        }
      }

      if (ACTIONS.has(url.pathname) && request.method === "POST") {
        const body = (await request
          .json()
          .catch(() => null)) as ActionBody | null;
        if (!body) {
          return json({ error: "An action needs a JSON body." }, 400);
        }
        if (url.pathname === "/scroll") {
          const parsed = parseScrollDelta(body.deltaY);
          if (!parsed.ok) {
            return json({ error: parsed.error }, 400);
          }
        }

        const startedAt = Date.now();
        try {
          const target = await currentPage(botId);
          session.control.assertBotMayAct(true);
          const detail = await performAction(
            session,
            target,
            url.pathname,
            body,
            // The caller going away is the stop signal: the surface aborts its request, the server
            // aborts the one it made to this computer, and Bun aborts this one in turn.
            request.signal,
          );
          return json({ ...detail, elapsedMs: Date.now() - startedAt });
        } catch (error) {
          /*
           * Stopped, not failed. The signal is checked rather than the error text: Playwright words an
           * abort differently per call, and the caller's own request going away is the fact that
           * matters either way.
           *
           * Logged because the response is not observed after the caller aborts. The log distinguishes
           * "stopped in time" from "ran to completion after cancellation".
           */
          if (
            error instanceof ControlError ||
            error instanceof SnapshotRequiredError
          )
            throw error;
          if (request.signal.aborted) {
            console.info(
              JSON.stringify({
                type: "action-stopped",
                action: url.pathname,
                ref: typeof body.ref === "string" ? body.ref : undefined,
                elapsedMs: Date.now() - startedAt,
              }),
            );
            // 499, the convention for a client that closed the request: this is not the computer
            // failing, and a 502 here would be counted as one.
            return json({ error: "Stopped.", stopped: true }, 499);
          }
          // A stale ref is the caller's mistake and is fixable by taking a new snapshot, so it is a 409
          // rather than a 502: the computer is fine and retrying the same call unchanged will not help.
          if (error instanceof StaleSnapshotError) {
            return json({ error: error.message, stale: true }, 409);
          }
          // The same reasoning: the computer is fine, and the Bot fixes it by sending another value.
          if (error instanceof FieldValueError) {
            return json({ error: error.message }, 400);
          }
          return json({ error: describe(error, "The action failed.") }, 502);
        }
      }

      return json({ error: "Not found." }, 404);
    } catch (error) {
      if (error instanceof ControlError) {
        const state = session.control.get();
        return json(
          {
            error: error.message,
            humanHasControl: true,
            requestId: state.request?.id,
            handoff: state.request,
          },
          409,
        );
      }
      if (error instanceof SnapshotRequiredError)
        return json(
          { error: error.message, stale: true, snapshotRequired: true },
          409,
        );
      if (error instanceof ControlRequestError)
        return json(
          { error: error.message, controlRequestError: true },
          error.status,
        );
      throw error;
    } finally {
      releaseAction?.();
    }
  },
});

type ActionBody = {
  ref?: unknown;
  snapshotId?: unknown;
  text?: unknown;
  key?: unknown;
  deltaY?: unknown;
  submit?: unknown;
};

const ACTIONS = new Set(["/click", "/type", "/key", "/scroll"]);

const HUMAN_INPUT = new Set([
  "/human/click",
  "/human/type",
  "/human/key",
  "/human/scroll",
]);

/**
 * Carry out one thing a person did with their mouse or keyboard.
 *
 * Coordinates are viewport pixels, which the surface works out from the screenshot it is displaying:
 * it knows the image's natural size and the size it drew it at, so it can scale a click back. Doing
 * that conversion in the browser rather than here keeps this endpoint bound to page coordinates
 * rather than window coordinates.
 *
 * Nothing a person types here reaches the model. It goes from their keyboard to this browser and
 * stops. That is what makes a password or a one-time code safe to enter during a takeover: not a
 * filter that strips it out afterwards, but a path the model is not on. The same reason the value is
 * never returned and never logged below.
 */
async function performHumanInput(
  target: Page,
  action: string,
  body: Record<string, unknown>,
): Promise<Record<string, unknown>> {
  const at = (): { x: number; y: number } => {
    const x = typeof body.x === "number" ? body.x : Number.NaN;
    const y = typeof body.y === "number" ? body.y : Number.NaN;
    if (!Number.isFinite(x) || !Number.isFinite(y)) {
      throw new Error("A click needs an x and a y inside the page.");
    }
    // Clamped rather than rejected. A click a pixel outside the viewport is a rounding artefact of
    // scaling the screenshot, not a mistake worth refusing.
    return {
      x: Math.min(Math.max(x, 0), VIEWPORT.width - 1),
      y: Math.min(Math.max(y, 0), VIEWPORT.height - 1),
    };
  };

  if (action === "/human/click") {
    const { x, y } = at();
    await target.mouse.click(x, y);
    return { action: "human_click", url: target.url() };
  }

  if (action === "/human/type") {
    if (typeof body.text !== "string") {
      throw new Error("Typing needs text.");
    }
    // `insertText` rather than per-key typing: a person pasting a one-time code should not have it
    // arrive one character at a time into a field that reformats as you go.
    await target.keyboard.insertText(body.text);
    // Length only, never the value. See the note above about the model not being on this path.
    return {
      action: "human_type",
      characters: body.text.length,
      url: target.url(),
    };
  }

  if (action === "/human/key") {
    if (typeof body.key !== "string" || !body.key) {
      throw new Error("A key press needs a key name.");
    }
    await target.keyboard.press(body.key);
    return { action: "human_key", key: body.key, url: target.url() };
  }

  const parsed = parseScrollDelta(body.deltaY);
  if (!parsed.ok) {
    throw new Error(parsed.error);
  }
  const deltaY = parsed.deltaY ?? 400;
  await target.mouse.wheel(0, deltaY);
  return { action: "human_scroll", deltaY, url: target.url() };
}

/**
 * Carry out one action on the page.
 *
 * Every action that addresses an element goes through {@link locateRef}, so the staleness check
 * cannot be forgotten at a call site. `/key` and `/scroll` may omit a ref and act on the page itself,
 * which is how a Bot presses Enter to submit or scrolls to bring more of a long form into view.
 *
 * Stop has to reach the browser. `signal` is the caller's request going away, the person pressed
 * Stop, and the abort travels from the surface, through the server, to here. Without passing it on,
 * pressing Stop ended the run in the transcript while the click it was meant to prevent carried on
 * landing on a live page. Stop must reach the browser before a high-impact click lands.
 */
async function performAction(
  session: BotSession,
  target: Page,
  action: string,
  body: ActionBody,
  signal?: AbortSignal,
): Promise<Record<string, unknown>> {
  // Passed to every Playwright call below. It does not disable the timeout, which still applies.
  const acting = { timeout: ACTION_TIMEOUT_MS, ...(signal ? { signal } : {}) };
  const expected =
    typeof body.snapshotId === "number" ? body.snapshotId : undefined;
  const ref = typeof body.ref === "string" && body.ref ? body.ref : undefined;

  if (action === "/click") {
    if (!ref) throw new Error("A click needs the ref of an element to click.");
    await (await resolveRef(session, target, ref, expected)).click(acting);
    return { action: "click", ref, url: target.url() };
  }

  if (action === "/type") {
    if (!ref) throw new Error("Typing needs the ref of a field to type into.");
    if (typeof body.text !== "string") {
      throw new Error("Typing needs the text to enter.");
    }
    const field = await resolveRef(session, target, ref, expected);
    // `fill` rather than keystrokes: it clears the field first, which is what "put this value in
    // this box" means. Typing into a field a previous attempt half-filled otherwise appends, and the
    // form ends up with "AlicAlice" in it. A dropdown is chosen from instead (see enterText).
    await enterText(field, body.text, acting);
    // The Bot's own value now, whoever filled the field before.
    await field
      .evaluate(unmarkPersonFilled, PERSON_FILLED_ATTRIBUTE)
      .catch(() => undefined);
    if (body.submit === true) {
      await field.press("Enter", acting);
    }
    // The text itself is deliberately NOT returned. It is echoed nowhere: this response is read by
    // the model and logged by the server, and a value typed into a form is exactly where a password
    // or a card number lives. The caller already knows what it sent.
    return {
      action: "type",
      ref,
      characters: body.text.length,
      submitted: body.submit === true,
      url: target.url(),
    };
  }

  if (action === "/key") {
    if (typeof body.key !== "string" || !body.key) {
      throw new Error("A key press needs a key name, such as Enter or Tab.");
    }
    if (ref) {
      await (await resolveRef(session, target, ref, expected)).press(
        body.key,
        acting,
      );
    } else {
      await target.keyboard.press(body.key);
    }
    return { action: "key", key: body.key, ref, url: target.url() };
  }

  // Scroll. A plain wheel event on the page, which is what moves a long form, rather than scrolling a
  // specific element into view: the Bot asked to see further down, not to hunt for one control.
  const parsed = parseScrollDelta(body.deltaY);
  if (!parsed.ok) {
    throw new Error(parsed.error);
  }
  const deltaY = parsed.deltaY ?? 600;
  await target.mouse.wheel(0, deltaY);
  return { action: "scroll", deltaY, url: target.url() };
}

function describe(error: unknown, fallback: string): string {
  return error instanceof Error ? error.message : fallback;
}

/**
 * Which status a file failure deserves.
 *
 * A path outside the workspace is the caller asking for something it may never have, so 403: retrying
 * it unchanged will never work, and it is not a fault. A missing download is 404, an oversized
 * download is 413, and an ordinary bad file request is 400, because a different request could
 * succeed. Collapsing these into 500 would tell the caller the computer is broken and invite a retry
 * of the same request.
 */
function fileStatus(error: unknown): 400 | 403 | 404 | 413 | 500 {
  if (error instanceof WorkspacePathError) return 403;
  if (error instanceof WorkspaceFileTooLargeError) return 413;
  if (error instanceof WorkspaceFileNotFoundError) return 404;
  if (error instanceof WorkspaceFileError) return 400;
  return 500;
}

console.info(`agent-computer listening on http://127.0.0.1:${PORT}`);

/**
 * Hand the profile back before dying.
 *
 * `docker stop` and a Kubernetes eviction both send SIGTERM and then wait, so this is the window in
 * which Chromium can flush its profile to the volume. This is the graceful-shutdown path for normal
 * container restarts.
 *
 * `stop_grace_period` in docker-compose.yml is what gives this time to run.
 */
let shuttingDown = false;

async function shutDown(reason: string, exitCode: number): Promise<void> {
  if (shuttingDown) return;
  // Set before the first await: a display exiting while Chromium flushes is part of this shutdown,
  // not a second failure racing it.
  shuttingDown = true;
  console.info(`${reason}: closing the browser so its profile is flushed`);
  await profiles.closeAll();
  await VIRTUAL_DISPLAY?.stop();
  process.exit(exitCode);
}

if (VIRTUAL_DISPLAY) {
  void VIRTUAL_DISPLAY.terminated.then(({ code, expected }) => {
    if (expected || shuttingDown) return;
    console.error(
      JSON.stringify({
        type: "computer-virtual-display-exited",
        exitCode: code,
      }),
    );
    // A headed Chromium cannot recover without its display. Let the container restart policy build
    // the pair together again instead of advertising a healthy service whose next browser fails.
    void shutDown("virtual display exited", 1);
  });
}

for (const signal of ["SIGTERM", "SIGINT"] as const) {
  process.on(signal, () => {
    void shutDown(signal, 0);
  });
}