Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

49.4 KB
Raw Blame
import { contractRange } from './artifacts';
import { furthest, OWN_EVENTS, type Position, readPosition } from './positions';
import { ReattachFence } from './reattach-fence';
import type { AgentBridgeEvent, SwitchCredentials } from './types';
import { RUNTIME_ARTIFACT, RUNTIME_VERSION } from './version';

/** This artifact's own range for the contract it speaks to Switch over. */
const AGENT_PROTOCOL = contractRange('agent-protocol', RUNTIME_ARTIFACT);

/**
 * The agent bridge's connection (CHOO-1857), client side.
 *
 * One WebSocket per connection, tied to a connection id we choose. It carries
 * the events down and the heartbeat both ways: the server sends `ping` every
 * heartbeat interval and we answer `pong` with our cursor, which is what keeps
 * the connection alive. The token is checked once, when the socket opens. The
 * connection outlives its socket: losing the socket stops delivery but does not
 * end the connection, so reconnecting within the heartbeat TTL keeps the room
 * slot and the role lease and resumes from the cursor.
 *
 * Each event frame says where the event is numbered, and on reconnect we tell
 * the server how far we have read, so it sends exactly the events we missed. A
 * server that speaks agent-protocol 9 tracks this as a position: one number for
 * the agent's own events and one for each room's conversation (see `Position`).
 * An older server understands only the own-events number, so we send that
 * number as well. A gap is never silent: if the server cannot serve our
 * position it says so, and `onGap` fires so the caller can tell the agent to
 * re-read context rather than carry on believing it saw everything.
 */

/** How long a request on the side (a room claim, placements) may take. */
const REQUEST_TIMEOUT_MS = 4000;
const INITIAL_BACKOFF_MS = 1000;
const MAX_BACKOFF_MS = 30_000;
/**
 * The close code a server sends when it is restarting (uvicorn sends it to
 * every open socket on shutdown).
 *
 * A restart is the one ending we know is short, and the doubling backoff fits
 * it badly: the socket closes as shutdown begins, so the first retries are
 * refused while the server is down and the next one lands seconds after it is
 * back. For a while after this code we retry at a short, randomised interval
 * instead, which brings the agent back within a second of the server and
 * spreads a fleet of agents over that second rather than one instant.
 */
const SERVICE_RESTART = 1012;
const RESTART_WINDOW_MS = 60_000;
const RESTART_RETRY_MIN_MS = 250;
const RESTART_RETRY_SPREAD_MS = 500;
/**
 * The refusal of a server that is opening as many connections as it can take
 * at once (core's `open_gate.py`). It names when to open again, which is this
 * client's place in the server's queue, so that is when the next open goes,
 * whatever the backoff or a restart would have said.
 */
const BUSY = 'busy';

/**
 * How long an open socket has to last before it counts as a working stream.
 *
 * A successful handshake is not evidence of one. A stream that opens and is
 * closed again immediately — the shape of a contested connection — would
 * otherwise reset the backoff on every attempt, so the curve never leaves its
 * first step and the two clients bounce off each other at one reconnect a
 * second indefinitely. A healthy stream lives for minutes.
 */
const STABLE_STREAM_MS = 30_000;

/** How long a placements update waits for the stream to be attached. */
const PLACEMENTS_WAIT_MS = 15_000;

/** Switch answered a placements update with anything but success. */
export class PlacementsRefusedError extends Error {
  constructor(
    readonly status: number,
    readonly detail: string
  ) {
    super(`Switch refused the session placements (HTTP ${status}): ${detail}`);
    this.name = 'PlacementsRefusedError';
  }
}

export type StreamScope = 'single' | 'all';
export type DeliveryFilter = 'all' | 'addressed';

export type { Position } from './positions';

/**
 * Why a stream ended, in the form a caller can branch on.
 *
 * The prose that comes with it is for a human reading a log and may be
 * reworded at any time. Branch on `code`.
 *
 * - `taken_over` — another client attached to this connection id. **Terminal.**
 *   Reopening is itself a takeover, so a displaced client that retries is how
 *   two clients trade one connection back and forth forever.
 * - `heartbeat_lapsed` — we stopped answering pings. Recoverable: reopen and resume.
 * - `closed` — the server ended it for some other reason; `roomId` names the
 *   room when it was about one.
 * - `credentials_rejected` — decided here rather than sent: every reopen would
 *   carry the same token. Terminal.
 */
export interface Eviction {
  code: string;
  reason: string;
  roomId: string | null;
}

export const EVICTION_TAKEN_OVER = 'taken_over';
export const EVICTION_HEARTBEAT_LAPSED = 'heartbeat_lapsed';
export const EVICTION_CLOSED = 'closed';
export const EVICTION_CREDENTIALS_REJECTED = 'credentials_rejected';

/**
 * Read the code off an `evicted` frame, falling back to its prose.
 *
 * A revision-1 server sends prose and no code. Rather than leave every consumer
 * matching strings — the bug this replaces, where one of them matched the short
 * heartbeat wording, missed the long one, and killed a watcher that only needed
 * to reconnect — the one remaining match lives here at the edge, against the
 * three phrasings that server actually produced. It goes when `accepts` rises
 * past revision 1.
 */
function evictionCode(data: Record<string, unknown>): string {
  const code = data.code;
  if (typeof code === 'string' && code.length > 0) return code;
  const reason = String(data.reason ?? '');
  if (reason.startsWith('heartbeat lapsed')) return EVICTION_HEARTBEAT_LAPSED;
  if (reason.startsWith('another stream attached')) return EVICTION_TAKEN_OVER;
  return EVICTION_CLOSED;
}

/**
 * Read the code off a refusal — a rejected open or reattach.
 *
 * Every refusal shares one status, and they call for opposite responses: being
 * superseded is terminal, because attaching is itself a takeover, while the
 * rest are recovered by reopening. A server that sends no code — one built
 * before the refusals were distinguishable — yields null, and null keeps the
 * reopen it has always had.
 */
export function refusalCode(body: string): string | null {
  try {
    const detail = (JSON.parse(body) as { detail?: unknown }).detail;
    if (typeof detail !== 'object' || detail === null) return null;
    const code = (detail as { code?: unknown }).code;
    return typeof code === 'string' && code.length > 0 ? code : null;
  } catch {
    return null;
  }
}

/** The wait a `busy` refusal names, in milliseconds, or null. */
export function busyRetryAfterMs(status: number, body: string): number | null {
  if (status !== 503 || refusalCode(body) !== BUSY) return null;
  try {
    const detail = (JSON.parse(body) as { detail?: { retry_after_s?: unknown } }).detail;
    const seconds = detail?.retry_after_s;
    return typeof seconds === 'number' && Number.isFinite(seconds) && seconds >= 0
      ? Math.round(seconds * 1000)
      : null;
  } catch {
    return null;
  }
}

/** Waits `ms`, or less if any of these signals aborts first, and leaves no timer behind. */
function waitUnless(ms: number, ...signals: AbortSignal[]): Promise<void> {
  let timer: ReturnType<typeof setTimeout> | undefined;
  return Promise.race([
    new Promise<void>((resolve) => {
      timer = setTimeout(resolve, ms);
    }),
    until(...signals),
  ]).finally(() => clearTimeout(timer));
}

/** Resolves when any of these signals aborts, so a wait can be given up on. */
export function until(...signals: AbortSignal[]): Promise<void> {
  const any = AbortSignal.any(signals);
  if (any.aborted) return Promise.resolve();
  return new Promise((resolve) => {
    any.addEventListener('abort', () => resolve(), { once: true });
  });
}

export interface EventStreamLogger {
  debug(message: string, meta?: Record<string, unknown>): void;
  warn(message: string, meta?: Record<string, unknown>): void;
  error(message: string, meta?: Record<string, unknown>): void;
}

/** A contract `Command`, as relayed; the receiver validates the rest. */
export interface SessionCommand {
  /**
   * The session it is for. Null when Switch knows only the room it was typed
   * in (`roomId`), as for an agent run by an agent controller: the agent
   * decides which of its sessions that is.
   */
  sessionId: string | null;
  roomId?: string | null;
  commandId: string;
  [key: string]: unknown;
}

export interface ApprovalOutcome {
  session_id: string;
  request_id: string;
  state: 'answered' | 'expired';
  answer: string | null;
  answered_by: string | null;
  answered_at: string | null;
}

function approvalOutcome(data: Record<string, unknown>): ApprovalOutcome | null {
  const text = (value: unknown) => (typeof value === 'string' ? value : null);
  const sessionId = text(data.session_id);
  const requestId = text(data.request_id);
  if (!sessionId || !requestId || (data.state !== 'answered' && data.state !== 'expired'))
    return null;
  return {
    session_id: sessionId,
    request_id: requestId,
    state: data.state,
    answer: text(data.answer),
    answered_by: text(data.answered_by),
    answered_at: text(data.answered_at),
  };
}

export interface SwitchEventStreamDeps {
  creds: SwitchCredentials;
  /** Chosen by us and reused across reconnects — that is what makes the
   * connection survive a dropped socket. */
  connectionId: string;
  scope: StreamScope;
  filter: DeliveryFilter;
  /**
   * Declares that this connection will start a session for a room on demand.
   *
   * The server keys the "Starting a session…" reply and the DORMANT status off
   * this rather than off the agent's configured `connection_model`, so a
   * watcher must say so — otherwise an addressed message in an unattended room
   * is answered with "my connector isn't reporting in" while this connection is
   * sitting right here, about to spawn.
   */
  spawnCapable?: boolean;
  /**
   * Where to begin, when it must not be "whatever happens next".
   *
   * A session spawned to answer a message needs the buffer position *before*
   * that message: the watcher already consumed it to decide to spawn, so
   * opening at head starts the session after the very thing it was started
   * for. Omitted for every other case, where head is right.
   */
  startCursor?: number;
  /** Rooms declared when the stream opens. Declared at open rather than
   * subscribed afterwards: catch-up runs immediately, and a room claimed a
   * moment later arrives too late for the buffered events a reconnect exists
   * to recover — they would be skipped as "not covered" AND the cursor
   * advanced past them. */
  rooms: string[];
  onEvent(event: AgentBridgeEvent): Promise<void> | void;
  /** A hint to fetch a session's durable command queue; independent of rooms/cursors. */
  onCommands?: (sessionIds: string[]) => Promise<void> | void;
  /**
   * A person answered one of this agent's approval requests, or it expired.
   * Sent again until the session reports it delivered, so it may repeat.
   * Only a connection speaking agent-protocol 4 with scope `all` receives it.
   */
  onApprovalOutcome?: (outcome: ApprovalOutcome) => Promise<void> | void;
  /**
   * A command for one of this agent's sessions, sent by its owner from Switch
   * Console. Switch keeps no copy: whoever receives it is where it lives.
   * Only a connection speaking agent-protocol 5 with scope `all` receives it.
   */
  onSessionCommand?: (command: SessionCommand) => Promise<void> | void;
  /**
   * The rooms the server says this connection covers — on connect, and again
   * whenever they change. The server is the authority here: a room claimed by
   * the session's own `connect_to_room` arrives this way, which is what lets a
   * supervisor learn its session's room from Switch rather than by watching
   * the agent's tool calls.
   */
  onRooms?: (rooms: string[]) => void;
  /**
   * A room we declared that the server refuses to serve, and has therefore
   * been dropped from the declared set.
   *
   * Terminal for that room: the id outlives the room, so whatever remembers it
   * has to forget, or the next connection declares the same dead room again.
   */
  onRoomRejected?: (info: { roomId: string; status: number; detail: string }) => void;
  /**
   * Another connection of this agent took over a room this one held, and
   * `sessionId` names which of this connection's sessions it was placed with,
   * where the server knew. What remembers that placement has to drop it.
   */
  onRoomReleased?: (info: { roomId: string; sessionId: string | null }) => Promise<void> | void;
  /**
   * The server confirmed an open: the first, and every reconnect after it.
   * Where state the server holds only in memory is restated.
   */
  onConnected?: () => void;
  /**
   * An open failed, or an open stream ended, and the stream is about to wait
   * and try again. Not fired for a deliberate reopen, nor once the stream has
   * stopped for good (that is `onEvicted`, or the caller's own signal).
   */
  onDisconnected?: (info: { error: string }) => void;
  /** Fired when the server reports missed events it cannot replay. */
  onGap(info: {
    fromSequence: number;
    reason: string;
    /** The rooms that lost events. Absent from a server that predates naming them. */
    rooms?: string[];
    /** The own-events number (`s`) the server resumes sending from after the gap. */
    resumedAt?: number;
    /** True when `resumedAt` is behind the own-events number we had already read,
     * which means the server restarted its numbering. */
    cursorReset?: boolean;
  }): void | Promise<void>;
  /** Fired when another stream took this connection over, or it was closed.
   * A `taken_over` eviction has already halted the connection before this runs:
   * there is nothing left to reconnect, only something to report. */
  onEvicted(eviction: Eviction): void;
  log: EventStreamLogger;
  /** Aborts the connection. */
  signal: AbortSignal;
}

/**
 * The server refused to open the connection: a `refused` frame, or a close
 * code of 4000 plus an HTTP status. `body` is shaped like the HTTP error body
 * the same refusal would have had (`{"detail": ...}`), so the remedies that
 * read it do not care which transport it came over.
 */
export class OpenRefused extends Error {
  constructor(
    readonly status: number,
    readonly body: string
  ) {
    super(`HTTP ${status}: ${body}`);
    this.name = 'OpenRefused';
  }
}

/** One frame off the socket: `{event, data, id}`, `id` being its sequence number. */
export type SocketFrame = { event: string; id?: string; data: Record<string, unknown> };

/** Node's WebSocket, which, unlike a browser's, takes request headers. */
type HeaderedWebSocket = new (url: string, init: { headers: Record<string, string> }) => WebSocket;

export class SwitchEventStream {
  private readonly deps: SwitchEventStreamDeps;
  /** How far this stream has read. We send it in every WebSocket pong and SSE
   * beat, and as `start_at` when we reopen. When the caller passes
   * `startCursor`, it becomes the own-events number, so the stream resumes from
   * a saved point instead of starting at the current end of the agent's own
   * events. */
  private _position: Position;
  /** Aborts only the current socket, so a reconnect can replace it without
   * tearing down the connection. */
  private socketAbort: AbortController | null = null;
  /** Aborts the connection for good. Distinct from the caller's signal: some
   * refusals can never be retried into a success, and the loop has to end. */
  private readonly halt = new AbortController();
  private rooms: string[];
  /**
   * Declared on every open, so it can be changed by reopening the socket. What
   * a connection may do on the agent's behalf is a setting a person can turn
   * off while it is connected, and the server only hears it here.
   */
  private spawnCapable: boolean;
  /** What the socket currently being opened actually declared, which is only
   * the value above while no change is waiting to be carried. */
  private declaredSpawnCapable: boolean;
  /**
   * Which incarnation of the connection id the server last told us we are.
   *
   * Sent as `expected_generation` on every reopen, so the server refuses a
   * reattach from a client that has been displaced: sharing the id is what
   * makes takeover work, and it is also what makes the loser's reopen
   * indistinguishable from the winner's.
   *
   * Kept across a dropped socket rather than cleared. The connection outlives
   * its socket, so a client whose socket merely dropped is still the holder,
   * and nulling this would have it reopen unfenced and take the connection
   * back off whoever holds it now. Null only before the first
   * `connection_state`.
   */
  private generation: number | null = null;
  /** The server leaves telling a room a session is starting to this client (`connection_state`). */
  private announces = false;
  /**
   * Closed from the moment an open starts until the frame naming the
   * incarnation that open made arrives.
   *
   * A reopen asked for in between waits on it (see `reopen`): there is nothing
   * current to claim, and a stale incarnation is worse than none, since the
   * server reads it as a takeover and we would stand down over our own
   * reconnect.
   */
  private readonly fence = new ReattachFence();
  /**
   * Reopens asked for, and how many of those the open now in progress was
   * built after — so a reopen asked for mid-open is carried out once that open
   * is answered, never by cancelling it. See `reopen`.
   */
  private reopensWanted = 0;
  /** Until when the server is taken to be restarting (see `SERVICE_RESTART`). */
  private restartingUntil = 0;
  private openCarries = 0;

  constructor(deps: SwitchEventStreamDeps) {
    this.deps = deps;
    this.rooms = [...deps.rooms];
    this._position = deps.startCursor === undefined ? {} : { [OWN_EVENTS]: deps.startCursor };
    this.spawnCapable = deps.spawnCapable === true;
    this.declaredSpawnCapable = this.spawnCapable;
  }

  get position(): Position {
    return { ...this._position };
  }

  /** The own-events number (`s`) from our position. A server older than
   * agent-protocol 9 understands only this number, so we also send it as
   * `start_from`, as `Last-Event-ID`, and as `cursor` in pongs and beats. */
  private get cursor(): number {
    return this._position[OWN_EVENTS] ?? 0;
  }

  /**
   * Whether this client tells a room it is starting a session for it, rather
   * than the server: an agent controller leaves it to the agent, Switch
   * itself does it for an agent connected on its own.
   */
  get announcesSessionStarts(): boolean {
    return this.announces;
  }

  /**
   * Switch places a session itself when it calls `connect_to_room` over this
   * connection, so a call's room never waits on the placements stated here.
   */
  readonly placementsNameCallRooms = false;

  start(): void {
    void this.streamLoop();
  }

  /** Repoint the stream at a different room without dropping the connection.
   * The room is claimed server-side first, then the socket is reopened so the
   * new room's buffered events are part of catch-up. */
  async repoint(roomId: string): Promise<void> {
    this.rooms = [roomId];
    await this.subscribe(roomId);
    // Not if the claim revealed we no longer hold the connection: reopening is
    // a takeover, and standing down only to reattach would undo it.
    if (this.halt.signal.aborted) return;
    this.reopen();
  }

  /**
   * Redeclare whether this connection will start a session on demand.
   *
   * The declaration only travels on an open, so the socket is reopened to carry
   * it. That is a reattach rather than a takeover — the incarnation goes with
   * it — so the connection itself survives and nothing else about it changes.
   * Without this a person turning automatic sessions off would leave the server
   * still promising a session this connection is no longer going to start.
   */
  setSpawnCapable(capable: boolean): void {
    if (capable === this.spawnCapable) return;
    this.spawnCapable = capable;
    this.redeclare();
  }

  /**
   * Reopen to carry the current declaration — but only from a socket the server
   * has already named an incarnation for.
   *
   * A reattach claims the incarnation this client believes it holds, and the
   * open that makes the next one is answered asynchronously. Reopening again
   * before that answer arrives claims the incarnation before it, which the
   * server has already moved past and refuses as a takeover — so two changes in
   * quick succession would permanently stand down the agent's only connection.
   * The confirmation carries whatever the declaration has settled on by then
   * instead, and a change that cancels itself out carries nothing. This is the
   * same hazard the fence exists for.
   *
   * Nothing to fence before the first `connection_state`: with no incarnation
   * to claim the reopen is a plain attach, which cannot be refused for being
   * stale.
   */
  private redeclare(): void {
    if (this.declaredSpawnCapable === this.spawnCapable) return;
    if (this.generation !== null && !this.fence.admitting) return;
    this.reopen();
  }

  /**
   * Replace the socket — but never by cancelling an open the server has not
   * answered yet.
   *
   * An open is a request the server acts on as soon as it arrives: it makes a
   * new incarnation of the connection there and then. Cancelling it
   * client-side does not undo that, it only stops us reading the answer that
   * names the new incarnation. The next open then claims the incarnation
   * before it, which the server has moved past, and is refused as a takeover
   * — terminal. The client stands down, and the "other client" it yielded to
   * was its own cancelled request.
   *
   * So a reopen asked for while an open is in flight is remembered instead,
   * and carried out when that open's `connection_state` arrives, by which time
   * we know the incarnation to claim. An open that was already built after the
   * request carries it and needs no second one.
   */
  private reopen(): void {
    this.reopensWanted += 1;
    if (!this.fence.admitting) return;
    // Held from this instant, not from when the stream loop gets round to its
    // own `detaching`, so a second reopen asked for straight after this one
    // waits for the incarnation it makes rather than claiming the old one.
    this.fence.closeAdmission();
    this.socketAbort?.abort();
  }

  /**
   * Pass the server's room list on, and keep our own copy in step.
   *
   * The local copy matters on reconnect: it is what gets declared on the open
   * URL, so a room the server claimed while we were connected is still ours
   * after a drop. Without it a reconnect would re-open with the room we were
   * first told about — or none — and quietly stop receiving.
   */
  private reportRooms(raw: unknown): void {
    if (!Array.isArray(raw)) return;
    const rooms = raw.filter((r): r is string => typeof r === 'string');
    this.rooms = rooms;
    this.deps.onRooms?.(rooms);
  }

  /**
   * Give up on a declared room the server refuses, keeping the connection.
   *
   * The server refuses the **whole** connection when one declared room cannot
   * be served, so re-declaring it means never connecting again: a room id
   * outlives its room, and nothing else in the open path would ever notice.
   * The room is dropped, the refusal is reported at error level, and the room
   * list goes out over the same callback the server's own updates arrive on,
   * so anything holding the room learns it is gone.
   *
   * Only rooms the body actually names are dropped. A refusal that names none
   * of them says nothing about which room is at fault — it stays a transport
   * error and keeps its backoff.
   */
  private dropRefusedRooms(status: number, body: string): boolean {
    if (status !== 403 && status !== 404) return false;
    const refused = this.rooms.filter((room) => body.includes(room));
    if (refused.length === 0) return false;
    const remaining = this.rooms.filter((room) => !refused.includes(room));
    const detail = body.slice(0, 500);
    this.deps.log.error('SwitchEventStream: the server refused a declared room — dropping it', {
      event: 'switch_stream_room_refused',
      status,
      rooms: refused,
      remaining,
      detail,
    });
    this.rooms = remaining;
    for (const roomId of refused) this.deps.onRoomRejected?.({ roomId, status, detail });
    this.deps.onRooms?.(remaining);
    return true;
  }

  /** A rejected credential is not an outage: every reopen would carry the same
   * token, so end the connection and tell the owner once. */
  private rejectCredentials(status: number, body: string): void {
    if (this.halt.signal.aborted) return;
    const detail = body.slice(0, 500);
    this.deps.log.error('SwitchEventStream: the server rejected our credentials — stopping', {
      event: 'switch_stream_credentials_rejected',
      status,
      detail,
    });
    this.halt.abort();
    this.deps.onEvicted({
      code: EVICTION_CREDENTIALS_REJECTED,
      reason: `Switch rejected the agent credentials (HTTP ${status})${detail ? `: ${detail}` : ''}`,
      roomId: null,
    });
  }

  /**
   * Whether a takeover refusal for a request sent under `sent` describes a
   * reopen of our own.
   *
   * A request carries the incarnation it was sent under, and the server
   * refuses it as taken over once the connection has moved past that. But
   * this client reopens its own socket — after an eviction, a dropped socket, a
   * repoint — and a request in flight across that reopen is refused the same
   * way, naming as the new holder the incarnation this client now is. Standing
   * down on that gives the connection up to ourselves, for good. A real
   * takeover after our reopen is still caught: the server tells the socket we
   * now hold with an `evicted` frame.
   */
  private supersededOurselves(sent: number | null): boolean {
    return sent !== null && this.generation !== null && this.generation !== sent;
  }

  /**
   * Give the connection up to the client that now holds it.
   *
   * Ends the connection and reports through the same callback an `evicted` frame
   * does, with the same code, so whatever records the stand-down does not have
   * to care which door the news came through.
   */
  private standDown(): void {
    if (this.halt.signal.aborted) return;
    this.deps.log.warn('SwitchEventStream: the connection was taken over — standing down', {
      event: 'switch_stream_superseded',
      connectionId: this.deps.connectionId,
    });
    this.halt.abort();
    this.deps.onEvicted({
      code: EVICTION_TAKEN_OVER,
      reason: 'another client took this connection over; this one stood down',
      roomId: null,
    });
  }

  private async subscribe(roomId: string): Promise<void> {
    // Not before the stream has told us which incarnation we are. Claiming
    // nothing is how a client too old to have one gets through, so sending it
    // in the window before the first frame would put us through the same door
    // — and that window is exactly when we might already have been displaced
    // without knowing it. A server too old to carry an incarnation still sends
    // the frame, so waiting costs such a client nothing.
    await Promise.race([this.fence.reached, until(this.deps.signal, this.halt.signal)]);
    if (this.halt.signal.aborted) return;
    const generation = this.generation;
    const resp = await this.post('connection/subscribe', {
      connection_id: this.deps.connectionId,
      room_id: roomId,
      // Claimed against the incarnation we hold, because this runs *before*
      // the reopen and so before the open's own check. A connection id
      // outlives a takeover: without this a client that has already been
      // displaced would rewrite the winner's rooms — and evict whoever holds
      // the room it asks for — and being refused the open afterwards would
      // come too late to undo any of it.
      generation,
    });
    if (!resp.ok) {
      const body = await resp.text();
      if (resp.status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
        if (this.supersededOurselves(generation))
          throw new Error(
            `subscribe to ${roomId} was answered for an incarnation this connection has since reopened past; ask again`
          );
        // Not this connection's client any more. Terminal, like every other
        // door onto a takeover: there is nothing to repoint.
        this.standDown();
        return;
      }
      // 409 means another live connection of this agent already holds the room
      // — usually a stale session. Loud: quietly carrying on would leave us
      // attached to a stream that will never deliver that room's events.
      throw new Error(`subscribe to ${roomId} failed: HTTP ${resp.status}`);
    }
  }

  /**
   * State which room each of this connection's sessions is placed in, replacing
   * whatever Switch held for it.
   *
   * Fenced like a subscribe, and for the same reason: sent before the stream
   * has named its incarnation, or by a client that has been displaced, it
   * would rewrite the placements of whoever holds the connection now. Waits
   * up to `PLACEMENTS_WAIT_MS` for the stream to be attached, then refuses.
   * Raises on any answer but success; a takeover also stands the stream down.
   */
  async replacePlacements(placements: Record<string, string>): Promise<{ refusedRooms: string[] }> {
    if (!(await this.attached()))
      throw new Error('the connection to Switch is not open, so it cannot take placements');
    const generation = this.generation;
    const resp = await this.post('connection/placements', {
      connection_id: this.deps.connectionId,
      placements,
      generation,
    });
    // Switch leaves out a session in a room the agent is no longer in, and
    // names it; one from before that answers no `refused` at all.
    if (resp.ok) return { refusedRooms: refusedRoomsOf(await resp.text()) };
    const body = await resp.text();
    if (
      resp.status === 409 &&
      refusalCode(body) === EVICTION_TAKEN_OVER &&
      !this.supersededOurselves(generation)
    )
      this.standDown();
    throw new PlacementsRefusedError(resp.status, body.slice(0, 500));
  }

  private async attached(): Promise<boolean> {
    let timer: ReturnType<typeof setTimeout> | undefined;
    const attached = await Promise.race([
      this.fence.reached.then(() => true),
      until(this.deps.signal, this.halt.signal).then(() => false),
      new Promise<boolean>((resolve) => {
        timer = setTimeout(() => resolve(false), PLACEMENTS_WAIT_MS);
      }),
    ]).finally(() => clearTimeout(timer));
    return attached && !this.halt.signal.aborted;
  }

  private post(path: string, body: unknown, timeoutMs = REQUEST_TIMEOUT_MS) {
    const { creds } = this.deps;
    return fetch(`${creds.apiEndpoint}/agents/${creds.agentId}/${path}`, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${creds.token}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify(body),
      signal: AbortSignal.any([AbortSignal.timeout(timeoutMs), this.deps.signal, this.halt.signal]),
    });
  }

  /**
   * Reopen the stream until it comes back, reporting on a curve.
   *
   * The reconnect itself is unconditional — an endpoint that is down now may be
   * up in a moment, and this loop is the only thing that would notice. What is
   * rationed is the reporting: an endpoint that is simply gone (a managed
   * stack the user stopped, one session per room, each retrying forever) writes
   * the same line every thirty seconds until the app is quit, which is how a
   * real failure elsewhere gets lost. The first failure is reported, then
   * powers of two, and recovery is stated so the outage has an end in the log
   * as well as a beginning.
   */
  private async streamLoop(): Promise<void> {
    const { creds, connectionId, scope, filter, log, signal } = this.deps;
    let backoff = INITIAL_BACKOFF_MS;
    let failures = 0;

    /** Wait before reopening, and widen the wait.
     *
     * Every ending an open socket can have comes through here — a transport
     * error and a clean close alike. The clean close is the one that used to be
     * free: the server ends the stream, the read loop finishes, and the next
     * open went out with no delay at all. That is a storm precisely when the
     * server is ending streams on purpose.
     */
    const pace = async (): Promise<void> => {
      if (Date.now() < this.restartingUntil) {
        const wait = RESTART_RETRY_MIN_MS + Math.random() * RESTART_RETRY_SPREAD_MS;
        await new Promise((r) => setTimeout(r, wait));
        return;
      }
      await new Promise((r) => setTimeout(r, backoff));
      backoff = Math.min(backoff * 2, MAX_BACKOFF_MS);
    };

    while (!signal.aborted && !this.halt.signal.aborted) {
      const socketAbort = new AbortController();
      this.socketAbort = socketAbort;
      // Closed before the open rather than after it lands: it is this open
      // that makes the new incarnation, so a reopen has to wait from here
      // until the frame naming it arrives.
      await this.fence.detaching(0);
      if (signal.aborted || this.halt.signal.aborted) return;
      let openedAt = 0;
      let failure: unknown = null;
      // The wait a busy server named for this open, if it was refused as busy.
      let toldMs: number | null = null;
      try {
        const params = new URLSearchParams({
          connection_id: connectionId,
          scope,
          filter,
          start_from:
            this.cursor > 0 || this.deps.startCursor !== undefined ? String(this.cursor) : 'head',
          // Our state vector: where we are in every source. A server that
          // speaks agent-protocol 9 uses it instead of `start_from`. When we
          // have read nothing yet we send `head`, which asks the server to
          // start at the current end of the agent's own events and of each room.
          start_at: Object.keys(this._position).length ? JSON.stringify(this._position) : 'head',
          // What we are and what we speak, declared on the connect we already
          // make (CHOO-1865). A client that says nothing records as unknown
          // server-side, and a declaration cannot be backfilled after the fact
          // — every release that ships silent is a permanent blind spot.
          protocol: String(AGENT_PROTOCOL.speaks),
          protocol_accepts: String(AGENT_PROTOCOL.accepts),
          client: RUNTIME_ARTIFACT,
          client_version: RUNTIME_VERSION,
        });
        this.declaredSpawnCapable = this.spawnCapable;
        this.openCarries = this.reopensWanted;
        if (this.declaredSpawnCapable) params.set('spawn_capable', 'true');
        if (this.rooms.length) params.set('rooms', this.rooms.join(','));
        // Reattaching, so say which incarnation we believe we still are and
        // let the server refuse us if we are wrong. An attach is a takeover,
        // and a client that missed its own eviction — a partition, a dropped
        // socket, an eviction frame that never arrived — would otherwise take
        // the connection straight back off whoever legitimately holds it. The
        // first open of this object's life sends nothing, which is how a
        // deliberate takeover still works: it has no incarnation to claim.
        if (this.generation !== null) params.set('expected_generation', String(this.generation));

        const url = `${creds.apiEndpoint.replace(/^http/, 'ws')}/agents/${creds.agentId}/connection/ws?${params}`;
        for await (const frame of this.readSocket(url, socketAbort.signal, () => {
          openedAt = Date.now();
          this.restartingUntil = 0;
          log.debug('SwitchEventStream: connection open', {
            event: 'switch_stream_open',
            connectionId,
            cursor: this.cursor,
            rooms: this.rooms,
          });
        })) {
          await this.handleFrame(frame);
        }
      } catch (error) {
        if (error instanceof OpenRefused) {
          const { status, body } = error;
          toldMs = busyRetryAfterMs(status, body);
          // Reopening without the refused room is a different request from the
          // one that just failed, and the declared set strictly shrinks, so
          // this cannot spin: retry now rather than serving the backoff a
          // transport failure earned.
          if (this.dropRefusedRooms(status, body)) continue;
          if (status === 401 || status === 403) {
            this.rejectCredentials(status, body);
            return;
          }
          if (status === 409 && refusalCode(body) === EVICTION_TAKEN_OVER) {
            // The reattach was refused because someone else holds the
            // connection now, and nothing was disturbed in refusing it.
            this.standDown();
            return;
          }
        }
        failure = error;
      }

      if (signal.aborted || this.halt.signal.aborted) return;
      // A deliberate reopen (repoint) aborts the socket. Not an ending to
      // count, and not one to wait out — reopening at once is the point of it.
      if (socketAbort.signal.aborted) continue;

      // Only a stream that lasted proves the endpoint is healthy. Resetting on
      // the handshake alone would let an immediately-closed stream clear the
      // curve it is supposed to be climbing.
      if (openedAt > 0 && Date.now() - openedAt >= STABLE_STREAM_MS) {
        if (failures > 0) {
          log.warn('SwitchEventStream: stream recovered', {
            event: 'switch_stream_recovered',
            afterFailures: failures,
          });
          failures = 0;
        }
        backoff = INITIAL_BACKOFF_MS;
      }

      failures += 1;
      const error = failure === null ? 'the server closed the stream' : String(failure);
      if ((failures & (failures - 1)) === 0) {
        log.warn('SwitchEventStream: stream ended — reopening', {
          event: 'switch_stream_error',
          endpoint: creds.apiEndpoint,
          failures,
          error,
          backoffMs: backoff,
        });
      }
      this.deps.onDisconnected?.({ error });
      if (toldMs !== null) await waitUnless(toldMs, signal, this.halt.signal);
      else await pace();
    }
  }

  /**
   * Open the connection's socket and yield its frames until it closes.
   *
   * The heartbeat is answered here, as each `ping` arrives, not queued behind
   * the frames: handling an event can take as long as the agent likes, and
   * the server must not decide we are gone while it does. The pong carries
   * the cursor as it stands, which is the last event fully handled.
   */
  private async *readSocket(
    url: string,
    abort: AbortSignal,
    onOpen: () => void
  ): AsyncGenerator<SocketFrame> {
    const Socket = (globalThis as { WebSocket?: HeaderedWebSocket }).WebSocket;
    if (!Socket) {
      throw new Error(
        `this runtime needs Node 22 or newer to connect to Switch (it has ${process.version}, with no WebSocket)`
      );
    }
    const socket = new Socket(url, {
      headers: { Authorization: `Bearer ${this.deps.creds.token}` },
    });
    const pending: SocketFrame[] = [];
    let refused: unknown = null;
    let closed: { code: number; reason: string } | null = null;
    let opened = false;
    let wake: (() => void) | null = null;
    const notify = (): void => {
      const resolve = wake;
      wake = null;
      resolve?.();
    };
    socket.addEventListener('open', () => {
      opened = true;
      onOpen();
    });
    socket.addEventListener('message', (message: MessageEvent) => {
      let parsed: { event?: unknown; data?: unknown; id?: unknown };
      try {
        parsed = JSON.parse(String(message.data)) as typeof parsed;
      } catch {
        return;
      }
      if (parsed.event === 'ping') {
        socket.send(
          JSON.stringify({ type: 'pong', cursor: this.cursor, position: this._position })
        );
        return;
      }
      if (parsed.event === 'refused') {
        refused = (parsed.data as { detail?: unknown } | undefined)?.detail ?? null;
        return;
      }
      if (typeof parsed.event !== 'string') return;
      pending.push({
        event: parsed.event,
        data: (parsed.data as Record<string, unknown> | undefined) ?? {},
        ...(parsed.id !== undefined ? { id: String(parsed.id) } : {}),
      });
      notify();
    });
    socket.addEventListener('close', (event: CloseEvent) => {
      closed = { code: event.code, reason: event.reason };
      if (event.code === SERVICE_RESTART) this.restartingUntil = Date.now() + RESTART_WINDOW_MS;
      notify();
    });
    socket.addEventListener('error', () => notify());
    const stop = (): void => socket.close(1000);
    abort.addEventListener('abort', stop, { once: true });
    try {
      for (;;) {
        while (pending.length > 0) yield pending.shift() as SocketFrame;
        if (closed !== null || abort.aborted) break;
        await new Promise<void>((resolve) => {
          wake = resolve;
        });
      }
      const ending = closed as { code: number; reason: string } | null;
      if (ending !== null && ending.code >= 4000 && ending.code < 5000) {
        throw new OpenRefused(
          ending.code - 4000,
          JSON.stringify({ detail: refused ?? ending.reason })
        );
      }
      if (!opened && !abort.aborted) {
        throw new Error(
          `could not connect to Switch (socket closed with ${ending?.code ?? 'no code'}): the server is unreachable, or older than this runtime and has no agent WebSocket`
        );
      }
    } finally {
      abort.removeEventListener('abort', stop);
      if (socket.readyState === socket.CONNECTING || socket.readyState === socket.OPEN) {
        socket.close(1000);
      }
    }
  }

  private async handleFrame(frame: SocketFrame): Promise<void> {
    const { log, onGap, onEvicted, onEvent } = this.deps;
    switch (frame.event) {
      case 'connection_state': {
        if (typeof frame.data.generation === 'number') this.generation = frame.data.generation;
        // `connection_state` is the first frame after the socket connects. Its
        // `position` is the point the server will start sending events from:
        // the `start_at` we asked for, or, when we asked for `head` because we
        // had read nothing yet, the current end of the agent's own events and of
        // each room. We fold it into our own position. Otherwise, if the
        // socket drops and reconnects before any event arrives, we would ask
        // for `head` again, start from wherever the end is by then, and
        // never receive the events that arrived in between.
        const startedAt = readPosition(frame.data.position);
        if (startedAt) this._position = furthest(this._position, startedAt);
        this.announces = frame.data.announce_session_starts === true;
        log.debug('SwitchEventStream: connection established', {
          event: 'switch_stream_connected',
          rooms: frame.data.rooms,
          generation: this.generation,
          // What the server says it is (CHOO-1865). Recorded, not acted on —
          // logging it is what makes "which versions are actually talking to
          // each other" answerable from a bug report rather than a guess.
          server: frame.data.server ?? null,
        });
        this.reportRooms(frame.data.rooms);
        // A reopen asked for while this open was in flight, for something this
        // open was built too early to carry. Now that we know the incarnation
        // to claim, it is safe to replace the socket — and done before the
        // gate opens, so nothing else is sent on a socket about to be dropped.
        if (this.reopensWanted > this.openCarries) {
          this.socketAbort?.abort();
          return;
        }
        this.fence.attached();
        this.redeclare();
        this.deps.onConnected?.();
        return;
      }
      case 'room_released': {
        const roomId = frame.data.room_id;
        const sessionId = frame.data.session_id ?? null;
        if (typeof roomId === 'string' && (sessionId === null || typeof sessionId === 'string')) {
          log.warn('SwitchEventStream: another connection took a room over', {
            event: 'switch_stream_room_released',
            roomId,
            sessionId,
          });
          await this.deps.onRoomReleased?.({ roomId, sessionId });
        } else
          log.warn('SwitchEventStream: unreadable room release dropped', {
            event: 'switch_stream_bad_release',
          });
        return;
      }
      case 'session_commands':
        if (
          Array.isArray(frame.data.session_ids) &&
          frame.data.session_ids.every((id) => typeof id === 'string')
        )
          await this.deps.onCommands?.(frame.data.session_ids as string[]);
        return;
      case 'subscription_changed':
        log.debug('SwitchEventStream: subscription changed', {
          event: 'switch_stream_subscription',
          rooms: frame.data.rooms,
        });
        this.reportRooms(frame.data.rooms);
        return;
      case 'gap': {
        const rooms = Array.isArray(frame.data.rooms) ? frame.data.rooms.map(String) : undefined;
        log.warn('SwitchEventStream: gap — events missed', {
          event: 'switch_stream_gap',
          fromSequence: frame.data.from_sequence,
          reason: frame.data.reason,
          rooms: rooms ?? null,
        });
        const resumedAt = frame.data.resumed_at;
        if (resumedAt !== undefined && (!Number.isSafeInteger(resumedAt) || Number(resumedAt) < 0))
          throw new Error('Switch returned an invalid gap resume cursor.');
        // A server that speaks agent-protocol 9 sends `resumed_position`, the
        // position it will resume sending from after the gap. We replace our
        // position with it rather than merging it with `furthest`, because a
        // gap is the only case where the server moves a position back. Merging
        // would keep our higher numbers, and the next reconnect would ask to
        // start past events we never received.
        const resumed = readPosition(frame.data.resumed_position);
        if (frame.data.resumed_position !== undefined && resumed === null)
          throw new Error('Switch returned an invalid gap resume position.');
        await onGap({
          fromSequence: Number(frame.data.from_sequence ?? 0),
          reason: String(frame.data.reason ?? 'events were missed'),
          ...(rooms === undefined ? {} : { rooms }),
          ...(resumedAt === undefined
            ? {}
            : { resumedAt: Number(resumedAt), cursorReset: Number(resumedAt) < this.cursor }),
        });
        if (resumed) this._position = resumed;
        else if (resumedAt !== undefined)
          this._position = { ...this._position, [OWN_EVENTS]: Number(resumedAt) };
        return;
      }
      case 'evicted': {
        const code = evictionCode(frame.data);
        log.warn('SwitchEventStream: evicted', {
          event: 'switch_stream_evicted',
          code,
          reason: frame.data.reason,
          roomId: frame.data.room_id ?? null,
        });
        // Halt before reporting: a takeover is the one ending that reopening
        // cannot recover, because reopening is itself a takeover. The
        // connection ends here, and nothing the callback does can restart them.
        if (code === EVICTION_TAKEN_OVER) this.halt.abort();
        onEvicted({
          code,
          reason: String(frame.data.reason ?? 'connection closed'),
          roomId: typeof frame.data.room_id === 'string' ? frame.data.room_id : null,
        });
        return;
      }
      case 'session_command': {
        const sessionId = frame.data.sessionId;
        if (typeof sessionId === 'string' && typeof frame.data.commandId === 'string')
          await this.deps.onSessionCommand?.({ ...frame.data, sessionId } as SessionCommand);
        else
          log.warn('SwitchEventStream: unreadable session command dropped', {
            event: 'switch_stream_bad_command',
          });
        return;
      }
      case 'approval_outcome': {
        const outcome = approvalOutcome(frame.data);
        if (outcome) await this.deps.onApprovalOutcome?.(outcome);
        else
          log.warn('SwitchEventStream: unreadable approval outcome dropped', {
            event: 'switch_stream_bad_outcome',
          });
        return;
      }
      default:
        if (typeof frame.data.type === 'string')
          await onEvent(frame.data as unknown as AgentBridgeEvent);
        else
          // A frame this runtime does not know and cannot read as a room event.
          log.warn('SwitchEventStream: unknown frame dropped', {
            event: 'switch_stream_unknown_frame',
            frame: frame.event,
          });
        this.passed(frame);
    }
  }

  /**
   * Advances our position past an event after it has been handled. A server
   * that speaks agent-protocol 9 sends the event's `position`; an older server
   * sends only its own-events number, as the frame id.
   *
   * This runs after the handler, not before, so that if the handler throws the
   * event is not counted as read and the server sends it again when the stream
   * reopens.
   */
  private passed(frame: SocketFrame): void {
    const eventPosition =
      readPosition(frame.data.position) ??
      (frame.id ? { [OWN_EVENTS]: Number(frame.id) || 0 } : null);
    if (eventPosition) this._position = furthest(this._position, eventPosition);
  }
}

/** The rooms a placements answer says the agent is not in. */
function refusedRoomsOf(body: string): string[] {
  try {
    const parsed = JSON.parse(body) as { refused?: { room_id?: unknown }[] };
    return [
      ...new Set(
        (parsed.refused ?? [])
          .map((each) => each.room_id)
          .filter((room): room is string => typeof room === 'string')
      ),
    ];
  } catch {
    return [];
  }
}