Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
71.2 KB
Raw Blame
/**
 * Blue/green rollover for the managed OpenCode engine.
 *
 * A config change the engine can only read at instance build time (provider
 * blocks, plugins, agents, permissions) normally forces a dispose, which
 * aborts every run in flight. Deferring the dispose until the engine idles
 * avoids the abort but leaves config hostage to long sessions.
 *
 * This pool takes the third option when a reload lands on a busy engine:
 * spawn a standby with the new config, point every new request at it, and
 * keep the old engine alive only until its running sessions finish. Sessions
 * live in OpenCode's shared SQLite, so only the live run is process-bound.
 *
 * Steady state stays one engine. During a drain there are two, and never more
 * than two long-lived ones: a config change arriving mid-drain coalesces into
 * a single pending rollover (latest wins) instead of stacking processes.
 *
 * Always on for managed engines. Attached engines have no pool, so their
 * callers keep the defer-while-busy behavior instead.
 */
import { createHash, randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";

import {
  buildEngineAuthProbeHeader,
  registerEngineInstance,
  removeEngineInstance,
  updateEngineInstanceRole,
} from "./engine-registry.js";
import { createManagedOpencodeServer, type ManagedOpencodeServer } from "./managed-opencode.js";
import { appendRuntimeRestartRecord } from "./runtime-restart-log.js";
import { loopbackFetch } from "./server-fetch.js";
import type { ServerConfig, WorkspaceInfo } from "./types.js";

export type EnginePoolLogger = {
  log: (level: "info" | "warn" | "error", message: string, attributes?: Record<string, unknown>) => void;
};

/** Everything needed to spawn a replacement engine identical to the first one. */
export type EngineSpawnTemplate = {
  bin?: string;
  cwd: string;
  runtimeConfigPath: string;
  env: Record<string, string | undefined>;
  /** Ports a standby must avoid (the OmniRush.ai server, and the live engine). */
  reservedPorts: () => number[];
  /** Readiness budget for a standby spawn. Defaults to the managed-engine default. */
  spawnTimeoutMs?: number;
};

export type EnginePoolStandby = {
  workspace: WorkspaceInfo;
  generationId: string;
  baseUrl: string;
  username: string;
  password: string;
};

export type EnginePoolHooks = {
  /** Today's in-place dispose. Used when the engine is idle. */
  reloadInPlace: (
    config: ServerConfig,
    workspace: WorkspaceInfo,
    options?: { awaitPostRefreshSync?: boolean },
  ) => Promise<void>;
  /** Whether the given engine has non-idle sessions. */
  engineBusy: (config: ServerConfig, workspace: WorkspaceInfo) => Promise<boolean>;
  /** Re-register runtime MCPs and reconcile cloud MCP against a fresh engine. */
  postRefreshSync: (config: ServerConfig, workspace: WorkspaceInfo) => Promise<void>;
  /**
   * Prepare a healthy standby before it becomes primary (managed provider
   * credentials). A spawned engine receives no provider keys in its
   * environment; without this the first sync after the flip re-delivers them,
   * reports a credential change, and forces yet another standby.
   */
  prepareStandby?: (config: ServerConfig, standby: EnginePoolStandby) => Promise<void>;
  /** Rebuild the engine-visible runtime config file (workspace-independent). */
  writeRuntimeConfigFile: (config: ServerConfig) => Promise<{ path: string }>;
  registerTrusted: (config: ServerConfig, generation: { baseUrl: string; identity: string; isAlive: () => boolean }) => void;
  clearTrusted: (config: ServerConfig, identity: string) => void;
  spawn?: (template: EngineSpawnTemplate) => Promise<ManagedOpencodeServer>;
  now?: () => number;
  schedule?: (operation: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
  waitForHealthy?: (handle: ManagedOpencodeServer) => Promise<void>;
  /**
   * One liveness probe of a running engine: "ok" when it answered at all,
   * "refused" when nothing listens on its port, "failed" for anything else
   * (timeout, reset). Defaults to GET /global/health.
   */
  probeHealth?: (handle: ManagedOpencodeServer) => Promise<"ok" | "refused" | "failed">;
  /**
   * Called right before the watchdog closes a primary it confirmed dead, so
   * the runs it was carrying can be queued for task recovery.
   */
  onPrimaryLost?: (config: ServerConfig) => void;
  /** Called once when the pool shuts down, before its engines are retired. */
  onDisposed?: () => void;
  /**
   * Called when an engine is closed while another serves (a drained
   * generation retired, a dead primary replaced), before it closes: whatever
   * still reads the closed engine (the session uploader's turn observers) moves to
   * `replacement`, which reads the same sessions from the shared database.
   */
  onEngineReplaced?: (closedBaseUrl: string, replacement: EnginePoolConnection) => void;
  logger?: EnginePoolLogger;
};

export type RolloverReason = string;

export type RolloverOutcome =
  | { action: "skipped"; reason: "unchanged" }
  | { action: "reloaded_in_place" }
  | { action: "coalesced" }
  | { action: "rolled_over"; generationId: string; drainingSessions: number };

type GenerationStatus = "starting" | "primary" | "draining" | "dead";

type Generation = {
  id: string;
  handle: ManagedOpencodeServer;
  status: GenerationStatus;
  spawnedAt: number;
  fingerprint: string;
  registryId: string | null;
  trustedIdentity: string | null;
  drainTimer: ReturnType<typeof setInterval> | null;
  drainDeadline: number | null;
  drainActivityWatch: AbortController | null;
  /** Proxied requests in flight to this engine (a drained engine closes only once they answered, or a grace period passed). */
  requestsInFlight: number;
  /** Re-runs the drain check (set while draining). */
  drainCheck: (() => void) | null;
};

export type EnginePoolSnapshot = {
  generations: Array<{
    role: GenerationStatus;
    pid: number | null;
    port: number | null;
    spawnedAt: number;
    drainRemainingMs: number | null;
  }>;
};

export type EnginePoolConnection = {
  generationId: string;
  role: "primary" | "draining";
  baseUrl: string;
  username: string;
  password: string;
};

export type EnginePoolRoute = {
  target: EnginePoolConnection;
  fallback: EnginePoolConnection | null;
};

export type EngineEventProxyLease = {
  signal: AbortSignal;
  release: () => void;
};

export type EnginePoolProcess = {
  pid: number | null;
  isAlive: () => boolean;
};

function positiveIntFromEnv(name: string, fallback: number): number {
  const raw = Number(process.env[name]);
  return Number.isFinite(raw) && raw > 0 ? raw : fallback;
}

/** Same, but an explicit 0 is meaningful (it disables the throttle). */
function nonNegativeIntFromEnv(name: string, fallback: number): number {
  const value = process.env[name]?.trim();
  if (!value) return fallback;
  const raw = Number(value);
  return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
}

/**
 * How long a draining engine may go without any owned-session activity before
 * its sessions are aborted. Activity on the engine event stream pushes the
 * deadline out, so this bounds inactivity, not total drain time.
 */
function drainTimeoutMs(): number {
  return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS", 15 * 60_000);
}

/**
 * How long a drained engine stays up for proxied requests still in flight to
 * it (a session create waiting on the instance's cold boot) before it closes
 * anyway.
 */
function drainRequestGraceMs(): number {
  return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS", 60_000);
}

/**
 * How long a session an engine was just sent a prompt for counts as running
 * there before the engine reports it (the engine accepts a prompt before its
 * run starts, which may wait on the folder's instance booting).
 */
function promptStartGraceMs(): number {
  return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_PROMPT_START_GRACE_MS", 60_000);
}

/** Delay before the drain activity watch reconnects to a lost engine event stream. */
function drainActivityReconnectMs(): number {
  return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_ACTIVITY_RECONNECT_MS", 1_000);
}

/** Floor between automatic spawns, so a burst of triggers cannot thrash. 0 disables it. */
function minSpawnIntervalMs(): number {
  return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS", 30_000);
}

/**
 * Spacing between the watchdog's liveness probes of a primary that stopped
 * answering proxied requests. Three probes spread over this interval must all
 * fail before a live engine process is closed.
 */
function recoveryProbeIntervalMs(): number {
  return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS", 10_000);
}

const RECOVERY_PROBE_ATTEMPTS = 3;

function drainPollIntervalMs(): number {
  return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_POLL_MS", 5_000);
}

/** Grace given to aborted sessions to unwind before the engine is closed. */
function abortSettleMs(): number {
  return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", 5_000);
}

function portOf(url: string): number {
  try {
    return Number(new URL(url).port) || 0;
  } catch {
    return 0;
  }
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

export function isEngineConnectionFailure(error: unknown): boolean {
  // Node/undici occasionally drops the transport cause. This classifier is
  // only used for managed OpenCode engine traffic, never external egress.
  if (error instanceof TypeError && error.message === "fetch failed" && error.cause === undefined) return true;
  const visited = new Set<object>();
  let current: unknown = error;
  for (let depth = 0; depth < 6 && current !== null && current !== undefined; depth += 1) {
    if (typeof current === "string") {
      return /ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|other side closed|socket hang up|Unable to connect|Headers Timeout Error/i.test(current);
    }
    if (!isRecord(current) || visited.has(current)) return false;
    visited.add(current);
    const code = current.code;
    // UND_ERR_SOCKET ("other side closed"): the engine closed the connection before it answered.
    if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "ETIMEDOUT" || code === "UND_ERR_CONNECT_TIMEOUT" || code === "UND_ERR_HEADERS_TIMEOUT" || code === "UND_ERR_SOCKET") return true;
    const message = current.message;
    if (typeof message === "string" && /ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|other side closed|socket hang up|Unable to connect|Headers Timeout Error/i.test(message)) return true;
    current = current.cause;
  }
  return false;
}

function isConnectionRefused(error: unknown): boolean {
  let current: unknown = error;
  for (let depth = 0; depth < 6 && isRecord(current); depth += 1) {
    if (current.code === "ECONNREFUSED") return true;
    current = current.cause;
  }
  return false;
}

export function isConnectionRefusedClassError(error: unknown): boolean {
  return isEngineConnectionFailure(error);
}

function sameOrigin(left: string, right: string): boolean {
  try {
    return new URL(left).origin === new URL(right).origin;
  } catch {
    return left === right;
  }
}

function isRoutableGeneration(
  generation: Generation,
): generation is Generation & { status: "primary" | "draining" } {
  return generation.status === "primary" || generation.status === "draining";
}

function normalizeProxyPath(proxyPath: string): string {
  const raw = proxyPath.trim() || "/";
  const withoutPrefix = raw.startsWith("/opencode") ? raw.slice("/opencode".length) : raw;
  const normalized = (withoutPrefix || "/").replace(/\/+$/, "");
  return normalized || "/";
}

function decodePathPart(value: string | undefined): string | null {
  if (!value) return null;
  try {
    return decodeURIComponent(value);
  } catch {
    return value;
  }
}

function sessionIdFromPath(proxyPath: string): string | null {
  const match = normalizeProxyPath(proxyPath).match(/^\/(?:api\/)?session\/([^/]+)(?:\/|$)/);
  const sessionId = decodePathPart(match?.[1]);
  return sessionId === "status" ? null : sessionId;
}

function requestIdFromPath(proxyPath: string): string | null {
  const normalized = normalizeProxyPath(proxyPath);
  const legacy = normalized.match(/^\/(?:permission|question)\/([^/]+)\/(?:reply|reject)$/);
  if (legacy) return decodePathPart(legacy[1]);
  const scoped = normalized.match(/^\/api\/session\/[^/]+\/(?:permission|question)\/([^/]+)\/(?:reply|reject)$/);
  return decodePathPart(scoped?.[1]);
}

function isLegacyPromptReply(proxyPath: string): boolean {
  return /^\/(?:permission|question)\/[^/]+\/(?:reply|reject)$/.test(normalizeProxyPath(proxyPath));
}

function isPromptishSessionRequest(method: string, proxyPath: string): boolean {
  if (method.toUpperCase() !== "POST") return false;
  const normalized = normalizeProxyPath(proxyPath);
  return /^\/(?:api\/)?session\/[^/]+\/(?:message|prompt(?:_async)?|command|shell)(?:\/|$)/.test(normalized);
}

function firstString(record: Record<string, unknown>, keys: string[]): string | null {
  for (const key of keys) {
    const value = record[key];
    if (typeof value === "string" && value.trim()) return value.trim();
  }
  return null;
}

function requestRecords(payload: unknown): Record<string, unknown>[] {
  if (Array.isArray(payload)) return payload.filter(isRecord);
  if (!isRecord(payload)) return [];
  for (const key of ["items", "permissions", "questions", "requests"]) {
    const value = payload[key];
    if (Array.isArray(value)) return value.filter(isRecord);
  }
  return [];
}

function eventIdentifiers(payload: unknown): { sessionIds: Set<string>; requestIds: Set<string> } {
  const sessionIds = new Set<string>();
  const requestIds = new Set<string>();
  const visit = (value: unknown, depth: number): void => {
    if (depth > 6 || !isRecord(value)) return;
    for (const [key, item] of Object.entries(value)) {
      if (typeof item === "string") {
        if (["sessionID", "sessionId", "session_id"].includes(key)) sessionIds.add(item);
        if (["requestID", "requestId", "permissionID", "questionID"].includes(key)) requestIds.add(item);
        continue;
      }
      if (Array.isArray(item)) {
        for (const child of item) visit(child, depth + 1);
      } else {
        visit(item, depth + 1);
      }
    }
  };
  visit(payload, 0);
  return { sessionIds, requestIds };
}

/**
 * Cap on the text one unterminated SSE frame may buffer before the stream is
 * treated as malformed. Sized well above any legitimate single engine event
 * (large tool outputs included) so real frames are never truncated; only a
 * stream that stops terminating frames hits it.
 */
export const ENGINE_SSE_FRAME_MAX_CHARS = 4 * 1024 * 1024;

/**
 * Incremental SSE frame splitter shared by the client event fan-in and the
 * drain activity watch. It bounds the memory a single unterminated frame can
 * hold: completed frames are always returned, and `overflow` turns true once
 * the pending remainder exceeds the cap so the caller can drop that
 * connection instead of buffering it forever.
 */
export class BoundedSseFrameBuffer {
  private readonly decoder = new TextDecoder();
  private buffered = "";

  constructor(private readonly maxFrameChars = ENGINE_SSE_FRAME_MAX_CHARS) {}

  push(chunk: Uint8Array): { frames: string[]; overflow: boolean } {
    this.buffered += this.decoder.decode(chunk, { stream: true });
    const frames: string[] = [];
    while (true) {
      const delimiter = this.buffered.match(/\r?\n\r?\n/);
      if (!delimiter || delimiter.index === undefined) break;
      frames.push(this.buffered.slice(0, delimiter.index));
      this.buffered = this.buffered.slice(delimiter.index + delimiter[0].length);
    }
    return { frames, overflow: this.buffered.length > this.maxFrameChars };
  }
}

/** Decode one SSE frame's `data:` payload; null when the frame carries none. */
function sseFramePayload(frame: string): unknown {
  const data = frame
    .split(/\r?\n/)
    .filter((line) => line.startsWith("data:"))
    .map((line) => line.slice("data:".length).trim())
    .join("\n");
  if (!data) return null;
  try {
    return JSON.parse(data);
  } catch {
    return null;
  }
}

/**
 * Everything a spawned engine reads at build time. Comparing this is what
 * lets a repeated no-op reload skip spawning a replacement.
 */
export async function computeEngineConfigFingerprint(template: EngineSpawnTemplate): Promise<string> {
  const content = await readFile(template.runtimeConfigPath, "utf8").catch(() => "");
  return createHash("sha256")
    .update(content)
    .update("\u0000")
    .update(template.bin?.trim() ?? "")
    .update("\u0000")
    .update(template.env.OPENCODE_MODELS_URL ?? "")
    .digest("hex");
}

/**
 * The chats behind busy sessions: each session's root (following parents up, at most `maxDepth` steps; a
 * longer chain throws, so the caller names no chat). Exported for tests.
 */
export async function rootChats(busy: string[], parentOf: (sessionId: string) => Promise<string | null>, maxDepth = 16): Promise<string[]> {
  const roots = await Promise.all([...new Set(busy)].map(async (sessionId) => {
    let current = sessionId;
    for (let step = 0; ; step += 1) {
      const parent = await parentOf(current);
      if (!parent) return current;
      if (step >= maxDepth) throw new Error("session chain too deep");
      current = parent;
    }
  }));
  return [...new Set(roots)];
}

export class EnginePool {
  private readonly config: ServerConfig;
  private readonly template: EngineSpawnTemplate;
  private readonly hooks: EnginePoolHooks;
  private generations: Generation[] = [];
  private inFlight: Promise<RolloverOutcome> | null = null;
  private pendingRollover: {
    reason: RolloverReason;
    workspace: WorkspaceInfo;
    manual: boolean;
    awaitPostRefreshSync: boolean;
    forceStandby: boolean;
    retireDrainingIf?: (baseUrl: string) => boolean;
  } | null = null;
  private lastSpawnAt = 0;
  private consecutiveConnectionFailures = 0;
  private lastRecoveryAt = Number.NEGATIVE_INFINITY;
  private recoveryTimer: ReturnType<typeof setTimeout> | null = null;
  private recoveryWorkspace: WorkspaceInfo | null = null;
  private recoveryInFlight: Promise<void> | null = null;
  private recoveryOrphan: ManagedOpencodeServer | null = null;
  private disposed = false;
  private readonly sessionOwnership = new Map<string, string>();
  private readonly pinnedRequests = new Map<string, string>();
  private readonly activeSessionsByGeneration = new Map<string, Set<string>>();
  private readonly eventProxyControllers = new Set<AbortController>();
  private readonly drainWaiters = new Set<() => void>();
  /**
   * Prompts sent to an engine whose session it may not report busy yet (the
   * engine answers a prompt before its run starts), by session: the engine
   * they went to and when. Such a session drains like a running one.
   */
  private readonly startingPrompts = new Map<string, { generationId: string; at: number }>();

  constructor(input: { config: ServerConfig; template: EngineSpawnTemplate; hooks: EnginePoolHooks }) {
    this.config = input.config;
    this.template = input.template;
    this.hooks = input.hooks;
  }

  /**
   * Register the engine spawned during startup as the first generation, so a
   * later rollover knows what it is replacing.
   */
  adoptPrimary(input: {
    handle: ManagedOpencodeServer;
    fingerprint: string;
    registryId: string | null;
    trustedIdentity: string | null;
  }): void {
    this.generations.push({
      id: randomUUID(),
      handle: input.handle,
      status: "primary",
      spawnedAt: Date.now(),
      fingerprint: input.fingerprint,
      registryId: input.registryId,
      trustedIdentity: input.trustedIdentity,
      drainTimer: null,
      drainDeadline: null,
      drainActivityWatch: null,
      requestsInFlight: 0,
      drainCheck: null,
    });
    this.lastSpawnAt = Date.now();
  }

  primaryUrl(): string | null {
    return this.generations.find((entry) => entry.status === "primary")?.handle.url ?? null;
  }

  primaryProcess(): EnginePoolProcess | null {
    const primary = this.generations.find((entry) => entry.status === "primary") ?? null;
    return primary ? { pid: primary.handle.pid ?? null, isAlive: primary.handle.isAlive } : null;
  }

  reportRequestSuccess(baseUrl: string): void {
    if (this.isPrimaryEndpoint(baseUrl)) this.consecutiveConnectionFailures = 0;
  }

  reportRequestFailure(baseUrl: string, error: unknown, workspace: WorkspaceInfo): void {
    if (!this.isPrimaryEndpoint(baseUrl) || !isEngineConnectionFailure(error)) return;
    this.consecutiveConnectionFailures += 1;
    if (this.consecutiveConnectionFailures < 3) return;
    const now = this.now();
    if (now - this.lastRecoveryAt < minSpawnIntervalMs()) return;
    const failures = this.consecutiveConnectionFailures;
    this.consecutiveConnectionFailures = 0;
    this.lastRecoveryAt = now;
    this.recoveryWorkspace = workspace;
    this.hooks.logger?.log("warn", "Managed engine stopped answering; checking whether it is still alive.", {
      "engine.recovery.consecutive_failures": failures,
      "engine.recovery.last_error": error instanceof Error ? error.message : String(error),
    });
    void this.recoverDeadPrimary(workspace).catch(() => undefined);
  }

  /**
   * A proxied request to `baseUrl` is going out: a draining engine is not
   * closed under it (for at most OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS).
   * Call the returned release once the engine answered or the request failed.
   */
  beginRequest(baseUrl: string, request?: { method: string; proxyPath: string }): () => void {
    const generation = this.generationForUrl(baseUrl);
    if (!generation) return () => undefined;
    const sessionId = request && isPromptishSessionRequest(request.method, request.proxyPath) ? sessionIdFromPath(request.proxyPath) : null;
    if (sessionId) this.noteStartingPrompt(generation, sessionId);
    generation.requestsInFlight += 1;
    let released = false;
    return () => {
      if (released) return;
      released = true;
      generation.requestsInFlight = Math.max(0, generation.requestsInFlight - 1);
      if (generation.requestsInFlight === 0 && generation.status === "draining") generation.drainCheck?.();
    };
  }

  /**
   * After a proxied request to `failedBaseUrl` failed to connect: the route
   * to retry it on once a rollover moved the engine it went to (that engine
   * is no longer serving, or a new primary took over while it was being
   * replaced). Waits for a rollover in flight to flip, at most `timeoutMs`.
   * Null when there is nothing to retry on: the engine it went to is still
   * the one serving.
   */
  async rerouteAfterFailure(method: string, proxyPath: string, failedBaseUrl: string, timeoutMs: number): Promise<EnginePoolRoute | null> {
    const deadline = this.now() + timeoutMs;
    while (!this.disposed) {
      const route = this.routeRequest(method, proxyPath);
      if (route && !sameOrigin(route.target.baseUrl, failedBaseUrl)) return route;
      // The engine it went to still serves: only a rollover (or a recovery) in flight can replace it.
      if ((!this.inFlight && !this.recoveryInFlight) || this.now() >= deadline) return null;
      await new Promise((resolve) => {
        const timer = setTimeout(resolve, 100);
        timer.unref?.();
      });
    }
    return null;
  }

  connections(): EnginePoolConnection[] {
    return this.generations
      .filter(isRoutableGeneration)
      .sort((left, right) => left.status === right.status ? 0 : left.status === "primary" ? -1 : 1)
      .map((entry) => this.connectionFor(entry));
  }

  routeRequest(method: string, proxyPath: string): EnginePoolRoute | null {
    const primary = this.generations.find(
      (entry): entry is Generation & { status: "primary" } => entry.status === "primary",
    ) ?? null;
    if (!primary) return null;
    const primaryConnection = this.connectionFor(primary);
    const sessionId = sessionIdFromPath(proxyPath);
    if (sessionId) {
      const owner = this.generationForId(this.sessionOwnership.get(sessionId));
      if (owner) {
        const active = this.activeSessionsByGeneration.get(owner.id)?.has(sessionId) === true;
        if (isPromptishSessionRequest(method, proxyPath) && !active) {
          this.sessionOwnership.delete(sessionId);
        } else {
          return { target: this.connectionFor(owner), fallback: null };
        }
      }
    }

    const requestId = requestIdFromPath(proxyPath);
    if (requestId) {
      const owner = this.generationForId(this.pinnedRequests.get(requestId));
      if (owner) {
        const alternate = isLegacyPromptReply(proxyPath)
          ? this.generations.find((entry) => isRoutableGeneration(entry) && entry.id !== owner.id) ?? null
          : null;
        return {
          target: this.connectionFor(owner),
          fallback: alternate && isRoutableGeneration(alternate) ? this.connectionFor(alternate) : null,
        };
      }
    }

    const draining = this.generations.find(
      (entry): entry is Generation & { status: "draining" } => entry.status === "draining",
    ) ?? null;
    const fallback = draining && isLegacyPromptReply(proxyPath) ? this.connectionFor(draining) : null;
    return { target: primaryConnection, fallback };
  }

  openEventProxy(clientSignal?: AbortSignal): EngineEventProxyLease {
    const controller = new AbortController();
    this.eventProxyControllers.add(controller);
    const signal = clientSignal
      ? AbortSignal.any([clientSignal, controller.signal])
      : controller.signal;
    let released = false;
    return {
      signal,
      release: () => {
        if (released) return;
        released = true;
        this.eventProxyControllers.delete(controller);
      },
    };
  }

  shouldForwardEvent(generationId: string, payload: unknown): boolean {
    const generation = this.generationForId(generationId);
    if (!generation) return false;
    const identifiers = eventIdentifiers(payload);
    if (generation.status === "draining") {
      const ownedSession = [...identifiers.sessionIds].some((id) => this.sessionOwnership.get(id) === generation.id);
      if (ownedSession) {
        for (const requestId of identifiers.requestIds) this.pinnedRequests.set(requestId, generation.id);
      }
      return ownedSession || [...identifiers.requestIds].some((id) => this.pinnedRequests.get(id) === generation.id);
    }
    return ![...identifiers.sessionIds].some((id) => {
      const owner = this.sessionOwnership.get(id);
      return owner !== undefined && owner !== generation.id;
    }) && ![...identifiers.requestIds].some((id) => {
      const owner = this.pinnedRequests.get(id);
      return owner !== undefined && owner !== generation.id;
    });
  }

  observePendingRequests(generationId: string, payload: unknown): string[] {
    const generation = this.generationForId(generationId);
    if (!generation) return [];
    const requestIds: string[] = [];
    for (const record of requestRecords(payload)) {
      const sessionId = firstString(record, ["sessionID", "sessionId", "session_id"]);
      const owner = this.generationForId(sessionId ? this.sessionOwnership.get(sessionId) : undefined) ?? generation;
      const requestId = firstString(record, ["id", "requestID", "requestId", "permissionID", "questionID"]);
      if (requestId) {
        this.pinnedRequests.set(requestId, owner.id);
        requestIds.push(requestId);
      }
      if (sessionId && !this.sessionOwnership.has(sessionId)) this.sessionOwnership.set(sessionId, generation.id);
    }
    return requestIds;
  }

  snapshot(): EnginePoolSnapshot {
    const now = Date.now();
    return {
      generations: this.generations
        .filter((entry) => entry.status !== "dead")
        .map((entry) => ({
          role: entry.status,
          pid: entry.handle.pid ?? null,
          port: portOf(entry.handle.url) || null,
          spawnedAt: entry.spawnedAt,
          drainRemainingMs: entry.drainDeadline === null ? null : Math.max(0, entry.drainDeadline - now),
        })),
    };
  }

  /**
   * Non-idle sessions on every routable engine, for the desktop's restart
   * guard. `unknown` counts the probes that could not be read; callers treat
   * those as busy. Only the given directories are probed, so an instance the
   * engine never booted is not booted just to learn that it is idle.
   */
  async activeSessions(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {
    const sessions = new Set<string>();
    let unknown = 0;
    const targets = this.generations.filter(isRoutableGeneration);
    await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {
      if (!generation.handle.isAlive()) return;
      try {
        const url = new URL("/session/status", generation.handle.url);
        url.searchParams.set("directory", directory);
        const response = await loopbackFetch(url.toString(), {
          headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
          signal: AbortSignal.timeout(5_000),
        });
        if (!response.ok) throw new Error(`status ${response.status}`);
        const payload: unknown = await response.json();
        if (!isRecord(payload)) throw new Error("unreadable status");
        for (const [sessionId, status] of Object.entries(payload)) {
          if (isRecord(status) && status.type !== "idle") sessions.add(sessionId);
        }
      } catch {
        unknown += 1;
      }
    })));
    for (const owned of this.activeSessionsByGeneration.values()) {
      for (const sessionId of owned) sessions.add(sessionId);
    }
    return { sessions: [...sessions], unknown };
  }

  /** Each session's parent (null: a top-level chat), as the engine reported it; it never changes. */
  private readonly sessionParents = new Map<string, string | null>();

  /**
   * The running top-level chats (sub-agent sessions left out), for computer use's "Allow for this chat".
   * `unknown` counts engines or sessions that could not be read; callers then name no chat.
   */
  async activeChats(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {
    const chats = new Set<string>();
    let unknown = 0;
    const targets = this.generations.filter(isRoutableGeneration);
    await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {
      if (!generation.handle.isAlive()) return;
      const headers = { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) };
      try {
        const url = new URL("/session/status", generation.handle.url);
        url.searchParams.set("directory", directory);
        const response = await loopbackFetch(url.toString(), { headers, signal: AbortSignal.timeout(5_000) });
        if (!response.ok) throw new Error(`status ${response.status}`);
        const payload: unknown = await response.json();
        if (!isRecord(payload)) throw new Error("unreadable status");
        const busy = Object.entries(payload).filter(([, status]) => isRecord(status) && status.type !== "idle").map(([sessionId]) => sessionId);
        // Each busy session counts as the chat at the root of its parent chain (a background sub-agent whose
        // chat is idle still counts as that chat). Parents not known yet are read (at most 64 per pass; more:
        // unknown, so no chat is named).
        let reads = 0;
        const parentOf = async (sessionId: string): Promise<string | null> => {
          if (this.sessionParents.has(sessionId)) return this.sessionParents.get(sessionId) ?? null;
          if (++reads > 64) throw new Error("too many sessions to read");
          const info = new URL(`/session/${encodeURIComponent(sessionId)}`, generation.handle.url);
          info.searchParams.set("directory", directory);
          const answer = await loopbackFetch(info.toString(), { headers, signal: AbortSignal.timeout(5_000) });
          if (!answer.ok) throw new Error(`session ${answer.status}`);
          const session: unknown = await answer.json();
          if (!isRecord(session)) throw new Error("unreadable session");
          const parent = typeof session.parentID === "string" && session.parentID ? session.parentID : null;
          this.sessionParents.set(sessionId, parent);
          if (this.sessionParents.size > 4096) this.sessionParents.delete(this.sessionParents.keys().next().value!);
          return parent;
        };
        for (const chat of await rootChats(busy, parentOf)) chats.add(chat);
      } catch {
        unknown += 1;
      }
    })));
    // Prompts on their way (the engine may not report them busy yet): chats the person prompted.
    for (const owned of this.activeSessionsByGeneration.values()) {
      for (const sessionId of owned) {
        // A prompt on its way: its chat (the root of a known chain, else the session itself). A chain too
        // long or a loop: unknown, like rootChats.
        let root = sessionId;
        let steps = 0;
        while (this.sessionParents.get(root) && steps <= 16) { root = this.sessionParents.get(root)!; steps += 1; }
        if (steps > 16) unknown += 1;
        else chats.add(root);
      }
    }
    return { sessions: [...chats], unknown };
  }

  /** Whether `baseUrl` is the engine new work goes to (not a draining one). */
  isPrimaryUrl(baseUrl: string): boolean {
    return this.isPrimaryEndpoint(baseUrl);
  }

  /**
   * Close draining engines now instead of letting their runs finish: their
   * running turns are aborted first. For a change the old engine must not keep
   * serving under (the provider lock); `shouldRetire` picks the engines by URL.
   */
  async retireDraining(shouldRetire: (baseUrl: string) => boolean, reason: string): Promise<number> {
    const targets = this.generations.filter(
      (entry): entry is Generation & { status: "draining" } => entry.status === "draining" && shouldRetire(entry.handle.url),
    );
    for (const generation of targets) {
      const sessions = new Set([
        ...(this.activeSessionsByGeneration.get(generation.id) ?? []),
        ...await this.nonIdleSessionIds(generation).catch(() => [] as string[]),
      ]);
      for (const sessionId of sessions) {
        await this.abortSession(generation, sessionId, { source: "engine_pool.retire_draining", reason }).catch(() => undefined);
      }
      if (sessions.size > 0) await new Promise((resolve) => setTimeout(resolve, abortSettleMs()));
      await this.retire(generation, "forced");
    }
    return targets.length;
  }

  /** Provider sync treats an active drain as busy so it defers instead of parking on waitForDrain(). */
  hasDrainingGeneration(): boolean {
    return this.generations.some((entry) => entry.status === "draining");
  }

  /**
   * Bring the engine onto current config. Idle engines reload in place; busy
   * ones roll over to a standby. Serialized: concurrent requests collapse into
   * one pending rollover so a burst never stacks processes.
   */
  async requestRollover(input: {
    reason: RolloverReason;
    workspace: WorkspaceInfo;
    manual?: boolean;
    awaitPostRefreshSync?: boolean;
    /** Apply a config change through a healthy standby even when the primary is idle. */
    forceStandby?: boolean;
    /** Draining engines this matches are closed (runs aborted) instead of drained. */
    retireDrainingIf?: (baseUrl: string) => boolean;
  }): Promise<RolloverOutcome> {
    if (this.disposed) return { action: "skipped", reason: "unchanged" };
    const request = {
      reason: input.reason,
      workspace: input.workspace,
      manual: input.manual === true,
      awaitPostRefreshSync: input.awaitPostRefreshSync !== false,
      forceStandby: input.forceStandby === true,
      ...(input.retireDrainingIf ? { retireDrainingIf: input.retireDrainingIf } : {}),
    };
    if (this.inFlight) {
      // Latest wins: a manual request keeps its manual flag so it still
      // bypasses the fingerprint guard when it runs.
      this.pendingRollover = {
        ...request,
        manual: request.manual || this.pendingRollover?.manual === true,
        awaitPostRefreshSync: request.awaitPostRefreshSync || this.pendingRollover?.awaitPostRefreshSync === true,
        forceStandby: request.forceStandby || this.pendingRollover?.forceStandby === true,
        ...((request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf)
          ? { retireDrainingIf: request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf }
          : {}),
      };
      this.hooks.logger?.log("info", "Engine rollover coalesced into the in-flight request.", {
        "engine.rollover.reason": request.reason,
      });
      return { action: "coalesced" };
    }
    const run = this.runRollover(request);
    this.inFlight = run;
    try {
      return await run;
    } finally {
      this.inFlight = null;
      const next = this.pendingRollover;
      this.pendingRollover = null;
      if (next && !this.disposed) {
        void this.requestRollover(next).catch(() => undefined);
      }
    }
  }

  private async runRollover(request: {
    reason: RolloverReason;
    workspace: WorkspaceInfo;
    manual: boolean;
    awaitPostRefreshSync: boolean;
    forceStandby: boolean;
    retireDrainingIf?: (baseUrl: string) => boolean;
  }): Promise<RolloverOutcome> {
    const { workspace, reason, manual, awaitPostRefreshSync, forceStandby, retireDrainingIf } = request;
    if (retireDrainingIf) await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
    // The standby reads config from disk at spawn, so make sure the file is
    // current before deciding anything.
    await this.hooks.writeRuntimeConfigFile(this.config).catch(() => undefined);
    const fingerprint = await this.currentFingerprint();
    const primary = this.generations.find((entry) => entry.status === "primary") ?? null;

    if (!manual && primary && primary.fingerprint === fingerprint) {
      // Nothing the engine reads at build time changed. Skipping here is what
      // keeps a repeating no-op sync from spawning an engine every pass.
      return { action: "skipped", reason: "unchanged" };
    }

    const busy = forceStandby
      ? true
      // Unknown activity counts as busy: a busy engine rolls over to a
      // standby, while an in-place dispose would abort whatever it runs.
      : await this.hooks.engineBusy(this.config, workspace).catch(() => true);
    if (!busy) {
      await this.hooks.reloadInPlace(this.config, workspace, { awaitPostRefreshSync });
      if (primary) primary.fingerprint = fingerprint;
      this.hooks.logger?.log("info", "Engine reloaded in place (idle).", {
        "engine.rollover.reason": reason,
      });
      return { action: "reloaded_in_place" };
    }

    const draining = this.generations.find((entry) => entry.status === "draining");
    if (draining) {
      if (forceStandby) {
        this.hooks.logger?.log("info", "Engine standby rollover waiting for the active drain.", {
          "engine.rollover.reason": reason,
        });
        await this.waitForDrain();
        if (this.disposed) return { action: "skipped", reason: "unchanged" };
        return this.runRollover(request);
      }
      // Cap: one primary plus one draining. Park this change; it lands when
      // the current drain finishes.
      this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
      this.hooks.logger?.log("info", "Engine rollover parked behind an active drain.", {
        "engine.rollover.reason": reason,
      });
      return { action: "coalesced" };
    }

    const sinceLastSpawn = Date.now() - this.lastSpawnAt;
    if (!manual && !forceStandby && sinceLastSpawn < minSpawnIntervalMs()) {
      this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
      this.hooks.logger?.log("info", "Engine rollover throttled by the minimum spawn interval.", {
        "engine.rollover.reason": reason,
        "engine.rollover.since_last_spawn_ms": sinceLastSpawn,
      });
      return { action: "coalesced" };
    }

    const outcome = await this.rollOver({ reason, workspace, fingerprint, primary });
    if (retireDrainingIf && outcome.action === "rolled_over") await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
    return outcome;
  }

  private async rollOver(input: {
    reason: RolloverReason;
    workspace: WorkspaceInfo;
    fingerprint: string;
    primary: Generation | null;
  }): Promise<RolloverOutcome> {
    const { reason, workspace, fingerprint, primary } = input;
    this.hooks.logger?.log("info", "Engine rollover requested.", { "engine.rollover.reason": reason });

    let handle: ManagedOpencodeServer;
    this.lastSpawnAt = Date.now();
    try {
      handle = await this.spawn();
    } catch (error) {
      this.hooks.logger?.log("error", "Engine rollover standby failed to start; the live engine is untouched.", {
        "engine.rollover.reason": reason,
        "engine.rollover.failure": error instanceof Error ? error.message : String(error),
      });
      throw error;
    }

    const generation: Generation = {
      id: randomUUID(),
      handle,
      status: "starting",
      spawnedAt: Date.now(),
      fingerprint,
      registryId: null,
      trustedIdentity: null,
      drainTimer: null,
      drainDeadline: null,
      drainActivityWatch: null,
      requestsInFlight: 0,
      drainCheck: null,
    };
    this.generations.push(generation);

    try {
      await this.waitForHealthy(handle);
    } catch (error) {
      // Never flip onto an engine that has not answered: close the partial
      // standby and leave the primary serving.
      this.generations = this.generations.filter((entry) => entry.id !== generation.id);
      await handle.close().catch(() => undefined);
      this.hooks.logger?.log("error", "Engine rollover standby never became healthy; the live engine is untouched.", {
        "engine.rollover.reason": reason,
        "engine.rollover.failure": error instanceof Error ? error.message : String(error),
      });
      throw error;
    }

    if (this.hooks.prepareStandby) {
      try {
        await this.hooks.prepareStandby(this.config, {
          workspace,
          generationId: generation.id,
          baseUrl: handle.url,
          username: handle.username,
          password: handle.password,
        });
      } catch (error) {
        // The standby is healthy and the primary is still serving; a failed
        // seed only means the next sync pass delivers credentials instead.
        this.hooks.logger?.log("warn", "Engine standby preparation failed; flipping anyway.", {
          "engine.rollover.reason": reason,
          "engine.rollover.failure": error instanceof Error ? error.message : String(error),
        });
      }
    }

    if (handle.pid) {
      generation.registryId = randomUUID();
      await registerEngineInstance(this.config, {
        id: generation.registryId,
        pid: handle.pid,
        port: portOf(handle.url),
        url: handle.url,
        startedAt: generation.spawnedAt,
        role: "starting",
        serverRunId: generation.id,
        ownerPid: process.pid,
        authProbe: buildEngineAuthProbeHeader(handle.username, handle.password),
        bin: this.template.bin?.trim() || "opencode",
      }).catch(() => undefined);
    }

    // A prompt the primary accepted but has not started yet drains with it too.
    const drainingSessions = this.withStartingPrompts(primary, await this.nonIdleSessionIds(primary));
    const pinnedRequestIds = await this.pendingRequestIds(primary);
    this.flip(generation, primary, drainingSessions, pinnedRequestIds);

    this.hooks.logger?.log("info", "Engine rollover flipped to the new generation.", {
      "engine.rollover.reason": reason,
      "engine.rollover.pid": handle.pid ?? null,
      "engine.rollover.draining_sessions": drainingSessions.length,
    });

    // Best-effort and off the critical path: the new engine already has disk
    // config; this pushes the runtime-DB MCPs a fresh instance cannot see.
    this.detachPostRefreshSync(workspace);

    if (primary) this.startDrainMonitor(primary, workspace);

    return { action: "rolled_over", generationId: generation.id, drainingSessions: drainingSessions.length };
  }

  /**
   * Point the server at the new generation. Synchronous on purpose: every
   * request resolves its engine from `config`, so there must be no await
   * between demoting the old engine and promoting the new one.
   */
  private flip(
    next: Generation,
    previous: Generation | null,
    drainingSessions: string[],
    pinnedRequestIds: string[],
  ): void {
    next.status = "primary";
    if (previous) previous.status = "draining";

    if (previous) {
      const active = new Set(drainingSessions);
      this.activeSessionsByGeneration.set(previous.id, active);
      for (const sessionId of active) this.sessionOwnership.set(sessionId, previous.id);
      for (const requestId of pinnedRequestIds) this.pinnedRequests.set(requestId, previous.id);
    }

    this.config.opencodeBaseUrl = next.handle.url;
    this.config.opencodeUsername = next.handle.username;
    this.config.opencodePassword = next.handle.password;
    for (const entry of this.config.workspaces) {
      if (entry.workspaceType === "remote") continue;
      entry.baseUrl = next.handle.url;
      entry.opencodeUsername = next.handle.username;
      entry.opencodePassword = next.handle.password;
      entry.directory = entry.path;
    }

    if (previous?.trustedIdentity) {
      try {
        this.hooks.clearTrusted(this.config, previous.trustedIdentity);
      } catch {
        // The registry is advisory; a stale entry must not block the flip.
      }
    }
    next.trustedIdentity = [next.handle.pid ?? "unknown", randomUUID()].join(":");
    try {
      this.hooks.registerTrusted(this.config, {
        baseUrl: next.handle.url,
        identity: next.trustedIdentity,
        isAlive: next.handle.isAlive,
      });
    } catch {
      // Same: diagnostics eligibility only.
    }

    if (next.registryId) void updateEngineInstanceRole(this.config, next.registryId, "primary").catch(() => undefined);
    if (previous?.registryId) {
      void updateEngineInstanceRole(this.config, previous.registryId, "draining").catch(() => undefined);
    }
    this.abortEventProxies();
  }

  /**
   * Watch a draining engine and close it once its runs finish. The grace
   * window bounds inactivity, not total drain time: engine events for an
   * owned session push the deadline out, so a run that is actively making
   * progress is never aborted mid-flight, while a non-idle session that stops
   * reporting anything for the whole window is aborted rather than kept alive
   * forever.
   */
  private startDrainMonitor(generation: Generation, workspace: WorkspaceInfo): void {
    generation.drainDeadline = Date.now() + drainTimeoutMs();
    this.watchDrainActivity(generation);
    // When the drained engine was first found idle with requests still in flight to it.
    let idleWithRequestsSince: number | null = null;
    const tick = async (): Promise<void> => {
      if (generation.status !== "draining") return;
      const remaining = this.withStartingPrompts(generation, await this.nonIdleSessionIds(generation));
      this.updateActiveSessions(generation, remaining);
      if (remaining.length === 0) {
        // A request it still serves (a session create waiting on a cold
        // instance) would fail mid-flight: it closes once they answered.
        if (generation.requestsInFlight > 0) {
          idleWithRequestsSince ??= this.now();
          if (this.now() - idleWithRequestsSince < drainRequestGraceMs()) return;
        }
        await this.retire(generation, "idle");
        return;
      }
      idleWithRequestsSince = null;
      if (generation.drainDeadline !== null && Date.now() >= generation.drainDeadline) {
        this.hooks.logger?.log("warn", "Engine drain saw no session activity for the grace period; aborting the remaining sessions.", {
          "engine.drain.sessions": remaining.join(","),
          "engine.drain.session_count": remaining.length,
        });
        for (const sessionId of remaining) {
          await this.abortSession(generation, sessionId).catch(() => undefined);
        }
        await new Promise((resolve) => setTimeout(resolve, abortSettleMs()));
        await this.retire(generation, "forced");
      }
    };
    const timer = setInterval(() => void tick().catch(() => undefined), drainPollIntervalMs());
    timer.unref?.();
    generation.drainTimer = timer;
    generation.drainCheck = () => void tick().catch(() => undefined);
    // Sessions can finish between the flip and the first poll.
    void tick().catch(() => undefined);
  }

  /**
   * Hold one global event-stream subscription on the draining engine so
   * owned-session activity keeps extending the drain deadline. The client
   * event fan-in only exists while a client is attached; the pool owns this
   * watch so a background run with no observer still counts as active.
   */
  private watchDrainActivity(generation: Generation): void {
    const controller = new AbortController();
    generation.drainActivityWatch = controller;
    void this.runDrainActivityWatch(generation, controller.signal).catch(() => undefined);
  }

  private async runDrainActivityWatch(
    generation: Generation,
    signal: AbortSignal,
  ): Promise<void> {
    while (!signal.aborted && generation.status === "draining") {
      try {
        const url = new URL("/global/event", generation.handle.url);
        const response = await loopbackFetch(url.toString(), {
          headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
          signal,
        });
        if (response.ok && response.body) {
          await this.consumeDrainActivityEvents(generation, response.body, signal);
        }
      } catch {
        // Losing the stream only pauses activity credit. The poll loop still
        // owns the abort decision, and an unreachable engine reports no
        // sessions to drain in the first place.
      }
      if (signal.aborted || generation.status !== "draining") return;
      await new Promise<void>((resolve) => {
        const timer = setTimeout(resolve, drainActivityReconnectMs());
        timer.unref?.();
      });
    }
  }

  private async consumeDrainActivityEvents(
    generation: Generation,
    body: ReadableStream<Uint8Array>,
    signal: AbortSignal,
  ): Promise<void> {
    const reader = body.getReader();
    const frameBuffer = new BoundedSseFrameBuffer();
    try {
      while (!signal.aborted && generation.status === "draining") {
        const chunk = await reader.read();
        if (chunk.done) return;
        const parsed = frameBuffer.push(chunk.value);
        for (const frame of parsed.frames) {
          this.noteDrainActivity(generation, sseFramePayload(frame));
        }
        // A frame that never terminates would buffer without bound; drop the
        // stream and let the watch loop reconnect.
        if (parsed.overflow) return;
      }
    } finally {
      await reader.cancel().catch(() => undefined);
      reader.releaseLock();
    }
  }

  /** An engine event naming an owned session is proof of progress: push the deadline out. */
  private noteDrainActivity(generation: Generation, payload: unknown): void {
    if (payload === null || generation.status !== "draining" || generation.drainDeadline === null) return;
    const owned = this.activeSessionsByGeneration.get(generation.id);
    if (!owned || owned.size === 0) return;
    const { sessionIds } = eventIdentifiers(payload);
    let hasOwnedSession = false;
    for (const sessionId of sessionIds) {
      if (owned.has(sessionId)) {
        hasOwnedSession = true;
        break;
      }
    }
    if (!hasOwnedSession) return;
    const extended = Date.now() + drainTimeoutMs();
    if (extended > generation.drainDeadline) generation.drainDeadline = extended;
  }

  private async retire(generation: Generation, cause: "idle" | "forced" | "shutdown"): Promise<void> {
    if (generation.status === "dead") return;
    generation.status = "dead";
    if (generation.drainTimer) {
      clearInterval(generation.drainTimer);
      generation.drainTimer = null;
    }
    generation.drainActivityWatch?.abort();
    generation.drainActivityWatch = null;
    generation.drainDeadline = null;
    generation.drainCheck = null;
    this.activeSessionsByGeneration.delete(generation.id);
    if (cause !== "shutdown") this.notifyEngineReplaced(generation);
    for (const [sessionId, generationId] of this.sessionOwnership) {
      if (generationId === generation.id) this.sessionOwnership.delete(sessionId);
    }
    for (const [requestId, generationId] of this.pinnedRequests) {
      if (generationId === generation.id) this.pinnedRequests.delete(requestId);
    }
    if (generation.trustedIdentity) {
      try {
        this.hooks.clearTrusted(this.config, generation.trustedIdentity);
      } catch {
        // Advisory only.
      }
      generation.trustedIdentity = null;
    }
    let closeError: unknown;
    try {
      await generation.handle.close();
    } catch (error) {
      closeError = error;
    }
    if (generation.registryId) {
      await removeEngineInstance(this.config, generation.registryId).catch(() => undefined);
      generation.registryId = null;
    }
    this.generations = this.generations.filter((entry) => entry.id !== generation.id);
    for (const resolve of this.drainWaiters) resolve();
    this.drainWaiters.clear();
    this.hooks.logger?.log("info", "Drained engine closed.", { "engine.drain.cause": cause });

    const next = this.pendingRollover;
    if (cause !== "shutdown" && next && !this.inFlight && !this.disposed) {
      this.pendingRollover = null;
      void this.requestRollover(next).catch(() => undefined);
    }
    if (closeError !== undefined && cause === "shutdown") throw closeError;
  }

  /** Close every engine this pool owns. Draining generations go first. */
  async disposeAll(): Promise<void> {
    this.disposed = true;
    this.hooks.onDisposed?.();
    this.pendingRollover = null;
    if (this.recoveryTimer) clearTimeout(this.recoveryTimer);
    this.recoveryTimer = null;
    this.abortEventProxies();
    const errors: unknown[] = [];
    const recoveryOrphan = this.recoveryOrphan;
    this.recoveryOrphan = null;
    if (recoveryOrphan) {
      try {
        await recoveryOrphan.close();
      } catch (error) {
        errors.push(error);
      }
    }
    const ordered = [...this.generations].sort((left, right) => {
      if (left.status === right.status) return 0;
      return left.status === "draining" ? -1 : 1;
    });
    for (const generation of ordered) {
      try {
        await this.retire(generation, "shutdown");
      } catch (error) {
        errors.push(error);
      }
    }
    this.generations = [];
    if (errors.length === 1) throw errors[0];
    if (errors.length > 1) throw new AggregateError(errors, "Failed to close managed engines");
  }

  private async currentFingerprint(): Promise<string> {
    return computeEngineConfigFingerprint(this.template);
  }

  private waitForDrain(): Promise<void> {
    if (!this.generations.some((entry) => entry.status === "draining")) return Promise.resolve();
    return new Promise((resolve) => {
      this.drainWaiters.add(resolve);
    });
  }

  private async recoverDeadPrimary(workspace: WorkspaceInfo): Promise<void> {
    if (this.disposed) return;
    if (this.recoveryInFlight) return this.recoveryInFlight;
    const recovery = this.runDeadPrimaryRecovery(workspace);
    this.recoveryInFlight = recovery;
    try {
      await recovery;
    } finally {
      this.recoveryInFlight = null;
    }
  }

  private async runDeadPrimaryRecovery(workspace: WorkspaceInfo): Promise<void> {
    const previous = this.generations.find((entry) => entry.status === "primary") ?? null;
    let replacement: ManagedOpencodeServer | null = null;
    try {
      if (this.recoveryOrphan) {
        await this.recoveryOrphan.close();
        if (this.recoveryOrphan.isAlive()) throw new Error("Failed replacement remained alive after close");
        this.recoveryOrphan = null;
      }
      if (previous) {
        const liveness = await this.confirmPrimaryDead(previous);
        if (!liveness.dead) {
          // A slow or overloaded engine still carries every live run; closing
          // it here would abort them all. Only a dead one is replaced.
          this.consecutiveConnectionFailures = 0;
          this.hooks.logger?.log("warn", "Managed engine recovery skipped: the engine still answers its health check.", {
            "engine.recovery.reason": liveness.reason,
          });
          return;
        }
        this.hooks.logger?.log("warn", "Managed engine recovery started.", {
          "engine.recovery.reason": liveness.reason,
          "engine.pid": previous.handle.pid ?? null,
        });
        appendRuntimeRestartRecord({
          kind: "engine",
          action: "restarted",
          reason: `engine_unreachable:${liveness.reason}`,
          source: "engine-pool-watchdog",
        });
        try {
          this.hooks.onPrimaryLost?.(this.config);
        } catch {
          // Recovery bookkeeping must never block replacing a dead engine.
        }
        if (previous.trustedIdentity) {
          try {
            this.hooks.clearTrusted(this.config, previous.trustedIdentity);
          } catch {
            // Advisory only; process teardown must still happen.
          }
        }
        await previous.handle.close();
        if (previous.handle.isAlive()) throw new Error("Managed OmniRush process remained alive after close");
        previous.status = "dead";
        if (previous.registryId) await removeEngineInstance(this.config, previous.registryId).catch(() => undefined);
        this.generations = this.generations.filter((entry) => entry.id !== previous.id);
      }

      const handle = await this.spawn();
      replacement = handle;
      await this.waitForHealthy(handle);
      const generation: Generation = {
        id: randomUUID(),
        handle,
        status: "starting",
        spawnedAt: this.now(),
        fingerprint: await this.currentFingerprint(),
        registryId: null,
        trustedIdentity: null,
        drainTimer: null,
        drainDeadline: null,
        drainActivityWatch: null,
        requestsInFlight: 0,
        drainCheck: null,
      };
      this.generations.push(generation);
      if (handle.pid) {
        generation.registryId = randomUUID();
        await registerEngineInstance(this.config, {
          id: generation.registryId,
          pid: handle.pid,
          port: portOf(handle.url),
          url: handle.url,
          startedAt: generation.spawnedAt,
          role: "starting",
          serverRunId: generation.id,
          ownerPid: process.pid,
          authProbe: buildEngineAuthProbeHeader(handle.username, handle.password),
          bin: this.template.bin?.trim() || "opencode",
        }).catch(() => undefined);
      }
      this.flip(generation, null, [], []);
      replacement = null;
      if (previous) this.notifyEngineReplaced(previous);
      this.consecutiveConnectionFailures = 0;
      this.recoveryWorkspace = null;
      if (this.recoveryTimer) clearTimeout(this.recoveryTimer);
      this.recoveryTimer = null;
      this.detachPostRefreshSync(workspace);
    } catch (error) {
      if (replacement) {
        try {
          await replacement.close();
          if (replacement.isAlive()) this.recoveryOrphan = replacement;
        } catch {
          this.recoveryOrphan = replacement;
        }
      }
      this.hooks.logger?.log("error", "Managed engine recovery failed; retry scheduled.", {
        "engine.recovery.failure": error instanceof Error ? error.message : String(error),
      });
      this.scheduleRecovery(workspace);
      throw error;
    }
  }

  /**
   * Decide whether a primary that failed several proxied requests is really
   * gone. A process that exited or a port nobody listens on is dead at once;
   * timeouts and resets are only trusted after every spaced probe fails.
   */
  private async confirmPrimaryDead(generation: Generation): Promise<{ dead: boolean; reason: string }> {
    for (let attempt = 1; attempt <= RECOVERY_PROBE_ATTEMPTS; attempt += 1) {
      if (!generation.handle.isAlive()) return { dead: true, reason: "process_exited" };
      const verdict = await this.probeHealth(generation.handle);
      if (verdict === "ok") return { dead: false, reason: "health_ok" };
      if (verdict === "refused") return { dead: true, reason: "connection_refused" };
      if (attempt < RECOVERY_PROBE_ATTEMPTS) {
        await new Promise((resolve) => {
          const timer = setTimeout(resolve, recoveryProbeIntervalMs());
          timer.unref?.();
        });
      }
    }
    return { dead: true, reason: "health_probe_failed" };
  }

  private async probeHealth(handle: ManagedOpencodeServer): Promise<"ok" | "refused" | "failed"> {
    if (this.hooks.probeHealth) return this.hooks.probeHealth(handle);
    try {
      await loopbackFetch(new URL("/global/health", handle.url).toString(), {
        headers: { Authorization: buildEngineAuthProbeHeader(handle.username, handle.password) },
        signal: AbortSignal.timeout(5_000),
      });
      return "ok";
    } catch (error) {
      return isConnectionRefused(error) ? "refused" : "failed";
    }
  }

  private scheduleRecovery(workspace: WorkspaceInfo): void {
    if (this.disposed || this.recoveryTimer) return;
    this.recoveryWorkspace = workspace;
    const delayMs = Math.max(1, minSpawnIntervalMs() - (this.now() - this.lastRecoveryAt));
    const schedule = this.hooks.schedule ?? ((operation: () => void, delay: number) => setTimeout(operation, delay));
    this.recoveryTimer = schedule(() => {
      this.recoveryTimer = null;
      const pendingWorkspace = this.recoveryWorkspace;
      if (!pendingWorkspace || this.disposed) return;
      this.lastRecoveryAt = this.now();
      void this.recoverDeadPrimary(pendingWorkspace).catch(() => undefined);
    }, delayMs);
    this.recoveryTimer.unref?.();
  }

  private spawn(): Promise<ManagedOpencodeServer> {
    if (this.hooks.spawn) return this.hooks.spawn(this.template);
    return createManagedOpencodeServer({
      bin: this.template.bin,
      cwd: this.template.cwd,
      excludedPorts: this.template.reservedPorts(),
      ...(this.template.spawnTimeoutMs ? { timeoutMs: this.template.spawnTimeoutMs } : {}),
      env: {
        ...this.template.env,
        OPENCODE_CONFIG: this.template.runtimeConfigPath,
      },
    });
  }

  private detachPostRefreshSync(workspace: WorkspaceInfo): void {
    void this.hooks.postRefreshSync(this.config, workspace).catch((error) => {
      this.hooks.logger?.log("error", "Engine post-refresh MCP sync failed.", {
        "workspace.id": workspace.id,
        "engine.post_refresh.failure": error instanceof Error ? error.message : String(error),
      });
    });
  }

  private now(): number {
    return this.hooks.now?.() ?? Date.now();
  }

  private noteStartingPrompt(generation: Generation, sessionId: string): void {
    const now = this.now();
    for (const [id, entry] of this.startingPrompts) {
      if (now - entry.at >= promptStartGraceMs()) this.startingPrompts.delete(id);
    }
    this.startingPrompts.set(sessionId, { generationId: generation.id, at: now });
  }

  /**
   * `nonIdle` (the sessions `generation` reports running) plus those it was
   * sent a prompt for within the start grace period and has not reported
   * running yet. A session seen running is tracked by its status from then on.
   */
  private withStartingPrompts(generation: Generation | null, nonIdle: string[]): string[] {
    if (!generation) return nonIdle;
    const sessions = new Set(nonIdle);
    const now = this.now();
    for (const [sessionId, entry] of this.startingPrompts) {
      if (entry.generationId !== generation.id) continue;
      if (sessions.has(sessionId) || now - entry.at >= promptStartGraceMs()) {
        this.startingPrompts.delete(sessionId);
        continue;
      }
      sessions.add(sessionId);
    }
    return [...sessions];
  }

  /** Tells the pool's owner that `closed` is going away and which engine serves its sessions now. */
  private notifyEngineReplaced(closed: Generation): void {
    const primary = this.generations.find(
      (entry): entry is Generation & { status: "primary" } => entry.status === "primary" && entry.id !== closed.id,
    );
    if (!primary || !this.hooks.onEngineReplaced) return;
    try {
      this.hooks.onEngineReplaced(closed.handle.url, this.connectionFor(primary));
    } catch {
      // Advisory: closing the engine never waits on its readers.
    }
  }

  private generationForUrl(baseUrl: string): Generation | null {
    return this.generations.find((entry) => entry.status !== "dead" && sameOrigin(entry.handle.url, baseUrl)) ?? null;
  }

  private isPrimaryEndpoint(baseUrl: string): boolean {
    try {
      const primary = this.primaryUrl();
      return primary !== null && new URL(primary).origin === new URL(baseUrl).origin;
    } catch {
      return false;
    }
  }

  private async waitForHealthy(handle: ManagedOpencodeServer, attempts = 10): Promise<void> {
    if (this.hooks.waitForHealthy) return this.hooks.waitForHealthy(handle);
    const authorization = buildEngineAuthProbeHeader(handle.username, handle.password);
    for (let attempt = 0; attempt < attempts; attempt += 1) {
      try {
        const response = await loopbackFetch(new URL("/global/health", handle.url).toString(), {
          headers: { Authorization: authorization },
          signal: AbortSignal.timeout(2_000),
        });
        if (response.ok) return;
      } catch {
        // Not up yet.
      }
      await new Promise((resolve) => setTimeout(resolve, 500));
    }
    throw new Error("Standby OmniRush engine did not report healthy");
  }

  private async nonIdleSessionIds(generation: Generation | null): Promise<string[]> {
    if (!generation || generation.status === "dead" || !generation.handle.isAlive()) return [];
    const sessionIds = new Set<string>();
    await Promise.all(this.engineProbeDirectories().map(async (directory) => {
      try {
        const url = new URL("/session/status", generation.handle.url);
        if (directory !== null) url.searchParams.set("directory", directory);
        const response = await loopbackFetch(url.toString(), {
          headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
          signal: AbortSignal.timeout(5_000),
        });
        if (!response.ok) return;
        const payload: unknown = await response.json();
        if (!isRecord(payload)) return;
        for (const [sessionId, status] of Object.entries(payload)) {
          if (isRecord(status) && status.type !== "idle") sessionIds.add(sessionId);
        }
      } catch {
        // Unknown activity never keeps a drain open forever; the grace timeout
        // still bounds it, and an unreachable engine has nothing left to drain.
      }
    }));
    return [...sessionIds];
  }

  private engineProbeDirectories(): Array<string | null> {
    const directories = new Set(
      this.config.workspaces
        .filter((workspace) => workspace.workspaceType !== "remote")
        .map((workspace) => workspace.path),
    );
    return directories.size > 0 ? [...directories] : [null];
  }

  private async pendingRequestIds(generation: Generation | null): Promise<string[]> {
    if (!generation || generation.status === "dead" || !generation.handle.isAlive()) return [];
    const requestIds = new Set<string>();
    await Promise.all(this.engineProbeDirectories().flatMap((directory) =>
      ["/permission", "/question", "/api/permission/request", "/api/question/request"].map(async (path) => {
        try {
          const url = new URL(path, generation.handle.url);
          if (directory !== null) url.searchParams.set("directory", directory);
          const response = await loopbackFetch(url.toString(), {
            headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
            signal: AbortSignal.timeout(2_000),
          });
          if (!response.ok) return;
          const payload: unknown = await response.json();
          for (const requestId of this.observePendingRequests(generation.id, payload)) {
            requestIds.add(requestId);
          }
        } catch {
          // The legacy and v2 surfaces vary by bundled engine version. A missing
          // list never blocks the rollover; path routing still handles scoped ids.
        }
      }),
    ));
    return [...requestIds];
  }

  private updateActiveSessions(generation: Generation, sessionIds: string[]): void {
    const next = new Set(sessionIds);
    this.activeSessionsByGeneration.set(generation.id, next);
    for (const [sessionId, generationId] of this.sessionOwnership) {
      if (generationId === generation.id && !next.has(sessionId)) this.sessionOwnership.delete(sessionId);
    }
    for (const sessionId of next) this.sessionOwnership.set(sessionId, generation.id);
  }

  private connectionFor(generation: Generation & { status: "primary" | "draining" }): EnginePoolConnection {
    return {
      generationId: generation.id,
      role: generation.status,
      baseUrl: generation.handle.url,
      username: generation.handle.username,
      password: generation.handle.password,
    };
  }

  private generationForId(generationId: string | undefined): (Generation & { status: "primary" | "draining" }) | null {
    if (!generationId) return null;
    const generation = this.generations.find((entry) => entry.id === generationId) ?? null;
    return generation && isRoutableGeneration(generation) ? generation : null;
  }

  private abortEventProxies(): void {
    const controllers = [...this.eventProxyControllers];
    this.eventProxyControllers.clear();
    for (const controller of controllers) controller.abort(new Error("OmniRush engine generation changed"));
  }

  private async abortSession(
    generation: Generation,
    sessionId: string,
    cause: { source: string; reason: string } = {
      source: "engine_pool.drain_timeout",
      reason: "draining engine saw no session activity for the grace period",
    },
  ): Promise<void> {
    this.hooks.logger?.log("info", "Aborting OmniRush session from engine pool.", {
      "abort.source": cause.source,
      "abort.initiator": "system",
      "abort.reason": cause.reason,
      "session.id": sessionId,
      "engine.generation_id": generation.id,
    });
    try {
      await loopbackFetch(new URL(`/session/${encodeURIComponent(sessionId)}/abort`, generation.handle.url).toString(), {
        method: "POST",
        headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
        signal: AbortSignal.timeout(5_000),
      });
      this.hooks.logger?.log("info", "OmniRush session abort from engine pool completed.", {
        "abort.source": cause.source,
        "abort.initiator": "system",
        "session.id": sessionId,
        "engine.generation_id": generation.id,
      });
    } catch (error) {
      this.hooks.logger?.log("error", "OmniRush session abort from engine pool failed.", {
        "abort.source": cause.source,
        "abort.initiator": "system",
        "session.id": sessionId,
        "engine.generation_id": generation.id,
        "error.message": error instanceof Error ? error.message : String(error),
      });
      throw error;
    }
  }
}

const poolByConfig = new WeakMap<ServerConfig, EnginePool>();

export function setEnginePoolForConfig(config: ServerConfig, pool: EnginePool): void {
  poolByConfig.set(config, pool);
}

export function enginePoolForConfig(config: ServerConfig): EnginePool | null {
  return poolByConfig.get(config) ?? null;
}

export function clearEnginePoolForConfig(config: ServerConfig): void {
  poolByConfig.delete(config);
}