Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
59.3 KB
Raw Blame
import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, test } from "bun:test";

import {
  BoundedSseFrameBuffer,
  clearEnginePoolForConfig,
  EnginePool,
  computeEngineConfigFingerprint,
  enginePoolForConfig,
  isEngineConnectionFailure,
  setEnginePoolForConfig,
  type EnginePoolHooks,
  type EngineSpawnTemplate,
  rootChats,
} from "./engine-pool.js";
import { createManagedOpencodeServer, type ManagedOpencodeServer } from "./managed-opencode.js";
import { proxyOpencodeRequest, startServer } from "./server.js";
import type { ServerConfig, WorkspaceInfo } from "./types.js";

const ENV_NAMES = [
  "OMNIRUSH_RUNTIME_DB",
  "OMNIRUSH_ENGINE_DRAIN_POLL_MS",
  "OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS",
  "OMNIRUSH_ENGINE_ABORT_SETTLE_MS",
  "OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS",
  "OMNIRUSH_POOL_LOG",
  "OMNIRUSH_POOL_STATE",
  "OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS",
  "OMNIRUSH_ENGINE_PROMPT_START_GRACE_MS",
];

const cleanups: Array<() => void | Promise<void>> = [];
const savedEnv = new Map<string, string | undefined>();

afterEach(async () => {
  while (cleanups.length) await cleanups.pop()?.();
  for (const [name, value] of savedEnv) {
    if (value === undefined) delete process.env[name];
    else process.env[name] = value;
  }
  savedEnv.clear();
});
function setEnv(name: string, value: string): void {
  if (!savedEnv.has(name)) savedEnv.set(name, process.env[name]);
  process.env[name] = value;
}

/**
 * A stand-in engine that answers the endpoints the pool depends on. Busy
 * sessions are read per request from a state file so the test can change what
 * a specific engine reports after it has been spawned.
 */
async function writeFakeEngineBin(root: string): Promise<string> {
  const binPath = join(root, "fake-engine.mjs");
  await writeFile(binPath, [
    "#!/usr/bin/env bun",
    "import { appendFileSync, readFileSync } from 'node:fs';",
    "const portIndex = process.argv.indexOf('--port');",
    "const requestedPort = Number(process.argv[portIndex + 1] ?? 0);",
    "const logPath = process.env.OMNIRUSH_POOL_LOG;",
    "const statePath = process.env.OMNIRUSH_POOL_STATE;",
    "const append = (line) => { if (logPath) appendFileSync(logPath, `${line}\\n`); };",
    "const busySessions = (port, directory) => {",
    "  try {",
    "    const state = JSON.parse(readFileSync(statePath, 'utf8'));",
    "    const value = state[String(port)];",
    "    if (Array.isArray(value)) return value;",
    "    if (!value || typeof value !== 'object') return [];",
    "    return value[directory ?? ''] ?? [];",
    "  } catch { return []; }",
    "};",
    "const eventSessions = (port) => {",
    "  try {",
    "    const state = JSON.parse(readFileSync(statePath, 'utf8'));",
    "    const events = state.__events;",
    "    if (!events || typeof events !== 'object') return null;",
    "    const value = events[String(port)];",
    "    return Array.isArray(value) ? value : null;",
    "  } catch { return null; }",
    "};",
    "const eventMode = (port) => {",
    "  try {",
    "    const state = JSON.parse(readFileSync(statePath, 'utf8'));",
    "    const modes = state.__eventMode;",
    "    if (!modes || typeof modes !== 'object') return null;",
    "    const value = modes[String(port)];",
    "    return typeof value === 'string' ? value : null;",
    "  } catch { return null; }",
    "};",
    "const server = Bun.serve({",
    "  hostname: '127.0.0.1',",
    "  port: requestedPort,",
    "  async fetch(request) {",
    "    const url = new URL(request.url);",
    "    append(`${server.port} ${request.method} ${url.pathname}`);",
    "    // A slow answer (a session create waiting on a cold instance).",
    "    const delay = Number(url.searchParams.get('delay') ?? 0);",
    "    if (delay > 0) await Bun.sleep(delay);",
    "    if (url.pathname === '/session/status') {",
    "      const entries = busySessions(server.port, url.searchParams.get('directory')).map((id) => [id, { type: 'busy' }]);",
    "      return Response.json(Object.fromEntries(entries));",
    "    }",
    "    if (/^\\/session\\/[^/]+$/.test(url.pathname) && request.method === 'GET') {",
    "      let parents = {};",
    "      try { parents = JSON.parse(readFileSync(statePath, 'utf8')).__parents ?? {}; } catch {}",
    "      const id = decodeURIComponent(url.pathname.slice('/session/'.length));",
    "      if (!(id in parents)) return new Response('not found', { status: 404 });",
    "      return Response.json({ id, ...(parents[id] ? { parentID: parents[id] } : {}) });",
    "    }",
    "    if (url.pathname === '/permission') {",
    "      const owner = Number(url.searchParams.get('owner') ?? 0);",
    "      if (owner && owner !== server.port) return Response.json([]);",
    "      const sessionID = url.searchParams.get('session') ?? busySessions(server.port, url.searchParams.get('directory'))[0] ?? `ses_${server.port}`;",
    "      return Response.json([{ id: url.searchParams.get('request') ?? `req_${server.port}`, sessionID }]);",
    "    }",
    "    if (url.pathname === '/event' || url.pathname === '/global/event') {",
    "      const mode = eventMode(server.port);",
    "      // stall: accept the socket but never answer with headers.",
    "      if (mode === 'stall') return new Promise(() => {});",
    "      // giant: one unterminated frame far above the cap on a stream that never closes.",
    "      if (mode === 'giant') {",
    "        const body = new ReadableStream({",
    "          start(controller) {",
    "            const chunk = new TextEncoder().encode('data: ' + 'x'.repeat(1024 * 1024));",
    "            for (let i = 0; i < 6; i += 1) controller.enqueue(chunk);",
    "          },",
    "        });",
    "        return new Response(body, { headers: { 'content-type': 'text/event-stream' } });",
    "      }",
    "    }",
    "    if (url.pathname === '/global/event') {",
    "      const frame = (id) => `data: ${JSON.stringify({ directory: '/workspace', payload: { type: 'session.updated', properties: { sessionID: id } } })}\\n\\n`;",
    "      if (eventSessions(server.port) !== null) {",
    "        // Test-controlled live bus: emit an event per configured session",
    "        // every 50ms on a held stream, re-reading state between beats.",
    "        let timer = null;",
    "        const body = new ReadableStream({",
    "          start(controller) {",
    "            timer = setInterval(() => {",
    "              for (const id of eventSessions(server.port) ?? []) controller.enqueue(new TextEncoder().encode(frame(id)));",
    "            }, 50);",
    "          },",
    "          cancel() { if (timer) clearInterval(timer); },",
    "        });",
    "        return new Response(body, { headers: { 'content-type': 'text/event-stream' } });",
    "      }",
    "      return new Response(frame(`ses_${server.port}`), { headers: { 'content-type': 'text/event-stream' } });",
    "    }",
    "    if (url.pathname === '/event') {",
    "      const frame = (id) => `data: ${JSON.stringify({ type: 'session.updated', properties: { sessionID: id } })}\\n\\n`;",
    "      const sessionID = busySessions(server.port, url.searchParams.get('directory'))[0] ?? `ses_${server.port}`;",
    "      if (url.searchParams.has('hold')) {",
    "        const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode(frame(sessionID))); } });",
    "        return new Response(body, { headers: { 'content-type': 'text/event-stream' } });",
    "      }",
    "      return new Response(frame(sessionID), { headers: { 'content-type': 'text/event-stream' } });",
    "    }",
    "    if (url.pathname.endsWith('/reply')) {",
    "      const owner = Number(url.searchParams.get('owner') ?? 0);",
    "      if (owner && owner !== server.port) return Response.json({ ok: false }, { status: 404 });",
    "    }",
    "    return Response.json({ ok: true, port: server.port, path: url.pathname });",
    "  },",
    "});",
    "console.log(`opencode server listening on http://127.0.0.1:${server.port}`);",
    "process.on('SIGTERM', () => { append(`${server.port} SIGTERM`); server.stop(true); process.exit(0); });",
  ].join("\n"));
  await chmod(binPath, 0o755);
  return binPath;
}

/** A binary that starts and never prints a readiness line. */
async function writeUnreadyEngineBin(root: string): Promise<string> {
  const binPath = join(root, "unready-engine.mjs");
  await writeFile(binPath, [
    "#!/usr/bin/env bun",
    "process.on('SIGTERM', () => process.exit(0));",
    "setInterval(() => undefined, 1000);",
  ].join("\n"));
  await chmod(binPath, 0o755);
  return binPath;
}

type Fixture = {
  root: string;
  config: ServerConfig;
  workspace: WorkspaceInfo;
  template: EngineSpawnTemplate;
  statePath: string;
  logPath: string;
  hookCalls: { reloadInPlace: number; postRefreshSync: number };
  hooks: EnginePoolHooks;
  setBusy: (port: number, sessionIds: string[]) => Promise<void>;
  setBusyForDirectory: (port: number, directory: string, sessionIds: string[]) => Promise<void>;
  setEventSessions: (port: number, sessionIds: string[] | null) => Promise<void>;
  setEventMode: (port: number, mode: "stall" | "giant" | null) => Promise<void>;
  logLines: () => Promise<string[]>;
  setRuntimeConfig: (content: string) => Promise<void>;
  spawnPrimary: () => Promise<ManagedOpencodeServer>;
  runtimeConfigPath: string;
};

async function createFixture(options?: { bin?: "ready" | "unready" }): Promise<Fixture> {
  const root = await mkdtemp(join(tmpdir(), "omnirush-engine-pool-"));
  cleanups.push(() => rm(root, { recursive: true, force: true }));

  const logPath = join(root, "engine.log");
  const statePath = join(root, "busy-state.json");
  const runtimeConfigPath = join(root, "runtime-opencode-config.json");
  await writeFile(statePath, "{}");
  await writeFile(runtimeConfigPath, JSON.stringify({ generation: 1 }));

  for (const name of ENV_NAMES) if (!savedEnv.has(name)) savedEnv.set(name, process.env[name]);
  process.env.OMNIRUSH_RUNTIME_DB = join(root, "runtime.sqlite");
  process.env.OMNIRUSH_POOL_LOG = logPath;
  process.env.OMNIRUSH_POOL_STATE = statePath;
  // Fast drain polling and no spawn throttle so the tests exercise the loop
  // rather than the clock.
  process.env.OMNIRUSH_ENGINE_DRAIN_POLL_MS = "100";
  process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "0";

  const bin = options?.bin === "unready"
    ? await writeUnreadyEngineBin(root)
    : await writeFakeEngineBin(root);

  const workspace: WorkspaceInfo = {
    id: "ws_pool",
    name: "Pool workspace",
    path: root,
    preset: "starter",
    workspaceType: "local",
  };
  const config = {
    host: "127.0.0.1",
    port: 0,
    token: "token",
    hostToken: "host-token",
    configPath: join(root, "server.json"),
    approval: { mode: "auto", timeoutMs: 1000 },
    corsOrigins: ["*"],
    workspaces: [workspace],
    authorizedRoots: [root],
    readOnly: false,
    startedAt: Date.now(),
    tokenSource: "cli",
    hostTokenSource: "cli",
    logFormat: "pretty",
    logRequests: false,
  } as ServerConfig;

  const template: EngineSpawnTemplate = {
    bin,
    cwd: root,
    runtimeConfigPath,
    env: {
      OMNIRUSH_POOL_LOG: logPath,
      OMNIRUSH_POOL_STATE: statePath,
      OPENCODE_CONFIG: runtimeConfigPath,
    },
    reservedPorts: () => [],
    spawnTimeoutMs: 2_000,
  };

  const hookCalls = { reloadInPlace: 0, postRefreshSync: 0 };
  const hooks: EnginePoolHooks = {
    reloadInPlace: async () => { hookCalls.reloadInPlace += 1; },
    engineBusy: async () => {
      const state = JSON.parse(await readFile(statePath, "utf8").catch(() => "{}")) as Record<string, string[]>;
      return Object.values(state).some((sessions) => sessions.length > 0);
    },
    postRefreshSync: async () => { hookCalls.postRefreshSync += 1; },
    writeRuntimeConfigFile: async () => ({ path: runtimeConfigPath }),
    registerTrusted: () => undefined,
    clearTrusted: () => undefined,
  };

  const setBusy = async (port: number, sessionIds: string[]): Promise<void> => {
    const state = JSON.parse(await readFile(statePath, "utf8").catch(() => "{}")) as Record<string, string[] | Record<string, string[]>>;
    if (sessionIds.length === 0) delete state[String(port)];
    else state[String(port)] = sessionIds;
    await writeFile(statePath, JSON.stringify(state));
  };

  const setBusyForDirectory = async (port: number, directory: string, sessionIds: string[]): Promise<void> => {
    const state = JSON.parse(await readFile(statePath, "utf8").catch(() => "{}")) as Record<string, string[] | Record<string, string[]>>;
    const current = state[String(port)];
    const byDirectory = current !== undefined && !Array.isArray(current) ? current : {};
    if (sessionIds.length === 0) delete byDirectory[directory];
    else byDirectory[directory] = sessionIds;
    if (Object.keys(byDirectory).length === 0) delete state[String(port)];
    else state[String(port)] = byDirectory;
    await writeFile(statePath, JSON.stringify(state));
  };

  /**
   * Configure which sessions the fake engine's event bus reports as active.
   * An array (even empty) switches that engine to a held live stream; null
   * restores the legacy single-shot event body.
   */
  const setEventSessions = async (port: number, sessionIds: string[] | null): Promise<void> => {
    const state = JSON.parse(await readFile(statePath, "utf8").catch(() => "{}")) as Record<string, string[] | Record<string, string[]>>;
    const current = state.__events;
    const events = current !== undefined && !Array.isArray(current) ? current : {};
    if (sessionIds === null) delete events[String(port)];
    else events[String(port)] = sessionIds;
    if (Object.keys(events).length === 0) delete state.__events;
    else state.__events = events;
    await writeFile(statePath, JSON.stringify(state));
  };

  /**
   * Force a misbehaving event endpoint on one engine: "stall" accepts the
   * socket but never returns headers; "giant" streams one unterminated frame
   * far above the frame cap; null restores normal behavior.
   */
  const setEventMode = async (port: number, mode: "stall" | "giant" | null): Promise<void> => {
    const state = JSON.parse(await readFile(statePath, "utf8").catch(() => "{}")) as Record<string, unknown>;
    const current = state.__eventMode;
    const modes: Record<string, string> = {};
    if (current !== undefined && typeof current === "object" && current !== null && !Array.isArray(current)) {
      for (const [key, value] of Object.entries(current)) {
        if (typeof value === "string") modes[key] = value;
      }
    }
    if (mode === null) delete modes[String(port)];
    else modes[String(port)] = mode;
    if (Object.keys(modes).length === 0) delete state.__eventMode;
    else state.__eventMode = modes;
    await writeFile(statePath, JSON.stringify(state));
  };

  const spawnPrimary = async (): Promise<ManagedOpencodeServer> => {
    const handle = await createManagedOpencodeServer({ bin, cwd: root, env: template.env });
    cleanups.push(() => handle.close().catch(() => undefined));
    return handle;
  };

  return {
    root,
    config,
    workspace,
    template,
    statePath,
    logPath,
    hookCalls,
    hooks,
    setBusy,
    setBusyForDirectory,
    setEventSessions,
    setEventMode,
    logLines: async () => (await readFile(logPath, "utf8").catch(() => "")).split("\n").filter(Boolean),
    setRuntimeConfig: (content: string) => writeFile(runtimeConfigPath, content),
    spawnPrimary,
    runtimeConfigPath,
  };
}

function portOf(url: string): number {
  return Number(new URL(url).port);
}

async function waitUntil(predicate: () => boolean | Promise<boolean>, timeoutMs: number): Promise<boolean> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (await predicate()) return true;
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  return await predicate();
}

async function createPool(fixture: Fixture): Promise<{ pool: EnginePool; primary: ManagedOpencodeServer }> {
  const primary = await fixture.spawnPrimary();
  const pool = new EnginePool({ config: fixture.config, template: fixture.template, hooks: fixture.hooks });
  setEnginePoolForConfig(fixture.config, pool);
  cleanups.push(async () => {
    clearEnginePoolForConfig(fixture.config);
    await pool.disposeAll().catch(() => undefined);
  });
  pool.adoptPrimary({
    handle: primary,
    fingerprint: await computeEngineConfigFingerprint(fixture.template),
    registryId: null,
    trustedIdentity: null,
  });
  fixture.config.opencodeBaseUrl = primary.url;
  fixture.config.workspaces[0]!.baseUrl = primary.url;
  return { pool, primary };
}

describe("engine pool", () => {
  test("classifies undici header timeouts as engine connection failures", () => {
    const error = new TypeError("fetch failed", {
      cause: { code: "UND_ERR_HEADERS_TIMEOUT", message: "Headers Timeout Error" },
    });

    expect(isEngineConnectionFailure(error)).toBe(true);
  });

  test("classifies a cause-less fetch failure only in the engine transport classifier", () => {
    expect(isEngineConnectionFailure(new TypeError("fetch failed"))).toBe(true);
    expect(isEngineConnectionFailure(new Error("fetch failed"))).toBe(false);
  });

  test("skips entirely when nothing the engine reads at build time changed", async () => {
    const fixture = await createFixture();
    const { pool } = await createPool(fixture);
    await fixture.setBusy(portOf(pool.primaryUrl() ?? "http://127.0.0.1:0"), ["ses_live"]);

    const outcome = await pool.requestRollover({ reason: "unchanged", workspace: fixture.workspace });

    expect(outcome).toEqual({ action: "skipped", reason: "unchanged" });
    expect(pool.snapshot().generations).toHaveLength(1);
    expect(fixture.hookCalls.reloadInPlace).toBe(0);
  });

  test("reloads in place instead of spawning when the engine is idle", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace });

    expect(outcome).toEqual({ action: "reloaded_in_place" });
    expect(fixture.hookCalls.reloadInPlace).toBe(1);
    expect(pool.snapshot().generations).toHaveLength(1);
    // Still the same process; nothing was replaced.
    expect(pool.primaryUrl()).toBe(primary.url);

    // The generation now matches the new config, so an immediate repeat is a
    // no-op rather than a second reload.
    expect(await pool.requestRollover({ reason: "repeat", workspace: fixture.workspace }))
      .toEqual({ action: "skipped", reason: "unchanged" });
    expect(fixture.hookCalls.reloadInPlace).toBe(1);
  });

  test("provider sync holds the serving primary until a standby is healthy and keeps it on spawn failure", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    let markStandbyHealthReached: () => void = () => undefined;
    const standbyHealthReached = new Promise<void>((resolve) => {
      markStandbyHealthReached = resolve;
    });
    let releaseStandbyHealth: () => void = () => undefined;
    const standbyHealthReleased = new Promise<void>((resolve) => {
      releaseStandbyHealth = resolve;
    });
    let standbyHealthChecks = 0;
    fixture.hooks.waitForHealthy = async () => {
      standbyHealthChecks += 1;
      markStandbyHealthReached();
      await standbyHealthReleased;
    };
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const rollover = pool.requestRollover({
      reason: "cloud_provider_sync",
      workspace: fixture.workspace,
      forceStandby: true,
    });
    await standbyHealthReached;

    const url = new URL("http://127.0.0.1/opencode/config");
    const oldPrimaryRead = await proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url),
      url,
      workspace: fixture.workspace,
      proxyPath: "/config",
    });
    expect(await oldPrimaryRead.json()).toMatchObject({ port: oldPort });
    expect(pool.primaryUrl()).toBe(primary.url);
    expect(fixture.hookCalls.reloadInPlace).toBe(0);
    expect((await fixture.logLines()).filter((line) => line === `${oldPort} POST /instance/dispose`)).toEqual([]);

    releaseStandbyHealth();
    expect((await rollover).action).toBe("rolled_over");
    const replacementUrl = pool.primaryUrl();
    expect(replacementUrl).not.toBe(primary.url);
    expect(standbyHealthChecks).toBe(1);

    expect(await pool.requestRollover({
      reason: "cloud_provider_sync_unchanged",
      workspace: fixture.workspace,
      forceStandby: true,
    })).toEqual({ action: "skipped", reason: "unchanged" });
    expect(standbyHealthChecks).toBe(1);

    expect(await waitUntil(() => pool.snapshot().generations.length === 1, 5_000)).toBe(true);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 3 }));
    fixture.hooks.spawn = async () => {
      throw new Error("standby spawn failed");
    };
    await expect(pool.requestRollover({
      reason: "cloud_provider_sync_failed",
      workspace: fixture.workspace,
      forceStandby: true,
    })).rejects.toThrow("standby spawn failed");

    expect(pool.primaryUrl()).toBe(replacementUrl);
    expect(pool.snapshot().generations).toEqual([expect.objectContaining({ role: "primary" })]);
    expect((await fixture.logLines()).some((line) => line.endsWith("POST /instance/dispose"))).toBe(false);
  });

  test("forwards detached post-refresh policy without returning before the in-place switch", async () => {
    const fixture = await createFixture();
    let markReloadStarted: () => void = () => undefined;
    const reloadStarted = new Promise<void>((resolve) => {
      markReloadStarted = resolve;
    });
    let releaseReload: () => void = () => undefined;
    const reloadReleased = new Promise<void>((resolve) => {
      releaseReload = resolve;
    });
    let awaitPostRefreshSync: boolean | undefined;
    fixture.hooks.reloadInPlace = async (_config, _workspace, options) => {
      awaitPostRefreshSync = options?.awaitPostRefreshSync;
      markReloadStarted();
      await reloadReleased;
    };
    const { pool } = await createPool(fixture);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const rollover = pool.requestRollover({
      reason: "workspace_activation",
      workspace: fixture.workspace,
      awaitPostRefreshSync: false,
    });
    await reloadStarted;
    expect(await Promise.race([
      rollover.then(() => true),
      new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 25)),
    ])).toBe(false);

    releaseReload();
    expect(await rollover).toEqual({ action: "reloaded_in_place" });
    expect(awaitPostRefreshSync).toBe(false);
  });

  test("rolls over to a standby when the engine is busy, then closes the drained engine", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace });

    expect(outcome.action).toBe("rolled_over");
    if (outcome.action !== "rolled_over") throw new Error("expected a rollover");
    expect(outcome.drainingSessions).toBe(1);
    // The live run was never disposed.
    expect(fixture.hookCalls.reloadInPlace).toBe(0);
    expect(primary.isAlive()).toBe(true);

    // Requests now resolve to the new engine: the pool stamps the same config
    // object every request path reads.
    const newUrl = pool.primaryUrl();
    expect(newUrl).not.toBe(primary.url);
    expect(fixture.config.opencodeBaseUrl ?? null).toBe(newUrl);
    expect(fixture.config.workspaces[0]?.baseUrl ?? null).toBe(newUrl);
    expect(pool.snapshot().generations.map((entry) => entry.role).sort()).toEqual(["draining", "primary"]);

    // The old engine is closed only once its session finishes.
    await new Promise((resolve) => setTimeout(resolve, 300));
    expect(primary.isAlive()).toBe(true);
    await fixture.setBusy(oldPort, []);

    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
    expect(await waitUntil(async () => pool.snapshot().generations.length === 1, 5_000)).toBe(true);
    expect(pool.snapshot().generations[0]?.role).toBe("primary");
    expect(await fixture.logLines()).toContain(`${oldPort} SIGTERM`);
  });

  test("activeChats: busy sessions count as their root chat; an unreadable session leaves the answer unknown", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const port = portOf(primary.url);
    const setParents = async (parents: Record<string, string | null>) => {
      const state = JSON.parse(await readFile(fixture.statePath, "utf8").catch(() => "{}")) as Record<string, unknown>;
      state.__parents = parents;
      await writeFile(fixture.statePath, JSON.stringify(state));
    };
    await setParents({ ses_root: null, ses_sub: "ses_root", ses_subsub: "ses_sub", ses_other: null });
    // A background sub-agent (two levels down) whose chat is not busy itself: that chat runs.
    await fixture.setBusyForDirectory(port, "/dir-a", ["ses_subsub"]);
    expect(await pool.activeChats(["/dir-a"])).toEqual({ sessions: ["ses_root"], unknown: 0 });
    // The chat and its sub-agent: one chat. Another chat too: two.
    await fixture.setBusyForDirectory(port, "/dir-a", ["ses_root", "ses_sub"]);
    expect(await pool.activeChats(["/dir-a"])).toEqual({ sessions: ["ses_root"], unknown: 0 });
    await fixture.setBusyForDirectory(port, "/dir-a", ["ses_sub", "ses_other"]);
    expect((await pool.activeChats(["/dir-a"])).sessions.sort()).toEqual(["ses_other", "ses_root"]);
    // A session the engine cannot describe: unknown (callers then name no chat).
    await fixture.setBusyForDirectory(port, "/dir-a", ["ses_missing"]);
    expect((await pool.activeChats(["/dir-a"])).unknown).toBe(1);
  });

  test("rootChats follows parents to the root and refuses a chain that is too long", async () => {
    const parents: Record<string, string | null> = { a: null, b: "a", c: "b", x: null };
    expect((await rootChats(["c", "b", "x"], async (id) => parents[id] ?? null)).sort()).toEqual(["a", "x"]);
    const loop: Record<string, string> = { p: "q", q: "p" };
    await expect(rootChats(["p"], async (id) => loop[id] ?? null, 4)).rejects.toThrow("too deep");
  });

  test("reports an active draining generation until it retires", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    expect(pool.hasDrainingGeneration()).toBe(false);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace });

    expect(pool.hasDrainingGeneration()).toBe(true);
    expect(primary.isAlive()).toBe(true);
    await fixture.setBusy(oldPort, []);
    expect(await waitUntil(() => !pool.hasDrainingGeneration(), 5_000)).toBe(true);
    // Retirement removes the draining status before asynchronous process exit.
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("seeds the healthy standby before flipping and flips anyway when seeding fails", async () => {
    const fixture = await createFixture();
    const seeded: Array<{ generationId: string; baseUrl: string; primaryAtSeed: string | null; hasAuth: boolean }> = [];
    let failSeed = false;
    fixture.hooks.prepareStandby = async (_config, standby) => {
      const pool = enginePoolForConfig(fixture.config);
      seeded.push({
        generationId: standby.generationId,
        baseUrl: standby.baseUrl,
        primaryAtSeed: pool?.primaryUrl() ?? null,
        hasAuth: standby.username.length > 0 && standby.password.length > 0,
      });
      if (failSeed) throw new Error("engine auth API rejected the seed");
    };
    const { pool, primary } = await createPool(fixture);
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace });
    expect(outcome.action).toBe("rolled_over");
    if (outcome.action !== "rolled_over") throw new Error("expected a rollover");

    // Seeded exactly once, with the standby's own connection, while the old
    // engine was still primary: the new generation never served unseeded.
    expect(seeded).toHaveLength(1);
    expect(seeded[0]?.generationId).toBe(outcome.generationId);
    const promotedUrl = pool.primaryUrl();
    if (!promotedUrl) throw new Error("expected a promoted primary");
    expect(seeded[0]?.baseUrl).toBe(promotedUrl);
    expect(seeded[0]?.baseUrl).not.toBe(primary.url);
    expect(seeded[0]?.primaryAtSeed).toBe(primary.url);
    expect(seeded[0]?.hasAuth).toBe(true);

    // A failed seed must not strand the pool on the old engine.
    await fixture.setBusy(portOf(primary.url), []);
    expect(await waitUntil(async () => pool.snapshot().generations.length === 1, 5_000)).toBe(true);
    const secondPrimaryUrl = pool.primaryUrl();
    await fixture.setBusy(portOf(secondPrimaryUrl ?? "http://127.0.0.1:0"), ["ses_live_2"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 3 }));
    failSeed = true;
    const second = await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace, manual: true });
    expect(second.action).toBe("rolled_over");
    expect(seeded).toHaveLength(2);
    expect(pool.primaryUrl()).not.toBe(secondPrimaryUrl);
  });

  test("keeps a live session from another workspace on the draining generation", async () => {
    const fixture = await createFixture();
    const secondWorkspace: WorkspaceInfo = {
      ...fixture.workspace,
      id: "ws_pool_second",
      name: "Second pool workspace",
      path: join(fixture.root, "second"),
    };
    fixture.config.workspaces.push(secondWorkspace);
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusyForDirectory(oldPort, fixture.workspace.path, ["ses_first_workspace"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({
      reason: "second_workspace_sync",
      workspace: secondWorkspace,
      forceStandby: true,
    });

    expect(outcome.action).toBe("rolled_over");
    if (outcome.action !== "rolled_over") throw new Error("expected a rollover");
    expect(outcome.drainingSessions).toBe(1);
    expect(primary.isAlive()).toBe(true);
    expect(pool.routeRequest("GET", "/session/ses_first_workspace/message")?.target.baseUrl).toBe(primary.url);

    await fixture.setBusyForDirectory(oldPort, fixture.workspace.path, []);
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("keeps live session and prompt traffic on the draining generation", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");

    const connections = pool.connections();
    const current = connections.find((connection) => connection.role === "primary");
    const draining = connections.find((connection) => connection.role === "draining");
    expect(current?.baseUrl).not.toBe(primary.url);
    expect(draining?.baseUrl).toBe(primary.url);

    expect(pool.routeRequest("GET", "/session/ses_live/message")?.target.baseUrl).toBe(primary.url);
    expect(pool.routeRequest("POST", "/session/ses_live/prompt_async")?.target.baseUrl).toBe(primary.url);
    expect(pool.routeRequest("POST", "/session/ses_new/prompt_async")?.target.baseUrl).toBe(current?.baseUrl);

    const oldEvent = {
      type: "permission.asked",
      properties: { sessionID: "ses_live", requestID: "req_old" },
    };
    expect(pool.shouldForwardEvent(draining?.generationId ?? "", oldEvent)).toBe(true);
    expect(pool.shouldForwardEvent(current?.generationId ?? "", oldEvent)).toBe(false);
    expect(pool.routeRequest("POST", "/permission/req_old/reply")?.target.baseUrl).toBe(primary.url);
    expect(pool.shouldForwardEvent(current?.generationId ?? "", {
      type: "session.created",
      properties: { sessionID: "ses_new" },
    })).toBe(true);

    await fixture.setBusy(oldPort, []);
    expect(await waitUntil(() => pool.connections().length === 1, 5_000)).toBe(true);
    const remainingPrimaryUrl = pool.primaryUrl();
    if (!remainingPrimaryUrl) throw new Error("expected the primary engine to remain live");
    expect(pool.routeRequest("GET", "/session/ses_live/message")?.target.baseUrl).toBe(remainingPrimaryUrl);
  });

  test("proxies owned sessions to the old engine and merges cross-generation reads", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");
    const newPort = portOf(pool.primaryUrl() ?? "http://127.0.0.1:0");
    const proxy = async (path: string, method = "GET") => {
      const url = new URL(`http://127.0.0.1/opencode${path}`);
      return proxyOpencodeRequest({
        config: fixture.config,
        request: new Request(url, { method }),
        url,
        workspace: fixture.workspace,
        proxyPath: path,
      });
    };

    const owned = await (await proxy("/session/ses_live/message")).json() as { port: number };
    const fresh = await (await proxy("/session/ses_new/message")).json() as { port: number };
    expect(owned.port).toBe(oldPort);
    expect(fresh.port).toBe(newPort);

    const statuses = await (await proxy("/session/status")).json() as Record<string, unknown>;
    expect(statuses.ses_live).toEqual({ type: "busy" });
    const pending = await (await proxy("/permission")).json() as Array<{ id: string }>;
    expect(pending.map((item) => item.id).sort()).toEqual([`req_${newPort}`, `req_${oldPort}`].sort());

    const reply = await (await proxy(`/permission/req_${oldPort}/reply`, "POST")).json() as { port: number };
    expect(reply.port).toBe(oldPort);

    const lateUrl = new URL(`http://127.0.0.1/opencode/permission?request=req_late&owner=${newPort}&session=ses_live`);
    const latePending = await proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(lateUrl),
      url: lateUrl,
      workspace: fixture.workspace,
      proxyPath: "/permission",
    });
    expect(await latePending.json()).toEqual([{ id: "req_late", sessionID: "ses_live" }]);
    const lateReply = await (await proxy("/permission/req_late/reply", "POST")).json() as { port: number };
    expect(lateReply.port).toBe(oldPort);

    const currentGeneration = pool.connections().find((connection) => connection.role === "primary");
    if (!currentGeneration) throw new Error("expected a current generation");
    pool.observePendingRequests(currentGeneration.generationId, [{ id: "req_stale", sessionID: "ses_new" }]);
    const staleUrl = new URL(`http://127.0.0.1/opencode/permission/req_stale/reply?owner=${oldPort}`);
    const staleReply = await proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(staleUrl, { method: "POST" }),
      url: staleUrl,
      workspace: fixture.workspace,
      proxyPath: "/permission/req_stale/reply",
    });
    expect(await staleReply.json()).toMatchObject({ port: oldPort });

    const events = await (await proxy("/event")).text();
    expect(events).toContain('"sessionID":"ses_live"');
    expect(events).toContain(`"sessionID":"ses_${newPort}"`);
  });

  test("a drained engine stays up while a request to it is in flight, and names the engine that took over before it closes", async () => {
    const fixture = await createFixture();
    const replaced: Array<{ closed: string; replacement: string; aliveWhenTold: boolean }> = [];
    let first: ManagedOpencodeServer | null = null;
    fixture.hooks.onEngineReplaced = (closed, replacement) => {
      replaced.push({ closed, replacement: replacement.baseUrl, aliveWhenTold: first?.isAlive() === true });
    };
    const { pool, primary } = await createPool(fixture);
    first = primary;
    // A session create went to the engine before the model catalog reload flipped it.
    const release = pool.beginRequest(primary.url);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true });

    expect(outcome).toMatchObject({ action: "rolled_over", drainingSessions: 0 });
    // Idle, yet not closed under the request.
    await new Promise((resolve) => setTimeout(resolve, 400));
    expect(primary.isAlive()).toBe(true);
    release();
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
    expect(replaced).toEqual([{ closed: primary.url, replacement: pool.primaryUrl() ?? "", aliveWhenTold: true }]);
  });

  test("a prompt the engine accepted but has not started when the catalog reload flips drains with it instead of being dropped", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    // The prompt was answered (204) before its run started: the engine does not report the session busy yet.
    pool.beginRequest(primary.url, { method: "POST", proxyPath: "/session/ses_starting/prompt_async" })();
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true });

    expect(outcome).toMatchObject({ action: "rolled_over", drainingSessions: 1 });
    expect(pool.routeRequest("GET", "/session/ses_starting/message")?.target.baseUrl).toBe(primary.url);
    await new Promise((resolve) => setTimeout(resolve, 400));
    expect(primary.isAlive()).toBe(true);
    // The run starts, then ends: the engine closes once it is idle.
    await fixture.setBusy(oldPort, ["ses_starting"]);
    await new Promise((resolve) => setTimeout(resolve, 300));
    expect(primary.isAlive()).toBe(true);
    await fixture.setBusy(oldPort, []);
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("a prompt that never starts holds a drained engine only for the start grace period", async () => {
    const fixture = await createFixture();
    setEnv("OMNIRUSH_ENGINE_PROMPT_START_GRACE_MS", "500");
    const { pool, primary } = await createPool(fixture);
    pool.beginRequest(primary.url, { method: "POST", proxyPath: "/session/ses_never/prompt_async" })();
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    expect(await pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true }))
      .toMatchObject({ action: "rolled_over", drainingSessions: 1 });
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("a request that never answers holds a drained engine only for the grace period", async () => {
    const fixture = await createFixture();
    setEnv("OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS", "300");
    const { pool, primary } = await createPool(fixture);
    pool.beginRequest(primary.url);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    expect((await pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true })).action)
      .toBe("rolled_over");
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("a session created while the startup catalog reload rolls the engine over is answered, never a 500", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    // The create waits on the old engine's cold instance while the reload flips to a standby.
    const url = new URL("http://127.0.0.1/opencode/session?delay=2500");
    const create = proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url, { method: "POST", body: JSON.stringify({ title: "new" }), headers: { "content-type": "application/json" } }),
      url,
      workspace: fixture.workspace,
      proxyPath: "/session",
    });
    const settled = create.then(() => "answered", () => "failed");
    await new Promise((resolve) => setTimeout(resolve, 100));
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true })).action)
      .toBe("rolled_over");
    await new Promise((resolve) => setTimeout(resolve, 300));
    expect(await Promise.race([settled, Promise.resolve("pending")])).toBe("pending");

    const response = await create;
    expect(response.status).toBe(200);
    expect(await response.json()).toMatchObject({ port: oldPort, path: "/session" });
    // Then the drained engine closes.
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
  });

  test("a request that finds its engine gone mid-rollover is sent again to the engine that took over", async () => {
    const fixture = await createFixture();
    let flip: () => void = () => undefined;
    const flipped = new Promise<void>((resolve) => { flip = resolve; });
    let standbyReady: () => void = () => undefined;
    const standbyHealthy = new Promise<void>((resolve) => { standbyReady = resolve; });
    fixture.hooks.prepareStandby = async () => {
      standbyReady();
      await flipped;
    };
    const { pool, primary } = await createPool(fixture);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    const rollover = pool.requestRollover({ reason: "omnirush_model_catalog", workspace: fixture.workspace, forceStandby: true });
    await standbyHealthy;
    // The old engine is gone before the flip; a request is routed to it all the same.
    await primary.close();
    const url = new URL("http://127.0.0.1/opencode/session");
    const create = proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url, { method: "POST", body: "{}", headers: { "content-type": "application/json" } }),
      url,
      workspace: fixture.workspace,
      proxyPath: "/session",
    });
    await new Promise((resolve) => setTimeout(resolve, 200));
    flip();
    expect((await rollover).action).toBe("rolled_over");

    const response = await create;
    expect(response.status).toBe(200);
    expect(await response.json()).toMatchObject({ port: portOf(pool.primaryUrl() ?? "http://127.0.0.1:0"), path: "/session" });
  });

  test("returns a controlled 502 when the selected engine is unreachable", async () => {
    const fixture = await createFixture();
    const { primary } = await createPool(fixture);
    await primary.close();
    const url = new URL("http://127.0.0.1/opencode/config");

    await expect(proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url, { method: "GET" }),
      url,
      workspace: fixture.workspace,
      proxyPath: "/config",
    })).rejects.toMatchObject({
      status: 502,
      code: "opencode_unreachable",
    });
  });

  test.serial("returns an uncaptured 502 for a cause-less loopback proxy fetch failure", async () => {
    const fixture = await createFixture();
    await createPool(fixture);
    const originalFetch = globalThis.fetch;
    const originalTelemetry = globalThis.__omnirushDesktopTelemetry;
    const captured: unknown[] = [];
    const server = await startServer(fixture.config);
    globalThis.__omnirushDesktopTelemetry = {
      captureException(error) {
        captured.push(error);
        return true;
      },
    };
    globalThis.fetch = Object.assign(
      async () => {
        throw new TypeError("fetch failed");
      },
      { preconnect: originalFetch.preconnect },
    );

    try {
      const response = await originalFetch(`http://127.0.0.1:${server.port}/opencode/config`, {
        headers: { Authorization: `Bearer ${fixture.config.token}` },
      });
      expect(response.status).toBe(502);
      expect(await response.json()).toMatchObject({ code: "opencode_unreachable" });
      expect(captured).toEqual([]);
    } finally {
      globalThis.fetch = originalFetch;
      globalThis.__omnirushDesktopTelemetry = originalTelemetry;
      await server.stop();
    }
  });

  test("ends existing event fan-in leases when a generation flips", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const lease = pool.openEventProxy();
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect(lease.signal.aborted).toBe(false);
    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");
    expect(lease.signal.aborted).toBe(true);
    lease.release();
  });

  test("closes an existing event response when a generation flips", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    const url = new URL("http://127.0.0.1/opencode/event?hold=1");
    const response = await proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url),
      url,
      workspace: fixture.workspace,
      proxyPath: "/event",
    });
    const reader = response.body?.getReader();
    if (!reader) throw new Error("expected an event response body");
    expect((await reader.read()).done).toBe(false);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");
    const closed = await Promise.race([
      reader.read().then((result) => result.done),
      new Promise<false>((resolve) => setTimeout(() => resolve(false), 2_000)),
    ]);
    expect(closed).toBe(true);
  });

  test("aborts the remaining sessions once the drain inactivity grace period expires", async () => {
    setEnv("OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS", "300");
    setEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", "100");
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    // This session reports busy forever but never emits an event, so only the
    // inactivity grace timeout can end the drain.
    await fixture.setBusy(oldPort, ["ses_stuck"]);
    await fixture.setEventSessions(oldPort, []);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");

    expect(await waitUntil(() => !primary.isAlive(), 15_000)).toBe(true);
    expect(await waitUntil(async () => (await fixture.logLines()).includes(`${oldPort} SIGTERM`), 2_000)).toBe(true);
    const lines = await fixture.logLines();
    expect(lines).toContain(`${oldPort} POST /session/ses_stuck/abort`);
    expect(lines).toContain(`${oldPort} SIGTERM`);
  });

  test("a rollover that retires matching draining engines aborts their running sessions and closes them at once", async () => {
    setEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", "50");
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    const outcome = await pool.requestRollover({
      reason: "config_changed",
      workspace: fixture.workspace,
      manual: true,
      forceStandby: true,
      retireDrainingIf: (url) => portOf(url) === oldPort,
    });

    expect(outcome.action).toBe("rolled_over");
    expect(primary.isAlive()).toBe(false);
    expect(pool.snapshot().generations.map((entry) => entry.role)).toEqual(["primary"]);
    expect(pool.isPrimaryUrl(pool.primaryUrl()!)).toBe(true);
    expect(pool.isPrimaryUrl(primary.url)).toBe(false);
    const lines = await fixture.logLines();
    expect(lines).toContain(`${oldPort} POST /session/ses_live/abort`);
    expect(lines).toContain(`${oldPort} SIGTERM`);
  });

  test("uses one global activity stream across many workspaces and never aborts an active session", async () => {
    setEnv("OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS", "300");
    setEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", "100");
    const fixture = await createFixture();
    for (let index = 1; index < 32; index += 1) {
      fixture.config.workspaces.push({
        ...fixture.workspace,
        id: `ws_pool_${index}`,
        name: `Pool workspace ${index}`,
        path: join(fixture.root, `workspace-${index}`),
      });
    }
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    // The run keeps streaming: the engine event bus names it continuously.
    await fixture.setBusy(oldPort, ["ses_streaming"]);
    await fixture.setEventSessions(oldPort, ["ses_streaming"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));

    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");
    expect(await waitUntil(async () => (await fixture.logLines()).includes(`${oldPort} GET /global/event`), 2_000))
      .toBe(true);

    // Wait out four full grace periods: an actively working session must
    // never be aborted, so the draining engine stays alive the whole time.
    await new Promise((resolve) => setTimeout(resolve, 1_200));
    expect(primary.isAlive()).toBe(true);
    const activeLines = await fixture.logLines();
    expect(activeLines.filter((line) => line === `${oldPort} GET /global/event`)).toHaveLength(1);
    expect(activeLines.some((line) => line === `${oldPort} GET /event`)).toBe(false);
    expect(activeLines.some((line) => line.includes("/abort"))).toBe(false);

    // The run finishes: the engine idles and the drained generation closes
    // without ever aborting anything.
    await fixture.setEventSessions(oldPort, []);
    await fixture.setBusy(oldPort, []);
    expect(await waitUntil(() => !primary.isAlive(), 5_000)).toBe(true);
    expect((await fixture.logLines()).some((line) => line.includes("/abort"))).toBe(false);
    expect(await fixture.logLines()).toContain(`${oldPort} SIGTERM`);
  });

  test("never runs more than one primary and one draining engine", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    expect((await pool.requestRollover({ reason: "first", workspace: fixture.workspace })).action)
      .toBe("rolled_over");

    // A burst of further changes while the first drain is still running must
    // coalesce instead of stacking processes.
    for (let index = 0; index < 4; index += 1) {
      await fixture.setRuntimeConfig(JSON.stringify({ generation: 3 + index }));
      const outcome = await pool.requestRollover({ reason: `burst_${index}`, workspace: fixture.workspace });
      expect(outcome).toEqual({ action: "coalesced" });
    }

    const roles = pool.snapshot().generations.map((entry) => entry.role).sort();
    expect(roles).toEqual(["draining", "primary"]);

    // The settings API must not report a parked change as already applied.
    // It preserves the streaming generation while the saved choice waits.
    const server = await startServer(fixture.config);
    try {
      const response = await fetch(`http://127.0.0.1:${server.port}/runtime-config/best-practices`, {
        method: "PUT",
        headers: { authorization: `Bearer ${fixture.config.token}`, "content-type": "application/json" },
        body: JSON.stringify({ enabled: false }),
      });
      expect(response.status).toBe(200);
      expect(await response.json()).toEqual({ ok: true, changed: true, enabled: false, engine: { status: "deferred" } });
      expect(primary.isAlive()).toBe(true);
      expect((await fixture.logLines()).some((line) => line.includes("/abort"))).toBe(false);
    } finally {
      await server.stop();
    }
  });

  test("leaves the live engine serving when the standby cannot start", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    // Point the template at a binary that never reports readiness.
    fixture.template.bin = await writeUnreadyEngineBin(fixture.root);

    await expect(pool.requestRollover({ reason: "bad_standby", workspace: fixture.workspace })).rejects.toThrow();

    expect(primary.isAlive()).toBe(true);
    expect(pool.primaryUrl()).toBe(primary.url);
    expect(fixture.config.opencodeBaseUrl).toBe(primary.url);
    expect(pool.snapshot().generations).toHaveLength(1);
    expect(pool.snapshot().generations[0]?.role).toBe("primary");
  });

  test("disposeAll closes the draining engine as well as the primary", async () => {
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    await fixture.setBusy(portOf(primary.url), ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    const outcome = await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace });
    expect(outcome.action).toBe("rolled_over");
    const replacementUrl = pool.primaryUrl();

    await pool.disposeAll();

    expect(primary.isAlive()).toBe(false);
    expect(pool.snapshot().generations).toHaveLength(0);
    const lines = await fixture.logLines();
    expect(lines).toContain(`${portOf(primary.url)} SIGTERM`);
    expect(lines).toContain(`${portOf(replacementUrl ?? "http://127.0.0.1:0")} SIGTERM`);
  });
});

describe("BoundedSseFrameBuffer", () => {
  const encoder = new TextEncoder();

  test("splits frames across chunk boundaries and both delimiter styles", () => {
    const buffer = new BoundedSseFrameBuffer(64);
    expect(buffer.push(encoder.encode("data: one\n\ndata: tw"))).toEqual({ frames: ["data: one"], overflow: false });
    expect(buffer.push(encoder.encode("o\r\n\r\ndata: three"))).toEqual({ frames: ["data: two"], overflow: false });
    expect(buffer.push(encoder.encode("\n\n"))).toEqual({ frames: ["data: three"], overflow: false });
  });

  test("keeps completed frames and flags overflow once the unterminated remainder exceeds the cap", () => {
    const buffer = new BoundedSseFrameBuffer(8);
    expect(buffer.push(encoder.encode("data: a\n\n0123456789"))).toEqual({ frames: ["data: a"], overflow: true });
  });

  test("never overflows while frames keep terminating", () => {
    const buffer = new BoundedSseFrameBuffer(16);
    for (let index = 0; index < 100; index += 1) {
      expect(buffer.push(encoder.encode("data: abcdefgh\n\n"))).toEqual({ frames: ["data: abcdefgh"], overflow: false });
    }
  });
});

describe("engine event stream bounds", () => {
  const proxyEvent = async (fixture: Fixture, path: string): Promise<Response> => {
    const url = new URL(`http://127.0.0.1/opencode${path}`);
    return proxyOpencodeRequest({
      config: fixture.config,
      request: new Request(url),
      url,
      workspace: fixture.workspace,
      proxyPath: url.pathname.slice("/opencode".length),
    });
  };

  test("a sibling that never returns headers does not stall the client event stream", async () => {
    setEnv("OMNIRUSH_ENGINE_EVENT_ESTABLISH_TIMEOUT_MS", "300");
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_live"]);
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");
    const newPort = portOf(pool.primaryUrl() ?? "http://127.0.0.1:0");
    // The draining sibling accepts the socket but never answers with headers.
    await fixture.setEventMode(oldPort, "stall");

    const startedAt = Date.now();
    const response = await proxyEvent(fixture, "/event");
    const events = await response.text();

    expect(response.status).toBe(200);
    expect(Date.now() - startedAt).toBeLessThan(2_000);
    expect(events).toContain(`"sessionID":"ses_${newPort}"`);
  });

  test("a quiet live event stream stays open past the establishment deadline", async () => {
    setEnv("OMNIRUSH_ENGINE_EVENT_ESTABLISH_TIMEOUT_MS", "300");
    const fixture = await createFixture();
    await createPool(fixture);

    // hold=1 sends one frame and then goes silent on an open stream.
    const response = await proxyEvent(fixture, "/event?hold=1");
    expect(response.status).toBe(200);
    if (!response.body) throw new Error("expected a streaming body");
    const reader = response.body.getReader();
    const first = await reader.read();
    expect(first.done).toBe(false);

    // The establishment deadline is long past; only establishment is bounded,
    // so the silent-but-live body must still be open.
    const idle = await Promise.race([
      reader.read().then((chunk) => (chunk.done ? "closed" : "data")),
      new Promise<string>((resolve) => setTimeout(() => resolve("open"), 1_200)),
    ]);
    expect(idle).toBe("open");
    await reader.cancel().catch(() => undefined);
  });

  test("closes an event connection whose frame never terminates instead of buffering it", async () => {
    const fixture = await createFixture();
    const { primary } = await createPool(fixture);
    await fixture.setEventMode(portOf(primary.url), "giant");

    const response = await proxyEvent(fixture, "/event");
    expect(response.status).toBe(200);
    // The runaway frame hits the cap: the connection is dropped and the
    // merged stream closes instead of buffering the frame forever.
    const events = await response.text();
    expect(events).toBe("");
  });

  test("the drain activity watch drops a runaway frame stream and reconnects", async () => {
    setEnv("OMNIRUSH_ENGINE_DRAIN_ACTIVITY_RECONNECT_MS", "100");
    const fixture = await createFixture();
    const { pool, primary } = await createPool(fixture);
    const oldPort = portOf(primary.url);
    await fixture.setBusy(oldPort, ["ses_stuck"]);
    await fixture.setEventMode(oldPort, "giant");
    await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
    expect((await pool.requestRollover({ reason: "config_changed", workspace: fixture.workspace })).action)
      .toBe("rolled_over");

    // Every malformed stream is dropped at the frame cap and re-dialed;
    // without the bound the first connection would buffer forever and a
    // second dial would never happen.
    expect(await waitUntil(async () => {
      const lines = await fixture.logLines();
      return lines.filter((line) => line === `${oldPort} GET /global/event`).length >= 2;
    }, 10_000)).toBe(true);
  });
});