Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
35.1 KB
Raw Blame
import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
import { execFile } from "node:child_process";
import { randomBytes } from "node:crypto";
import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { zstdDecompressSync } from "node:zlib";

import { startCaptureService, type CaptureService, type CaptureServiceOptions } from "./capture-client.js";
import { FakeArchiveServer, slowPartTwo } from "./session-archive/fake-archive-server.js";
import { openArchive } from "./session-archive/test-helpers.js";

/** Schema 1/2 trace uploads carry their events only in files[0] (__omnirush__/trace.json): read them back as `trace`. */
function withTraceEvents<T>(envelope: T): T {
  const record = envelope as Record<string, unknown>;
  if (record.trace !== undefined || !Array.isArray(record.files)) return envelope;
  const file = (record.files as Array<{ path?: string; content?: string }>).find((item) => item.path === "__omnirush__/trace.json");
  if (!file?.content) return envelope;
  return { ...record, trace: (JSON.parse(file.content) as { events?: unknown[] }).events ?? [] } as T;
}


/**
 * The capture worker: the session uploader's and the archiver's work runs off the
 * main event loop, a resumed session starts with the whole workspace, and
 * shutdown with work in flight settles in bounded time with archive uploads
 * aborted first.
 */

const execFileAsync = promisify(execFile);
const cleanups: Array<() => unknown> = [];

afterEach(async () => {
  while (cleanups.length) await cleanups.pop()?.();
});

type Envelope = {
  session_id: string;
  snapshot_type: string;
  trigger: string;
  files_scope: string;
  changed_paths?: string[];
  files: Array<{ path: string; content: string }>;
  manifest: Array<{ path: string; sha256: string }>;
};

async function tempDir(prefix: string): Promise<string> {
  const dir = await mkdtemp(join(tmpdir(), `omnirush-capture-${prefix}-`));
  cleanups.push(() => rm(dir, { recursive: true, force: true }));
  return dir;
}

async function git(root: string, ...args: string[]): Promise<void> {
  await execFileAsync("git", ["-C", root, "-c", "commit.gpgsign=false", "-c", "user.name=Dev", "-c", "[email protected]", ...args]);
}

/**
 * A git workspace of `count` source files of about 5 KiB each, every one with
 * an e-mail address and a secret-named assignment, so the scrubber rewrites
 * all of them: a scan reads, hashes and scrubs every byte.
 */
async function syntheticWorkspace(count: number): Promise<string> {
  const root = await tempDir("workspace");
  await git(root, "init", "-q");
  const words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"];
  const files = Array.from({ length: count }, (_, index) => index);
  for (let start = 0; start < files.length; start += 256) {
    await Promise.all(files.slice(start, start + 256).map(async (index) => {
      const dir = join(root, "src", `m${index % 64}`);
      await mkdir(dir, { recursive: true });
      const lines = [`// module ${index}`, `export const owner = "dev${index}@example.com";`, `const password = "Pa55-word-${index}-x";`];
      for (let line = 0; line < 90; line += 1) lines.push(`export const v${line} = "${words[(index + line) % words.length]} ${index * 31 + line}";`);
      await writeFile(join(dir, `file${index}.ts`), `${lines.join("\n")}\n`);
    }));
  }
  return root;
}

function uploadSink() {
  const compressed: Array<{ sessionId: string; bytes: Uint8Array }> = [];
  return {
    compressed,
    // Only kept here: decoding a large envelope on this thread would itself stall the loop being measured.
    upload: async (sessionId: string, bytes: Uint8Array) => {
      compressed.push({ sessionId, bytes });
      return Response.json({ ok: true }, { status: 201 });
    },
    envelopes: (): Envelope[] => compressed.map((item) => withTraceEvents(JSON.parse(zstdDecompressSync(item.bytes).toString("utf8")))),
  };
}

function service(options: Partial<CaptureServiceOptions> & Pick<CaptureServiceOptions, "stateDir" | "sessionUploader">): CaptureService {
  const capture = startCaptureService({
    appVersion: "0.0.0-test",
    engineVersion: "0.0.0-test",
    log: () => undefined,
    archive: { enabled: false, excludedDirs: [] },
    worker: true,
    ...options,
  });
  cleanups.push(() => capture.stop());
  return capture;
}

/** The longest the main event loop went without running a 5 ms timer, less the 5 ms. */
function loopMonitor() {
  let last = performance.now();
  let worst = 0;
  const timer = setInterval(() => {
    const now = performance.now();
    worst = Math.max(worst, now - last - 5);
    last = now;
  }, 5);
  return { stop: () => {
    clearInterval(timer);
    return worst;
  } };
}

async function until(check: () => boolean | Promise<boolean>, timeoutMs: number, what: string): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  while (!(await check())) {
    if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`);
    await new Promise((resolvePromise) => setTimeout(resolvePromise, 20));
  }
}

describe("capture worker", () => {
  // These pin the worker's exact envelope sequence; capture context would add a trace of its own.
  let contextSwitch: string | undefined;
  beforeAll(() => {
    contextSwitch = process.env.OMNIRUSH_CAPTURE_CONTEXT;
    process.env.OMNIRUSH_CAPTURE_CONTEXT = "0";
  });
  afterAll(() => {
    if (contextSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_CONTEXT;
    else process.env.OMNIRUSH_CAPTURE_CONTEXT = contextSwitch;
  });
  test("a 6,000-file start snapshot is read, scrubbed, compressed and uploaded without a main-loop stall over 100 ms", async () => {
    const root = await syntheticWorkspace(6_000);
    const sink = uploadSink();
    const capture = service({ stateDir: await tempDir("state"), sessionUploader: { upload: sink.upload } });
    const monitor = loopMonitor();
    const started = performance.now();
    capture.startSession("session-offload-0001", "workspace-offload", root);
    await until(() => sink.compressed.length > 0, 90_000, "the start snapshot");
    const elapsedMs = performance.now() - started;
    const worstStallMs = monitor.stop();
    expect(capture.mode()).toBe("worker");

    const [start] = sink.envelopes();
    expect(start).toMatchObject({ snapshot_type: "start", trigger: "session_start", files_scope: "full" });
    const sources = start!.files.filter((file) => file.path.startsWith("src/"));
    expect(sources).toHaveLength(6_000);
    // The privacy rails ran on the worker exactly as in-process.
    expect(sources.every((file) => !file.content.includes("@example.com") && file.content.includes("[REDACTED_PII]"))).toBe(true);
    expect(sources.every((file) => file.content.includes('const password = "[REDACTED]";'))).toBe(true);
    const diagnostics = await capture.diagnostics();
    expect(diagnostics?.metrics.fileRedactions).toBeGreaterThanOrEqual(6_000);
    // Seconds of reading, hashing and scrubbing, and the loop kept running throughout.
    expect(elapsedMs).toBeGreaterThan(200);
    expect(worstStallMs).toBeLessThan(100);
    await capture.stop();
  }, 120_000);

  test("a resumed session's start snapshot through the worker carries the whole workspace, with what changed while the app was closed", async () => {
    const root = await syntheticWorkspace(50);
    const stateDir = await tempDir("state");
    const sessionId = "session-resume-0001";
    const firstSink = uploadSink();
    const first = service({ stateDir, sessionUploader: { upload: firstSink.upload } });
    first.startSession(sessionId, "workspace-resume", root);
    await first.stop();
    expect(firstSink.envelopes().map((item) => [item.snapshot_type, item.files_scope])).toEqual([["start", "full"], ["end", "changed"]]);

    await writeFile(join(root, "src", "m7", "file7.ts"), "export const edited = true;\n");
    const secondSink = uploadSink();
    const second = service({ stateDir, sessionUploader: { upload: secondSink.upload } });
    second.startSession(sessionId, "workspace-resume", root);
    await second.idle();
    const [resumed] = secondSink.envelopes();
    expect(resumed).toMatchObject({ snapshot_type: "start", trigger: "resume", files_scope: "full" });
    expect(resumed!.changed_paths).toBeUndefined();
    const sources = resumed!.files.filter((file) => file.path.startsWith("src/"));
    expect(sources).toHaveLength(50);
    expect(sources.find((file) => file.path === "src/m7/file7.ts")?.content).toBe("export const edited = true;\n");
    expect(resumed!.manifest).toHaveLength(50);
    await second.stop();
  }, 60_000);

  test("stopping with a start snapshot in flight finishes it and the session's end snapshot, then ends the worker", async () => {
    const root = await syntheticWorkspace(3_000);
    const sink = uploadSink();
    const capture = service({ stateDir: await tempDir("state"), sessionUploader: { upload: sink.upload } });
    capture.startSession("session-stop-00001", "workspace-stop", root);
    await capture.diagnostics();
    const stopping = performance.now();
    await capture.stop();
    expect(performance.now() - stopping).toBeLessThan(15_000);
    expect(capture.mode()).toBe("down");
    expect(sink.envelopes().map((item) => item.snapshot_type)).toEqual(["start", "end"]);
    // Nothing runs after stop: calls are dropped, queries answer at once.
    capture.startSession("session-stop-00002", "workspace-stop", root);
    expect(await capture.diagnostics()).toBeNull();
  }, 60_000);

  test("stopping aborts an archive part upload in flight on the main thread within a second", async () => {
    const root = await tempDir("project");
    await writeFile(join(root, "README.md"), "# project\n".repeat(200));
    await git(root, "init", "-q");
    await git(root, "add", "README.md");
    await git(root, "commit", "-q", "-m", "initial");
    const archive = new FakeArchiveServer();
    archive.partSize = 256;
    const slow = slowPartTwo(archive);
    const engine = Bun.serve({
      hostname: "127.0.0.1",
      port: 0,
      fetch: (request) => (new URL(request.url).pathname.endsWith("/message") ? Response.json([]) : Response.json({ id: "ses_archive_0001" })),
    });
    cleanups.push(() => engine.stop(true));
    const sink = uploadSink();
    const capture = service({
      stateDir: await tempDir("state"),
      sessionUploader: { upload: sink.upload },
      archive: {
        enabled: true,
        excludedDirs: [],
        request: (path, init) => archive.respond(`https://api.omnirush.test/omnirush/${path}`, {
          method: init.method,
          headers: { authorization: `Bearer ${archive.token}` },
          ...(init.body === undefined ? {} : { body: init.body }),
        }),
        refreshAccessToken: async () => null,
        fetch: slow.fetch,
        baseIdleMs: 0,
      },
    });
    capture.startSession("ses_archive_0001", "workspace-archive", root);
    capture.archiveSessionStarted("ses_archive_0001", root, { baseUrl: `http://127.0.0.1:${engine.port}`, headers: [], search: "", engine: "v1" });
    await slow.started;

    slow.markCommand();
    await capture.stop();
    expect(slow.puts).toEqual([{ part: 2, bytes: 256, abortedAfterMs: expect.any(Number), completed: false }]);
    expect(slow.puts[0]!.abortedAfterMs!).toBeLessThan(1_000);
    // The session uploader still delivered the session's end snapshot on the way out.
    expect(sink.envelopes().map((item) => item.snapshot_type)).toContain("end");
  }, 60_000);

  test("a stop that runs out of time aborts the session upload this thread still serves for the worker, and does not wait for it", async () => {
    const root = await syntheticWorkspace(5);
    const signals: AbortSignal[] = [];
    let started!: () => void;
    const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });
    const capture = service({
      stateDir: await tempDir("state"),
      stopTimeoutMs: 300,
      sessionUploader: {
        // The gateway never answers and this hook ignores its signal, as a stalled upload on the main thread would.
        upload: (_sessionId, _bytes, signal) => {
          signals.push(signal!);
          started();
          return new Promise<Response>(() => undefined);
        },
      },
    });
    capture.startSession("session-served-0001", "workspace-served", root);
    await inFlight;
    expect(signals[0]!.aborted).toBe(false);
    const stopping = performance.now();
    await capture.stop();
    expect(performance.now() - stopping).toBeLessThan(5_000);
    expect(capture.mode()).toBe("down");
    // Terminating the worker ended the request it had asked this thread to make.
    expect(signals).toHaveLength(1);
    expect(signals[0]!.aborted).toBe(true);
  }, 60_000);

  test("in-process, a stop that runs out of time aborts the upload in flight and spools it for the next start", async () => {
    const root = await syntheticWorkspace(5);
    const stateDir = await tempDir("state");
    const signals: AbortSignal[] = [];
    let started!: () => void;
    const inFlight = new Promise<void>((resolvePromise) => { started = resolvePromise; });
    const capture = service({
      stateDir,
      worker: false,
      stopTimeoutMs: 300,
      sessionUploader: {
        upload: (_sessionId, _bytes, signal) => {
          signals.push(signal!);
          started();
          return new Promise<Response>(() => undefined);
        },
      },
    });
    capture.startSession("session-local-stop-01", "workspace-local-stop", root);
    await inFlight;
    const stopping = performance.now();
    await capture.stop();
    expect(performance.now() - stopping).toBeLessThan(5_000);
    expect(signals[0]!.aborted).toBe(true);
    // The start snapshot and the session's end snapshot wait in the spool, the end one never sent.
    const spoolDir = join(stateDir, "omnirush-upload-spool");
    const spooledTypes = async () => (await Promise.all((await readdir(spoolDir).catch(() => [] as string[]))
      .filter((name) => name.endsWith(".json"))
      .map(async (name) => (JSON.parse(await readFile(join(spoolDir, name), "utf8")) as { snapshot_type: string }).snapshot_type))).sort();
    await until(async () => (await spooledTypes()).length === 2, 10_000, "the spooled snapshots");
    expect(await spooledTypes()).toEqual(["end", "start"]);
    expect(signals).toHaveLength(1);
  }, 60_000);

  test("stopping packs the project archive's final archives on the worker, unless the account is gone", async () => {
    const results: string[][] = [];
    for (const archiveFinals of [undefined, Promise.resolve(false)]) {
      const root = await tempDir("project");
      await writeFile(join(root, "README.md"), "# project\n");
      await git(root, "init", "-q");
      await git(root, "add", "README.md");
      await git(root, "commit", "-q", "-m", "initial");
      const archive = new FakeArchiveServer();
      const engine = Bun.serve({
        hostname: "127.0.0.1",
        port: 0,
        fetch: (request) => (new URL(request.url).pathname.endsWith("/message") ? Response.json([]) : Response.json({ id: "ses_final_stop_01" })),
      });
      cleanups.push(() => engine.stop(true));
      const stateDir = await tempDir("state");
      const capture = service({
        stateDir,
        sessionUploader: { upload: uploadSink().upload },
        archive: {
          enabled: true,
          excludedDirs: [],
          request: (path, init) => archive.respond(`https://api.omnirush.test/omnirush/${path}`, {
            method: init.method,
            headers: { authorization: `Bearer ${archive.token}` },
            ...(init.body === undefined ? {} : { body: init.body }),
          }),
          refreshAccessToken: async () => null,
          fetch: archive.respond,
          baseIdleMs: 0,
        },
      });
      capture.startSession("ses_final_stop_01", "workspace-final", root);
      capture.archiveSessionStarted("ses_final_stop_01", root, { baseUrl: `http://127.0.0.1:${engine.port}`, headers: [], search: "", engine: "v1" });
      await capture.idle();
      expect(archive.objects().map((object) => object.request.kind)).toEqual(["base"]);
      // Edited after the chat's last turn, then the app quits (or the user signs out).
      await writeFile(join(root, "notes.md"), "after the last turn\n");
      await capture.stop(archiveFinals === undefined ? {} : { archiveFinals });
      results.push(await readdir(join(stateDir, "omnirush-archive", "queue")));
    }
    expect(results[0]).toHaveLength(1);
    expect(results[1]).toEqual([]);
  }, 60_000);

  test("the archiver on the worker reads the all-folders policy from the key and archives a folder without .git only while it is on", async () => {
    const engine = Bun.serve({
      hostname: "127.0.0.1",
      port: 0,
      fetch: (request) => (new URL(request.url).pathname.endsWith("/message") ? Response.json([]) : Response.json({ id: "ses_folder_0001" })),
    });
    cleanups.push(() => engine.stop(true));
    for (const policy of [undefined, { all_folders: true }]) {
      const home = await tempDir("home");
      const root = join(home, "notes");
      await mkdir(root);
      await writeFile(join(root, "plan.md"), "# plan\n");
      const archive = new FakeArchiveServer();
      archive.policy = policy;
      const refreshFlags: unknown[] = [];
      const capture = service({
        stateDir: await tempDir("state"),
        sessionUploader: { upload: uploadSink().upload },
        archive: {
          enabled: true,
          excludedDirs: [],
          folderGate: { homeDir: home },
          request: (path, init) => {
            refreshFlags.push("refresh" in init ? init.refresh : "absent");
            return archive.respond(`https://api.omnirush.test/omnirush/${path}`, {
              method: init.method,
              headers: { authorization: `Bearer ${archive.token}` },
              ...(init.body === undefined ? {} : { body: init.body }),
            });
          },
          refreshAccessToken: async () => null,
          fetch: archive.respond,
          baseIdleMs: 0,
        },
      });
      capture.startSession("ses_folder_0001", "workspace-folder", root);
      capture.archiveSessionStarted("ses_folder_0001", root, { baseUrl: `http://127.0.0.1:${engine.port}`, headers: [], search: "", engine: "v1" });
      await until(() => archive.calls.some((call) => call.path === "archives/key"), 20_000, "the key read");
      await capture.idle();
      expect(capture.mode()).toBe("worker");
      if (policy) {
        await until(() => archive.objects().length === 1, 20_000, "the folder's base archive");
        expect(archive.objects()[0]!.request).toMatchObject({ session_id: "ses_folder_0001", kind: "base", marker: "folder" });
      } else {
        expect(archive.callPaths()).toEqual(["GET archives/key 200"]);
      }
      // The policy probe reached the main thread's request hook with no bearer refresh.
      expect(refreshFlags[0]).toBe(false);
      await capture.stop();
    }
  }, 60_000);

  test("on the worker, the paths the session uploader sees a session touch reach the touched-files archive: a folder without .git gets only those files", async () => {
    const sessionId = "ses_touched_0001";
    const engine = Bun.serve({
      hostname: "127.0.0.1",
      port: 0,
      fetch: (request) => (new URL(request.url).pathname.endsWith("/message") ? Response.json([]) : Response.json({ id: sessionId })),
    });
    cleanups.push(() => engine.stop(true));
    const home = await tempDir("home");
    const root = join(home, "report");
    await mkdir(root);
    const brief = randomBytes(64 * 1024);
    await writeFile(join(root, "brief.pdf"), brief);
    await writeFile(join(root, "untouched.txt"), "never touched\n");
    const archive = new FakeArchiveServer();
    archive.policy = { touched_files: true };
    const stateDir = await tempDir("state");
    const capture = service({
      stateDir,
      sessionUploader: { upload: uploadSink().upload },
      archive: {
        enabled: true,
        excludedDirs: [],
        folderGate: { homeDir: home },
        request: (path, init) => archive.respond(`https://api.omnirush.test/omnirush/${path}`, {
          method: init.method,
          headers: { authorization: `Bearer ${archive.token}` },
          ...(init.body === undefined ? {} : { body: init.body }),
        }),
        refreshAccessToken: async () => null,
        fetch: archive.respond,
        baseIdleMs: 0,
      },
    });
    capture.startSession(sessionId, "workspace-touched", root);
    capture.archiveSessionStarted(sessionId, root, { baseUrl: `http://127.0.0.1:${engine.port}`, headers: [], search: "", engine: "v1" });
    await until(() => archive.calls.some((call) => call.path === "archives/key"), 20_000, "the key read");
    await capture.idle();
    expect(capture.mode()).toBe("worker");
    // Registered: nothing packed before a capture with a touched file.
    expect(archive.callPaths()).toEqual(["GET archives/key 200"]);
    // The agent reads the PDF; the app quits: the quit's final archive is the chain's base.
    capture.recordTrace(sessionId, "tool.read", { input: { filePath: join(root, "brief.pdf") } });
    await capture.idle();
    await capture.stop();
    const archiveDir = join(stateDir, "omnirush-archive");
    const queue = await readdir(join(archiveDir, "queue"));
    expect(queue).toHaveLength(1);
    const record = JSON.parse(await readFile(join(archiveDir, "queue", queue[0]!), "utf8"));
    expect(record.request).toMatchObject({ session_id: sessionId, kind: "base", sequence: 0, turn: 0, marker: "touched" });
    const members = await openArchive(await readFile(join(archiveDir, "pending", record.sealed_file)));
    expect(members.map((member) => member.name)).toEqual(["__omnirush__/manifest.json", "brief.pdf"]);
    expect(members[1]!.content.equals(brief)).toBe(true);
  }, 60_000);

  test("through the worker, uploads carry their Idempotency-Key, the integrity read reaches the main thread and the spool is stamped with the account", async () => {
    const stateDir = await tempDir("keys-state");
    const root = await tempDir("keys-root");
    await writeFile(join(root, "app.txt"), "hello\n");
    const keys: Array<string | undefined> = [];
    const integrity: string[] = [];
    const capture = service({
      stateDir,
      accountId: async () => "user-worker-1",
      sessionIntegrity: async (sessionId, options) => {
        integrity.push(`${sessionId}:${options?.summary === true}`);
        return Response.json({ detail: "session_not_found" }, { status: 404 });
      },
      sessionUploader: {
        capabilities: async () => ({ schema_versions: [1, 2], canonical_trace: false, idempotency_key: true, integrity: true, integrity_summary: true }),
        upload: async (_sessionId, _bytes, _signal, request) => {
          keys.push(request?.idempotencyKey);
          return Response.json({ detail: "unavailable" }, { status: 503 });
        },
      },
    });
    capture.startSession("session-worker-keys", "workspace-keys", root);
    await capture.idle();
    await capture.stop();
    expect(integrity).toEqual(["session-worker-keys:true"]);
    expect(keys.length).toBeGreaterThan(0);
    expect(keys.every((key) => typeof key === "string" && /^[0-9a-f]{64}$/.test(key))).toBe(true);
    const spool = join(stateDir, "omnirush-upload-spool");
    const metas = await Promise.all((await readdir(spool)).filter((name) => name.endsWith(".json")).map(async (name) => JSON.parse(await readFile(join(spool, name), "utf8")) as Record<string, unknown>));
    expect(metas.length).toBeGreaterThan(0);
    expect(metas.every((meta) => meta.account_id === "user-worker-1" && typeof meta.idempotency_key === "string")).toBe(true);
  });

  test("OMNIRUSH_CAPTURE_WORKER=0 captures in-process with the same envelopes", async () => {
    const root = await syntheticWorkspace(20);
    const results: Array<Array<[string, string, number]>> = [];
    for (const worker of [true, false]) {
      const sink = uploadSink();
      const capture = service({ stateDir: await tempDir("state"), sessionUploader: { upload: sink.upload }, worker });
      expect(capture.mode()).toBe(worker ? "starting" : "local");
      capture.startSession("session-modes-0001", "workspace-modes", root);
      expect(capture.hasSession("session-modes-0001")).toBe(true);
      expect(capture.recordWebVisit("session-modes-0001", { url: "https://example.com/docs", title: "Docs", text: "mail [email protected]" })).toBe(true);
      expect(capture.recordWebVisit("session-modes-0001", { url: "http://localhost:3000/" })).toBe(false);
      await capture.stop();
      expect(capture.mode()).toBe("down");
      results.push(sink.envelopes().map((item) => [item.snapshot_type, item.files_scope, item.files.length]));
    }
    expect(results[0]).toEqual([["start", "full", 21], ["trace", "full", 1], ["end", "changed", 1]]);
    expect(results[1]).toEqual(results[0]!);
  }, 60_000);
});

describe("capture worker restarts", () => {
  // The worker's test-only crash hook (capture-worker.ts) answers only with this set.
  let crashSwitch: string | undefined;
  beforeAll(() => {
    crashSwitch = process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
    process.env.OMNIRUSH_CAPTURE_TEST_CRASH = "1";
  });
  afterAll(() => {
    if (crashSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
    else process.env.OMNIRUSH_CAPTURE_TEST_CRASH = crashSwitch;
  });

  function crashing(options: Partial<CaptureServiceOptions> = {}) {
    const restarts: Array<Record<string, unknown>> = [];
    const modes: string[] = [];
    const capture = service({
      stateDir: "",
      sessionUploader: {},
      log: (_level, message, attributes) => {
        if (message.includes("starting a new") && attributes) restarts.push(attributes);
      },
      onModeChange: (mode) => modes.push(mode),
      ...options,
    });
    const crash = async () => {
      await until(() => capture.mode() === "worker", 10_000, "a ready worker");
      capture.recordTrace("session-crash-0001", "omnirush.test.worker_crash");
      await until(() => capture.mode() === "restarting", 10_000, "the worker's exit");
    };
    return { capture, restarts, modes, crash };
  }

  test("a worker that keeps exiting is replaced every time, after a pause that doubles up to its cap, and is \"down\" only once stopped", async () => {
    const { capture, restarts, modes, crash } = crashing({ stateDir: await tempDir("state"), restartBackoffMs: { baseMs: 40, maxMs: 160 } });
    for (let index = 0; index < 5; index += 1) await crash();
    expect(restarts.map((item) => [item.code, item.attempt, item.delayMs])).toEqual([[70, 1, 40], [70, 2, 80], [70, 3, 160], [70, 4, 160], [70, 5, 160]]);
    expect(capture.status()).toMatchObject({ mode: "restarting", restarts: 5 });
    await until(() => capture.mode() === "worker", 10_000, "the sixth worker");
    expect(await capture.diagnostics()).not.toBeNull();
    expect(modes).toEqual([...Array.from({ length: 5 }, () => ["worker", "restarting", "starting"]).flat(), "worker"]);
    await capture.stop();
    expect(capture.mode()).toBe("down");
    expect(modes.at(-1)).toBe("down");
  }, 60_000);

  test("a worker that stayed up long enough starts the backoff over", async () => {
    const { capture, restarts, crash } = crashing({ stateDir: await tempDir("state"), restartBackoffMs: { baseMs: 40, maxMs: 1_000, stableMs: 300 } });
    await crash();
    await crash();
    await until(() => capture.mode() === "worker", 10_000, "the third worker");
    await new Promise((resolvePromise) => setTimeout(resolvePromise, 400));
    await crash();
    expect(restarts.map((item) => [item.attempt, item.delayMs])).toEqual([[1, 40], [2, 80], [1, 40]]);
    expect(capture.status().restarts).toBe(3);
  }, 30_000);

  test("calls made while restarting reach the next worker", async () => {
    const root = await syntheticWorkspace(5);
    const sink = uploadSink();
    const { capture, crash } = crashing({ stateDir: await tempDir("state"), sessionUploader: { upload: sink.upload }, restartBackoffMs: { baseMs: 400, maxMs: 400 } });
    await crash();
    capture.startSession("session-restart-0001", "workspace-restart", root);
    const diagnostics = capture.diagnostics();
    expect(capture.mode()).toBe("restarting");
    expect(await diagnostics).not.toBeNull();
    expect(capture.mode()).toBe("worker");
    await capture.idle();
    expect(sink.envelopes().map((item) => [item.session_id, item.snapshot_type])).toEqual([["session-restart-0001", "start"]]);
  }, 30_000);
});

describe("turns left open: worker restarts and quits", () => {
  let crashSwitch: string | undefined;
  let contextSwitch: string | undefined;
  beforeAll(() => {
    crashSwitch = process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
    contextSwitch = process.env.OMNIRUSH_CAPTURE_CONTEXT;
    process.env.OMNIRUSH_CAPTURE_TEST_CRASH = "1";
    process.env.OMNIRUSH_CAPTURE_CONTEXT = "0";
  });
  afterAll(() => {
    if (crashSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_TEST_CRASH;
    else process.env.OMNIRUSH_CAPTURE_TEST_CRASH = crashSwitch;
    if (contextSwitch === undefined) delete process.env.OMNIRUSH_CAPTURE_CONTEXT;
    else process.env.OMNIRUSH_CAPTURE_CONTEXT = contextSwitch;
  });

  const SESSION = "ses_recovered_0001";

  /** A fake engine with one chat: its status and its messages. */
  function fakeEngine() {
    const control = { busy: true, messages: [] as Array<{ info: Record<string, unknown>; parts: unknown[] }> };
    const message = (id: string, role: "user" | "assistant") => ({
      info: { id, sessionID: SESSION, role, time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 } },
      parts: [{ id: `${id}_p`, messageID: id, sessionID: SESSION, type: "text", text: `${role} ${id}` }],
    });
    const server = Bun.serve({
      hostname: "127.0.0.1",
      port: 0,
      fetch(request) {
        const url = new URL(request.url);
        if (url.pathname === "/session/status") return Response.json(control.busy ? { [SESSION]: { type: "busy" } } : {});
        if (url.pathname === `/session/${SESSION}/message`) return Response.json(control.messages);
        if (url.pathname === `/session/${SESSION}/children`) return Response.json([]);
        if (url.pathname === `/session/${SESSION}`) return Response.json({ id: SESSION });
        return Response.json({ code: "not_found" }, { status: 404 });
      },
    });
    cleanups.push(() => server.stop(true));
    const target = { baseUrl: `http://127.0.0.1:${server.port}`, headers: [] as Array<[string, string]>, search: "", engine: "v1" as const };
    return { control, message, target };
  }

  test("a worker killed mid-turn: the next worker uploads the turn's journaled events, marked recovered, and settles the turn from the engine", async () => {
    const root = await tempDir("recover-root");
    await writeFile(join(root, "app.txt"), "hello\n");
    const sink = uploadSink();
    const engine = fakeEngine();
    engine.control.messages = [engine.message("msg_1", "user")];
    const capture = service({
      stateDir: await tempDir("recover-state"),
      sessionUploader: { upload: sink.upload },
      restartBackoffMs: { baseMs: 50, maxMs: 50 },
    });
    capture.startSession(SESSION, "workspace-recover", root);
    capture.captureSnapshot(SESSION, "prompt");
    capture.recordTrace(SESSION, "engine.request", { body: "do the task" });
    capture.recordTrace(SESSION, "tool.call", { name: "edit", marker: "before-the-crash" });
    capture.observeSession(SESSION, engine.target);
    // The journal is on disk within its flush interval; then the worker dies.
    await new Promise((resolvePromise) => setTimeout(resolvePromise, 800));
    capture.recordTrace(SESSION, "omnirush.test.worker_crash");
    await until(() => capture.mode() === "restarting", 10_000, "the worker's exit");
    // The turn finishes while the new worker comes up.
    engine.control.messages.push(engine.message("msg_2", "assistant"));
    engine.control.busy = false;
    await until(() => sink.envelopes().some((item) => JSON.stringify(item).includes("\"recovered\":true")), 30_000, "the settled recovered turn");
    const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;
    const recovered = envelopes.find((item) => item.trace?.[0]?.type === "collector.recovered");
    expect(recovered).toBeDefined();
    expect(JSON.stringify(recovered)).toContain("before-the-crash");
    const settled = envelopes.find((item) => item.trace?.some((event) => event.type === "turn.completed"));
    const completed = settled?.trace?.find((event) => event.type === "turn.completed");
    expect(completed?.data).toMatchObject({ recovered: true });
    expect(JSON.stringify(completed)).toContain("msg_2");
  }, 60_000);

  test("a recovered turn whose chat is resumed first (the app's task recovery) is marked recovered on that chat", async () => {
    const root = await tempDir("resumed-root");
    await writeFile(join(root, "app.txt"), "hello\n");
    const stateDir = await tempDir("resumed-state");
    const journal = join(stateDir, "omnirush-upload-journal");
    await mkdir(journal, { recursive: true });
    const meta = { v: 1, session_id: SESSION, workspace_id: "workspace-resumed", root, segment: 1, turn_open: true, at: new Date().toISOString() };
    await writeFile(join(journal, "left.jsonl"), `${JSON.stringify({ m: meta })}\n${JSON.stringify({ e: { at: meta.at, type: "tool.call", data: { n: 1 } } })}\n`);
    const sink = uploadSink();
    let lookups = 0;
    const capture = service({
      stateDir,
      worker: false,
      sessionUploader: { upload: sink.upload },
      // No engine yet at the first look; the chat is resumed before the next one.
      engineTarget: async () => {
        lookups += 1;
        if (lookups === 1) capture.startSession(SESSION, "workspace-resumed", root);
        return null;
      },
    });
    await until(() => lookups >= 2 || sink.envelopes().length >= 2, 20_000, "the second look");
    capture.flushTrace(SESSION);
    await capture.stop();
    const events = (sink.envelopes() as unknown as Array<{ trace?: Array<{ type: string; data?: Record<string, unknown> }> }>).flatMap((item) => item.trace ?? []);
    expect(events.some((event) => event.type === "collector.recovered" && event.data?.settled_by === "resumed_chat")).toBe(true);
  }, 40_000);

  test("an app quit settles a turn still running before the chat ends", async () => {
    const root = await tempDir("quit-root");
    await writeFile(join(root, "app.txt"), "hello\n");
    const sink = uploadSink();
    const engine = fakeEngine();
    engine.control.messages = [engine.message("msg_1", "user"), engine.message("msg_2", "assistant")];
    const capture = service({ stateDir: await tempDir("quit-state"), sessionUploader: { upload: sink.upload } });
    capture.startSession(SESSION, "workspace-quit", root);
    capture.captureSnapshot(SESSION, "prompt");
    capture.observeSession(SESSION, engine.target);
    await new Promise((resolvePromise) => setTimeout(resolvePromise, 500));
    await capture.stop();
    const envelopes = sink.envelopes() as unknown as Array<{ snapshot_type: string; trace?: Array<{ type: string; data?: unknown }> }>;
    const completed = envelopes.flatMap((item) => item.trace ?? []).find((event) => event.type === "turn.completed");
    expect(completed?.data).toMatchObject({ reason: "app_quit" });
    expect(JSON.stringify(completed)).toContain("msg_2");
    expect(envelopes.at(-1)?.snapshot_type).toBe("end");
  }, 60_000);
});