Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

59.1 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import {
  RECORDER_SIGNED_OUT,
  type TrajectoryGrant,
} from "@openbot/shared/recorder-api";
import { RECORDER_EXTENSION_ID } from "@openbot/shared/recorder-extension";
import type {
  CapturedEvent,
  JsonValue,
} from "@openbot/shared/recorder-masking";
import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb";
import {
  classifyGrantError,
  GrantError,
  type GrantVerdict,
  PASS_HINTS,
} from "../src/offscreen/grant";
import {
  type Outbox,
  openOutbox,
  TRAJECTORY_META_PREFIX,
  type TrajectoryMeta,
} from "../src/offscreen/outbox";
import {
  type ChannelLike,
  createPhoenixSession,
  type PhoenixLike,
  type PushLike,
} from "../src/offscreen/phoenix-session";
import {
  createStreamer,
  type KeptReport,
  type Streamer,
  type StreamerDeps,
  type StreamerSession,
  type StreamerStatus,
} from "../src/offscreen/streamer";
import { installChromeFake } from "./support/chrome-fake";
import { type FakeGateway, startFakeGateway } from "./support/fake-gateway";

// The table's sentences, pinned here as the person reads them.
const BUSY = "Intelligence is busy or unreachable. Retrying; nothing is lost.";
const ENTITLEMENT =
  "Intelligence could not confirm this organization's licence (entitlement_unavailable). Retrying; nothing is lost.";
const MARKETPLACE =
  "Intelligence needs its marketplace licence before it can capture. Retrying; nothing is lost.";
const GATEWAY_OFF =
  "Learning capture is off: the Intelligence gateway has trajectory capture turned off.";
const PASS_REFUSED =
  "Learning capture is off: Intelligence refused the capture pass.";
const NOT_ENABLED =
  "Learning capture is paused: Intelligence says this organization's licence does not include Product Trajectories, or that its billing is blocked.";
const MESSAGES_REFUSED =
  "Learning capture is off: Intelligence refused the recorder's messages.";
const checkOrigin = (id: string) =>
  `Intelligence refused the connection. If it sets CHECK_ORIGIN, that list must include chrome-extension://${id}.`;
/** An id Edge Add-ons could assign the Recorder: published there, it runs under that one, not the fixed one. */
const EDGE_ADDONS_ID = "jdnkfgmofbmnjdbcpjgbgmmlhoiopnkp";

const MINUTE = 60_000;

type IndexedDbGlobals = {
  indexedDB?: IDBFactory;
  IDBKeyRange?: typeof FakeIDBKeyRange;
};
const globals = globalThis as unknown as IndexedDbGlobals;
let savedIndexedDb: IndexedDbGlobals = {};

const sleep = (ms: number) =>
  new Promise<void>((resolve) => setTimeout(resolve, ms));

async function until(
  condition: () => boolean | Promise<boolean>,
  what: string,
  timeoutMs = 5_000,
): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  while (!(await condition())) {
    if (Date.now() > deadline) throw new Error(`Timed out waiting for ${what}`);
    await sleep(5);
  }
}

/** Fake timers and a fake clock for the streamer; the network and IndexedDB stay real, so each step lets them run. */
function virtualClock() {
  let time = 1_800_000_000_000;
  let nextHandle = 1;
  const pending = new Map<number, { fn: () => void; at: number }>();
  return {
    now: () => time,
    timers: {
      set(fn: () => void, ms: number) {
        const handle = nextHandle++;
        pending.set(handle, { fn, at: time + Math.max(0, ms) });
        return handle;
      },
      clear(handle: unknown) {
        pending.delete(handle as number);
      },
    },
    /** Pending timers, as delays from now. */
    delays: () => [...pending.values()].map((timer) => timer.at - time),
    async advance(ms: number) {
      const target = time + ms;
      for (;;) {
        await sleep(2);
        let due: [number, { fn: () => void; at: number }] | undefined;
        for (const entry of pending) {
          if (entry[1].at <= target && (!due || entry[1].at < due[1].at))
            due = entry;
        }
        if (!due) break;
        pending.delete(due[0]);
        time = Math.max(time, due[1].at);
        due[1].fn();
      }
      time = target;
      await sleep(2);
    },
  };
}
type Clock = ReturnType<typeof virtualClock>;

/** A PhoenixLike whose join answers as scripted, for what the fake gateway cannot do (a join that never answers). */
class ScriptedPhoenix implements PhoenixLike {
  connectCalls = 0;
  disconnected = false;
  left = false;
  offSocketRefs: string[] = [];
  offChannel: [string, number][] = [];
  private socketHandlers = new Map<
    string,
    { kind: "open" | "error" | "close"; callback: (value?: unknown) => void }
  >();
  private channelHandlers = new Map<
    number,
    { event: string; callback: (payload: unknown) => void }
  >();
  private nextRef = 0;

  constructor(readonly join: "silent" | "ok") {}

  connect = () => {
    this.connectCalls += 1;
    for (const handler of this.socketHandlers.values())
      if (handler.kind === "open") handler.callback();
  };
  disconnect() {
    this.disconnected = true;
  }
  channel(_topic: string): ChannelLike {
    const silent: PushLike = { receive: () => silent };
    return {
      join: () => {
        const hooks: { status: string; callback: (r?: unknown) => void }[] = [];
        const push: PushLike = {
          receive: (status, callback) => {
            hooks.push({ status, callback });
            return push;
          },
        };
        if (this.join === "ok") {
          queueMicrotask(() => {
            for (const hook of hooks)
              if (hook.status === "ok") hook.callback({});
          });
        }
        return push;
      },
      push: () => silent,
      leave: () => {
        this.left = true;
      },
      on: (event, callback) => this.bind(event, callback),
      off: (event, ref) => {
        this.offChannel.push([event, ref]);
        this.channelHandlers.delete(ref);
      },
      onError: (callback) => this.bind("phx_error", callback),
      onClose: (callback) => this.bind("phx_close", callback),
    };
  }
  onOpen(callback: () => void) {
    return this.bindSocket("open", callback);
  }
  onError(callback: (error: unknown) => void) {
    return this.bindSocket("error", callback);
  }
  onClose(callback: (event: { code?: number } | undefined) => void) {
    return this.bindSocket("close", (event) =>
      callback(event as { code?: number } | undefined),
    );
  }
  off(refs: string[]) {
    this.offSocketRefs.push(...refs);
    for (const ref of refs) this.socketHandlers.delete(ref);
  }
  /** What phoenix.js does on a transport error: the socket's error callbacks, then a transport error on the channel. */
  failTransport() {
    for (const handler of [...this.socketHandlers.values()])
      if (handler.kind === "error") handler.callback(new Error("gone"));
    this.fireChannel("phx_error", {
      source: "transport",
      reason: "connection_error",
    });
    this.fireChannel("phx_error", { reason: "channel crashed" });
  }
  fireChannel(event: string, payload: unknown) {
    for (const handler of [...this.channelHandlers.values()])
      if (handler.event === event) handler.callback(payload);
  }
  get boundChannelHandlers() {
    return this.channelHandlers.size;
  }
  private bind(event: string, callback: (payload: unknown) => void) {
    const ref = this.nextRef++;
    this.channelHandlers.set(ref, { event, callback });
    return ref;
  }
  private bindSocket(
    kind: "open" | "error" | "close",
    callback: (value?: unknown) => void,
  ) {
    const ref = `${kind}-${this.nextRef++}`;
    this.socketHandlers.set(ref, { kind, callback });
    return ref;
  }
}

type Harness = {
  gateway: FakeGateway;
  clock: Clock;
  outbox: Outbox;
  dbName: string;
  grants: string[];
  tokens: string[];
  passScript: GrantVerdict[];
  gate: { promise: Promise<void>; release(): void } | null;
  beforeSocket: (() => void) | null;
  scripted: ScriptedPhoenix[];
  factoryCalls: number;
  sockets: WebSocket[];
  socketUrls: string[];
  statuses: { at: number; status: StreamerStatus }[];
  kept: KeptReport[];
  signedOut: string[];
  onTrajectory: string[];
  /** Each rollover with the recording it belongs to. */
  rollovers: { trajectoryId: string; recordingId: string }[];
  /** Trajectories in the order the streamer finished them (their meta saved with `closed`). */
  closed: string[];
  last(): StreamerStatus;
  stored(trajectoryId: string): { seq: number; name: string; raw: unknown }[];
  pending(trajectoryId: string): Promise<number>;
  deps(overrides?: Partial<StreamerDeps>): StreamerDeps;
  start(
    session?: Partial<StreamerSession>,
    overrides?: Partial<StreamerDeps>,
  ): Promise<Streamer>;
  trajectoryId: string;
};

/** The outbox, recording each trajectory the streamer finishes: its meta is saved with `closed` set. */
function watchClosing(outbox: Outbox, closed: string[]): Outbox {
  return {
    ...outbox,
    setMeta: async <T>(key: string, value: T) => {
      await outbox.setMeta(key, value);
      const id = key.slice(TRAJECTORY_META_PREFIX.length);
      if (
        key.startsWith(TRAJECTORY_META_PREFIX) &&
        (value as TrajectoryMeta & { closed?: boolean }).closed &&
        !closed.includes(id)
      )
        closed.push(id);
    },
  };
}

let harness: Harness | null = null;
const streamers: Streamer[] = [];
let dbCounter = 0;

function makeGate() {
  let release = () => {};
  const promise = new Promise<void>((resolve) => {
    release = resolve;
  });
  return { promise, release };
}

async function setup(options: { eventCap?: number } = {}): Promise<Harness> {
  const gateway = await startFakeGateway(options);
  const clock = virtualClock();
  const dbName = `streamer-test-${Date.now()}-${dbCounter++}`;
  const outbox = await openOutbox(dbName);
  let seq = 1;
  const sequencer = { next: () => seq++, flush: async () => {} };
  const h: Harness = {
    gateway,
    clock,
    outbox,
    dbName,
    grants: [],
    tokens: [],
    passScript: [],
    gate: null,
    beforeSocket: null,
    scripted: [],
    factoryCalls: 0,
    sockets: [],
    socketUrls: [],
    statuses: [],
    kept: [],
    signedOut: [],
    onTrajectory: [],
    rollovers: [],
    closed: [],
    trajectoryId: crypto.randomUUID(),
    last: () => {
      const entry = h.statuses.at(-1);
      if (!entry) throw new Error("No status yet");
      return entry.status;
    },
    stored: (trajectoryId) => gateway.events(trajectoryId),
    pending: (trajectoryId) => h.outbox.count(trajectoryId),
    deps(overrides = {}) {
      class CountingWebSocket extends WebSocket {
        constructor(url: string | URL, protocols?: string | string[]) {
          super(url, protocols);
          h.sockets.push(this);
          h.socketUrls.push(String(url));
        }
      }
      return {
        async fetchGrant(trajectoryId: string): Promise<TrajectoryGrant> {
          h.grants.push(trajectoryId);
          if (h.gate) await h.gate.promise;
          const scripted = h.passScript.shift();
          if (scripted) throw new GrantError(scripted);
          h.beforeSocket?.();
          const joinToken = gateway.mintToken(trajectoryId);
          h.tokens.push(joinToken);
          return {
            joinToken,
            realtime: {
              clientUrl: `${gateway.url}/client`,
              topic: `trajectory:${trajectoryId}`,
            },
          };
        },
        allowedGatewayUrls: null,
        outbox: watchClosing(h.outbox, h.closed),
        sequencer,
        status: (status: StreamerStatus) =>
          h.statuses.push({ at: clock.now(), status }),
        onTrajectory: (trajectoryId: string, recordingId: string) => {
          h.onTrajectory.push(trajectoryId);
          h.rollovers.push({ trajectoryId, recordingId });
        },
        onSignedOut: (reason: string) => h.signedOut.push(reason),
        onKept: (kept: KeptReport) => h.kept.push(kept),
        randomUUID: () => crypto.randomUUID(),
        socketFactory: (url: string, params: Record<string, string>) => {
          h.factoryCalls += 1;
          return (
            h.scripted.shift() ??
            createPhoenixSession(url, params, { transport: CountingWebSocket })
          );
        },
        now: clock.now,
        timers: clock.timers,
        ...overrides,
      };
    },
    async start(session = {}, overrides = {}) {
      const streamer = await createStreamer(
        {
          trajectoryId: h.trajectoryId,
          title: "File the weekly report",
          recordingId: "rec-1",
          version: "0.1.0",
          ...session,
        },
        h.deps(overrides),
      );
      streamers.push(streamer);
      return streamer;
    },
  };
  harness = h;
  return h;
}

const click = (
  i: number,
  extra: Record<string, JsonValue> = {},
): CapturedEvent => ({
  type: "CUSTOM",
  name: "click",
  timestamp: 1_700_000_000_000 + i,
  value: { i, ...extra },
});

/** An event no trimming can bring under Intelligence's 16 KiB: its URL alone is larger. */
const oversize = (): CapturedEvent => ({
  type: "CUSTOM",
  name: "page",
  timestamp: 1_700_000_000_000,
  value: { url: `https://example.com/${"a".repeat(20_000)}` },
});

const payloadOf = (entry: { raw: unknown }) =>
  (entry.raw as { value: Record<string, JsonValue> }).value;

const clicks = (entries: { name: string; raw: unknown }[]) =>
  entries
    .filter((entry) => entry.name === "click")
    .map((entry) => payloadOf(entry).i);

const range = (count: number, from = 0) =>
  Array.from({ length: count }, (_, index) => from + index);

const notEnabledPass = () =>
  classifyGrantError(403, {
    code: "TRAJECTORIES_NOT_ENABLED",
    message: "Trajectories are not enabled.",
  });
const busyPass = () =>
  classifyGrantError(502, {
    code: "CONNECTION_FAILED",
    message: "Could not connect to Intelligence",
  });

async function streaming(h: Harness) {
  await until(() => h.last().intelligence === "streaming", "streaming");
}

async function started(h: Harness, trajectoryId = h.trajectoryId) {
  await until(() => h.stored(trajectoryId).length >= 2, "the first events");
  await until(
    async () => (await h.pending(trajectoryId)) === 0,
    "an empty outbox",
  );
  await streaming(h);
}

beforeEach(() => {
  savedIndexedDb = {
    indexedDB: globals.indexedDB,
    IDBKeyRange: globals.IDBKeyRange,
  };
  globals.indexedDB = new IDBFactory();
  globals.IDBKeyRange = FakeIDBKeyRange;
});

afterEach(async () => {
  for (const streamer of streamers.splice(0)) await streamer.discard();
  if (harness) await harness.gateway.stop();
  harness = null;
  for (const key of ["indexedDB", "IDBKeyRange"] as const) {
    if (savedIndexedDb[key] === undefined) Reflect.deleteProperty(globals, key);
    else Object.assign(globals, { [key]: savedIndexedDb[key] });
  }
});

describe("streamer", () => {
  test("120 events arrive once, in seq order, in batches within Intelligence's limits", async () => {
    const h = await setup();
    h.gate = makeGate();
    const s = await h.start();
    for (let i = 0; i < 60; i++) s.push(click(i));
    for (let i = 60; i < 120; i++)
      s.push(click(i, { text: "x".repeat(3_000) }));
    await until(
      async () => (await h.pending(h.trajectoryId)) === 122,
      "all queued",
    );
    h.gate.release();
    await until(() => h.stored(h.trajectoryId).length === 122, "all stored");
    await until(
      async () => (await h.pending(h.trajectoryId)) === 0,
      "an empty outbox",
    );

    const stored = h.stored(h.trajectoryId);
    const seqs = stored.map((entry) => entry.seq);
    expect(seqs).toEqual([...seqs].sort((a, b) => a - b));
    expect(new Set(seqs).size).toBe(122);
    expect(stored.slice(0, 2).map((entry) => entry.name)).toEqual([
      "recording.started",
      "outcome",
    ]);
    expect(payloadOf(stored[0])).toMatchObject({
      recordingId: "rec-1",
      source: "openbot-recorder",
      version: "0.1.0",
    });
    expect(payloadOf(stored[1])).toMatchObject({
      workflow: "File the weekly report",
      phase: "started",
    });
    expect(clicks(stored)).toEqual(range(120));
    for (const batch of h.gateway.batches) {
      expect(batch.count).toBeLessThanOrEqual(50);
      expect(batch.bytes).toBeLessThanOrEqual(65_536);
    }
    expect(h.gateway.batches.some((batch) => batch.count === 50)).toBe(true);
    expect(
      Math.max(...h.gateway.batches.map((batch) => batch.bytes)),
    ).toBeGreaterThan(60_000);
    expect(h.last().events).toEqual({
      sent: 122,
      pending: 0,
      rejected: 0,
      dropped: 0,
    });
  });

  test("a dropped socket reconnects with a fresh pass and loses nothing", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    for (let i = 0; i < 10; i++) s.push(click(i));
    await until(() => h.stored(h.trajectoryId).length === 12, "the first ten");
    h.gateway.dropConnections();
    for (let i = 10; i < 20; i++) s.push(click(i));
    await until(() => h.stored(h.trajectoryId).length === 22, "the second ten");
    expect(h.grants).toHaveLength(2);
    expect(new Set(h.tokens).size).toBe(2);
    expect(clicks(h.stored(h.trajectoryId))).toEqual(range(20));
  });

  test("an unacknowledged batch is resent and stored once", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    h.gateway.delayNextAck(15_000);
    for (let i = 0; i < 3; i++) s.push(click(i));
    await until(
      () => h.stored(h.trajectoryId).length >= 3,
      "the unanswered batch stored",
    );
    const batches = h.gateway.batches.length;
    await h.clock.advance(10_000);
    await until(
      async () => (await h.pending(h.trajectoryId)) === 0,
      "the resend acknowledged",
    );
    expect(h.grants).toHaveLength(2);
    expect(h.gateway.batches.length).toBeGreaterThan(batches);
    expect(h.stored(h.trajectoryId)).toHaveLength(5);
    expect(clicks(h.stored(h.trajectoryId))).toEqual(range(3));
  });

  test("a resent batch carries dropped 0, so a drop is reported once", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    s.push(oversize());
    h.gateway.delayNextAck(15_000);
    s.push(click(0));
    await until(
      () => h.stored(h.trajectoryId).length === 3,
      "the unanswered batch stored",
    );
    expect(h.gateway.batches.at(-1)?.dropped).toBe(1);
    await h.clock.advance(10_000);
    await until(
      async () => (await h.pending(h.trajectoryId)) === 0,
      "the resend acknowledged",
    );
    expect(h.gateway.batches.at(-1)?.dropped).toBe(0);
    expect(h.gateway.droppedReported(h.trajectoryId)).toBe(1);
    expect(h.last().events.dropped).toBe(1);
  });

  test("a refusal Intelligence also answers for a failed entitlement lookup keeps every event and retries", async () => {
    const h = await setup();
    h.passScript = [notEnabledPass(), notEnabledPass()];
    const s = await h.start();
    for (let i = 0; i < 38; i++) s.push(click(i));
    await until(
      () => h.statuses.some((entry) => entry.status.intelligence === "held"),
      "held",
    );
    expect(h.last()).toMatchObject({
      intelligence: "held",
      reason: PASS_HINTS.TRAJECTORIES_NOT_ENABLED,
    });
    await until(
      async () => (await h.pending(h.trajectoryId)) === 40,
      "40 kept",
    );
    s.push(click(38));
    s.push(click(39));
    await until(
      async () => (await h.pending(h.trajectoryId)) === 42,
      "new events still queue",
    );
    expect(h.stored(h.trajectoryId)).toHaveLength(0);

    await h.clock.advance(5 * MINUTE);
    await until(() => h.grants.length === 2, "the second attempt");
    expect(h.last().intelligence).toBe("held");
    await h.clock.advance(5 * MINUTE);
    await until(
      () => h.stored(h.trajectoryId).length === 42,
      "everything sent",
    );
    expect(clicks(h.stored(h.trajectoryId))).toEqual(range(40));
    expect(h.kept).toHaveLength(1);
    expect(h.kept[0]).toMatchObject({
      parked: false,
      reason: PASS_HINTS.TRAJECTORIES_NOT_ENABLED,
      recordingId: "rec-1",
      trajectoryId: h.trajectoryId,
    });

    await s.stop("succeeded");
    await s.drained();
    expect(h.closed).toEqual([h.trajectoryId]);
    expect(h.stored(h.trajectoryId)).toHaveLength(43);
  });

  test("a definitive refusal parks: events kept, no automatic retry, and the channel never rejoins on its own", async () => {
    const h = await setup();
    h.gateway.refuseNextJoin("trajectories_unavailable");
    const s = await h.start();
    for (let i = 0; i < 8; i++) s.push(click(i));
    await until(
      () => h.statuses.some((entry) => entry.status.intelligence === "parked"),
      "parked",
    );
    expect(h.last()).toMatchObject({
      intelligence: "parked",
      reason: GATEWAY_OFF,
    });
    await until(
      async () => (await h.pending(h.trajectoryId)) === 10,
      "everything kept",
    );
    const joins = h.gateway.joins.length;

    await h.clock.advance(30 * MINUTE);
    await sleep(1_300); // phoenix.js would rejoin an errored channel after 1 s of real time
    expect(await h.pending(h.trajectoryId)).toBe(10);
    expect(h.grants).toHaveLength(1);
    expect(h.gateway.joins).toHaveLength(joins);
    expect(h.kept).toEqual([
      {
        reason: GATEWAY_OFF,
        parked: true,
        events: expect.any(Number),
        recordingId: "rec-1",
        trajectoryId: h.trajectoryId,
      },
    ]);

    // The next browser start: a resumed streamer over the same database sends them all.
    h.outbox = await openOutbox(h.dbName);
    await h.start({ resume: true });
    await until(
      () => h.stored(h.trajectoryId).length === 10,
      "everything sent",
    );
    expect(clicks(h.stored(h.trajectoryId))).toEqual(range(8));
  });

  describe("trajectories_not_enabled from the gateway holds, because its answer can lag", () => {
    test("on the join (row 29)", async () => {
      const h = await setup();
      h.gateway.refuseNextJoin("trajectories_not_enabled");
      const s = await h.start();
      s.push(click(0));
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(NOT_ENABLED);
      await until(async () => (await h.pending(h.trajectoryId)) === 3, "kept");
      await h.clock.advance(5 * MINUTE);
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent after the hold",
      );
      expect(h.kept).toHaveLength(1);
    });

    test("on a push, and the channel close that follows is not counted again (row 35)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.replyNextPush({ reason: "trajectories_not_enabled" });
      s.push(click(0));
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(NOT_ENABLED);
      await sleep(50);
      expect(h.grants).toHaveLength(1);
      expect(h.kept).toHaveLength(1);
      s.push(click(1));
      await h.clock.advance(5 * MINUTE);
      await until(
        () => h.stored(h.trajectoryId).length === 4,
        "sent after the hold",
      );
      expect(clicks(h.stored(h.trajectoryId))).toEqual([0, 1]);
    });

    test("on an expired channel (row 43)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.expire(h.trajectoryId, "trajectories_not_enabled");
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(NOT_ENABLED);
      s.push(click(0));
      await until(async () => (await h.pending(h.trajectoryId)) === 1, "kept");
      await h.clock.advance(5 * MINUTE);
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent after the hold",
      );
    });
  });

  test("a signed-out pass keeps every event for its person", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    for (let i = 0; i < 3; i++) s.push(click(i));
    await until(() => h.stored(h.trajectoryId).length === 5, "the first three");
    h.passScript = [
      classifyGrantError(401, { code: RECORDER_SIGNED_OUT, reason: "revoked" }),
    ];
    h.gateway.dropConnections();
    await until(() => h.signedOut.length === 1, "signed out");
    expect(h.signedOut).toEqual(["revoked"]);
    for (let i = 3; i < 7; i++) s.push(click(i));
    await until(async () => (await h.pending(h.trajectoryId)) === 4, "kept");
    await until(
      () => h.sockets.every((socket) => socket.readyState === WebSocket.CLOSED),
      "every socket closed",
    );
    await h.clock.advance(30 * MINUTE);
    expect(h.grants).toHaveLength(2);
    expect(h.signedOut).toHaveLength(1);
    expect(h.last().intelligence).toBe("off");

    // Connecting this browser again as the same person: a resumed streamer with a new token sends them all, once each.
    h.outbox = await openOutbox(h.dbName);
    await h.start({ resume: true });
    await until(() => h.stored(h.trajectoryId).length === 9, "everything sent");
    expect(clicks(h.stored(h.trajectoryId))).toEqual(range(7));
  });

  test("a retry that never clears settles into a hold after 30 minutes", async () => {
    const h = await setup();
    h.passScript = Array.from({ length: 500 }, busyPass);
    const begun = h.clock.now();
    await h.start();
    await until(() => h.grants.length === 1, "the first attempt");
    await h.clock.advance(29 * MINUTE);
    expect(h.last()).toMatchObject({ intelligence: "retrying", reason: BUSY });
    expect(
      h.statuses.some((entry) => entry.status.intelligence === "held"),
    ).toBe(false);
    await h.clock.advance(2 * MINUTE);
    const held = h.statuses.find(
      (entry) => entry.status.intelligence === "held",
    );
    expect(held?.status.reason).toBe(BUSY);
    expect(held?.at).toBeGreaterThanOrEqual(begun + 30 * MINUTE);
    expect(held?.at).toBeLessThan(begun + 31 * MINUTE);
    const attempts = h.grants.length;
    await h.clock.advance(5 * MINUTE);
    expect(h.grants).toHaveLength(attempts + 1);
    await h.clock.advance(5 * MINUTE);
    expect(h.grants).toHaveLength(attempts + 2);
    expect(h.last().intelligence).toBe("held");
    expect(h.kept).toHaveLength(1);
  });

  for (const [store, id] of [
    [
      "unpacked, under policy or from the Chrome Web Store",
      RECORDER_EXTENSION_ID,
    ],
    ["from Edge Add-ons", EDGE_ADDONS_ID],
  ] as const)
    test(`a refused socket is retired and retried with a fresh pass, and the third refusal names CHECK_ORIGIN with the id the Recorder runs under, ${store}`, async () => {
      // The offscreen document's own chrome.runtime, under that id.
      const fake = installChromeFake({
        extensionId: id,
        context: "offscreen-document",
      });
      try {
        await refusedThreeTimes(checkOrigin(id));
      } finally {
        fake.uninstall();
      }
    });

  async function refusedThreeTimes(CHECK_ORIGIN: string): Promise<void> {
    const h = await setup();
    let refusals = 3;
    h.beforeSocket = () => {
      if (refusals-- > 0) h.gateway.refuseNextSocket();
    };
    const s = await h.start();
    s.push(click(0));
    await until(
      () => h.clock.delays().includes(2_000),
      "the back-off before the third pass",
    );
    expect(h.grants).toHaveLength(2);
    expect(
      h.statuses.some((entry) => entry.status.reason === CHECK_ORIGIN),
    ).toBe(false);
    await h.clock.advance(2_000);
    await until(() => h.last().intelligence === "held", "held");
    expect(h.last().reason).toBe(CHECK_ORIGIN);
    await sleep(300); // phoenix.js would reconnect a dropped socket after 10 ms of real time
    expect(h.grants).toHaveLength(3);
    expect(new Set(h.tokens).size).toBe(3);
    expect(h.socketUrls).toHaveLength(3);
    expect(
      h.socketUrls.map((url) => new URL(url).searchParams.get("join_token")),
    ).toEqual(h.tokens);
    expect(await h.pending(h.trajectoryId)).toBe(3);
  }

  describe("join rows", () => {
    test("entitlement_unavailable retries with its sentence (row 30)", async () => {
      const h = await setup();
      h.gateway.refuseNextJoin("entitlement_unavailable");
      h.gateway.refuseNextJoin("entitlement_unavailable");
      await h.start();
      await until(() => h.clock.delays().includes(1_000), "the first back-off");
      expect(h.last().intelligence).toBe("connecting");
      await h.clock.advance(1_000);
      await until(
        () => h.clock.delays().includes(2_000),
        "the second back-off",
      );
      expect(h.last()).toMatchObject({
        intelligence: "retrying",
        reason: ENTITLEMENT,
      });
      await h.clock.advance(2_000);
      await started(h);
      expect(h.grants).toHaveLength(3);
    });

    test("marketplace_license_required retries (row 28)", async () => {
      const h = await setup();
      h.gateway.refuseNextJoin("marketplace_license_required");
      h.gateway.refuseNextJoin("marketplace_license_required");
      await h.start();
      await until(() => h.clock.delays().includes(1_000), "the first back-off");
      await h.clock.advance(1_000);
      await until(
        () => h.clock.delays().includes(2_000),
        "the second back-off",
      );
      expect(h.last()).toMatchObject({
        intelligence: "retrying",
        reason: MARKETPLACE,
      });
      await h.clock.advance(2_000);
      await started(h);
    });

    test("trajectories_unavailable parks with every event kept (row 26)", async () => {
      const h = await setup();
      h.gateway.refuseNextJoin("trajectories_unavailable");
      const s = await h.start();
      s.push(click(0));
      await until(() => h.last().intelligence === "parked", "parked");
      expect(await h.pending(h.trajectoryId)).toBe(3);
      expect(h.clock.delays()).toEqual([]);
    });

    for (const reason of ["unauthorized", "token_not_trajectory_scoped"]) {
      test(`${reason} re-grants once and parks on the second (row 27)`, async () => {
        const h = await setup();
        h.gateway.refuseNextJoin(reason);
        h.gateway.refuseNextJoin(reason);
        await h.start();
        await until(() => h.last().intelligence === "parked", "parked");
        expect(h.last().reason).toBe(PASS_REFUSED);
        expect(h.grants).toHaveLength(2);
        expect(new Set(h.tokens).size).toBe(2);
        expect(h.kept[0]).toMatchObject({ parked: true, reason: PASS_REFUSED });
      });
    }

    test("unmatched topic parks (row 31)", async () => {
      const h = await setup();
      h.gateway.refuseNextJoin("unmatched topic");
      await h.start();
      await until(() => h.last().intelligence === "parked", "parked");
      expect(h.last().reason).toBe(PASS_REFUSED);
      expect(h.grants).toHaveLength(1);
    });

    test("a join with no reply in 10 s retries (row 32)", async () => {
      const h = await setup();
      h.scripted = [new ScriptedPhoenix("silent")];
      const s = await h.start();
      s.push(click(0));
      await until(() => h.clock.delays().includes(10_000), "the join timeout");
      await h.clock.advance(10_000);
      await until(() => h.clock.delays().includes(1_000), "the back-off");
      expect(h.grants).toHaveLength(1);
      await h.clock.advance(1_000);
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent on the second pass",
      );
      expect(h.grants).toHaveLength(2);
    });

    test("join crashed re-grants and a second holds (row 32a)", async () => {
      const h = await setup();
      h.gateway.replyNextJoin("join crashed");
      h.gateway.replyNextJoin("join crashed");
      await h.start();
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(
        "Learning capture is paused: Intelligence refused to open the capture channel (join crashed).",
      );
      expect(h.grants).toHaveLength(2);
    });

    test("a join reason no row names holds (row 32a)", async () => {
      const h = await setup();
      h.gateway.replyNextJoin("something new");
      await h.start();
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(
        "Learning capture is paused: Intelligence refused to open the capture channel (something new).",
      );
      expect(h.grants).toHaveLength(1);
    });
  });

  describe("push rows", () => {
    test("an ok acknowledgement removes the batch (row 33)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      s.push(click(0));
      await until(
        async () =>
          (await h.pending(h.trajectoryId)) === 0 &&
          h.stored(h.trajectoryId).length === 3,
        "acknowledged",
      );
      expect(h.last().events).toEqual({
        sent: 3,
        pending: 0,
        rejected: 0,
        dropped: 0,
      });
    });

    test("an acknowledgement whose counts do not add up re-grants and resends (row 34)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.ackWithBadCounts();
      s.push(click(0));
      s.push(click(1));
      await until(() => h.grants.length === 2, "a fresh pass");
      await until(
        async () => (await h.pending(h.trajectoryId)) === 0,
        "resent",
      );
      expect(clicks(h.stored(h.trajectoryId))).toEqual([0, 1]);
    });

    for (const reason of [
      "entitlement_unavailable",
      "trajectory_outbox_full",
      "trajectory_over_share",
      "storage_unavailable",
    ]) {
      test(`${reason} with retryable: true resends the same batch on the same channel (rows 36, 37)`, async () => {
        const h = await setup();
        const s = await h.start();
        await started(h);
        h.gateway.replyNextPush({ reason });
        s.push(click(0));
        await until(() => h.clock.delays().includes(1_000), "the back-off");
        expect(h.stored(h.trajectoryId)).toHaveLength(2);
        await h.clock.advance(1_000);
        await until(() => h.stored(h.trajectoryId).length === 3, "resent");
        expect(h.grants).toHaveLength(1);
        const last = h.gateway.batches.slice(-2);
        expect(last.map((batch) => batch.count)).toEqual([1, 1]);
      });
    }

    test("batch_too_large splits the batch in two and both halves are stored (row 38)", async () => {
      const h = await setup();
      h.gate = makeGate();
      const s = await h.start();
      for (let i = 0; i < 10; i++) s.push(click(i));
      await until(
        async () => (await h.pending(h.trajectoryId)) === 12,
        "queued",
      );
      h.gateway.replyNextPush({ reason: "batch_too_large" });
      h.gate.release();
      await until(
        async () => (await h.pending(h.trajectoryId)) === 0,
        "both halves",
      );
      expect(h.gateway.batches.map((batch) => batch.count)).toEqual([12, 6, 6]);
      expect(clicks(h.stored(h.trajectoryId))).toEqual(range(10));
    });

    test("a lone event the gateway still refuses is dropped and reported once (row 38)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.replyNextPush({ reason: "batch_too_large" });
      s.push(click(0, { text: "x".repeat(16_000) }));
      await until(() => h.last().events.dropped === 1, "dropped");
      s.push(click(1));
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "the next event",
      );
      expect(clicks(h.stored(h.trajectoryId))).toEqual([1]);
      expect(h.gateway.batches.at(-1)?.dropped).toBe(1);
      expect(h.gateway.droppedReported(h.trajectoryId)).toBe(1);
    });

    for (const reason of [
      "invalid_batch",
      "trajectory_mismatch",
      "scope_mismatch",
      "unsupported_event",
    ]) {
      test(`${reason} parks with every event kept (rows 39, 40)`, async () => {
        const h = await setup();
        const s = await h.start();
        await started(h);
        h.gateway.replyNextPush({ reason });
        s.push(click(0));
        s.push(click(1));
        await until(() => h.last().intelligence === "parked", "parked");
        expect(h.last().reason).toBe(MESSAGES_REFUSED);
        await until(
          async () => (await h.pending(h.trajectoryId)) === 2,
          "kept",
        );
        await h.clock.advance(30 * MINUTE);
        expect(h.grants).toHaveLength(1);
      });
    }

    test("a per-event rejection removes the batch, shows the count and is never resent (rows 33, 46)", async () => {
      const h = await setup();
      h.gate = makeGate();
      const s = await h.start();
      for (let i = 0; i < 3; i++) s.push(click(i));
      await until(
        async () => (await h.pending(h.trajectoryId)) === 5,
        "queued",
      );
      h.gateway.rejectNextEvents(2);
      h.gate.release();
      await until(
        async () => (await h.pending(h.trajectoryId)) === 0,
        "removed",
      );
      await sleep(50);
      expect(h.gateway.batches).toHaveLength(1);
      expect(h.stored(h.trajectoryId)).toHaveLength(3);
      expect(h.last().events).toMatchObject({ sent: 3, rejected: 2 });
    });

    test("no reply, then a channel error, re-grants and resends (row 41)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.delayNextAck(15_000);
      s.push(click(0));
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "stored, unanswered",
      );
      h.gateway.sendChannelError(h.trajectoryId);
      await until(() => h.grants.length === 2, "a fresh pass");
      await until(
        async () => (await h.pending(h.trajectoryId)) === 0,
        "resent",
      );
      expect(h.stored(h.trajectoryId)).toHaveLength(3);
    });

    test("an unknown reason with retryable: true resends, and without it holds (row 41a)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.replyNextPush({ reason: "something_new", retryable: true });
      s.push(click(0));
      await until(() => h.clock.delays().includes(1_000), "the back-off");
      await h.clock.advance(1_000);
      await until(() => h.stored(h.trajectoryId).length === 3, "resent");

      h.gateway.replyNextPush({ reason: "something_else" });
      s.push(click(1));
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(
        "Learning capture is paused: Intelligence refused the recorder's messages (something_else).",
      );
      expect(await h.pending(h.trajectoryId)).toBe(1);
    });
  });

  describe("channel rows", () => {
    test("a channel error re-grants (row 45)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.sendChannelError(h.trajectoryId);
      await until(() => h.grants.length === 2, "a fresh pass");
      s.push(click(0));
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent on the new channel",
      );
    });

    test("expired with an unknown reason re-grants and the third in a row holds (row 45a)", async () => {
      const h = await setup();
      await h.start();
      await started(h);
      const expire = () =>
        h.gateway.expire(
          h.trajectoryId,
          "session_revoked" as "session_expired",
        );
      expire();
      await until(() => h.grants.length === 2, "the first re-grant");
      await streaming(h);
      expire();
      await until(() => h.clock.delays().includes(2_000), "the back-off");
      await h.clock.advance(2_000);
      await until(() => h.grants.length === 3, "the second re-grant");
      await streaming(h);
      expire();
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(
        "Learning capture is paused: Intelligence ended the capture session (session_revoked).",
      );
    });

    test("expired session_expired re-grants at once, backs off, and the next one in a row holds (row 42)", async () => {
      const h = await setup();
      await h.start();
      await started(h);
      h.gateway.expire(h.trajectoryId, "session_expired");
      await until(() => h.grants.length === 2, "the first re-grant, at once");
      await streaming(h);
      h.gateway.expire(h.trajectoryId, "session_expired");
      await until(() => h.clock.delays().includes(2_000), "the back-off");
      await h.clock.advance(1_999);
      expect(h.grants).toHaveLength(2);
      await h.clock.advance(1);
      await until(
        () => h.grants.length === 3,
        "the second re-grant, after the back-off",
      );
      await streaming(h);
      h.gateway.expire(h.trajectoryId, "session_expired");
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(
        "Learning capture is paused: Intelligence ended this person's capture sessions.",
      );
      expect(h.grants).toHaveLength(3);
    });

    test("expired entitlement_unavailable retries (row 44)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.expire(h.trajectoryId, "entitlement_unavailable");
      await until(() => h.clock.delays().includes(1_000), "the back-off");
      s.push(click(0));
      await h.clock.advance(1_000);
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent after the retry",
      );
      expect(h.grants).toHaveLength(2);
    });
  });

  describe("socket rows", () => {
    test("a draining gateway (1012) re-grants (row 25)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.closeSockets(1012);
      await until(() => h.grants.length === 2, "a fresh pass");
      s.push(click(0));
      await until(
        () => h.stored(h.trajectoryId).length === 3,
        "sent on the new socket",
      );
    });

    test("closes 1011 and 1000 re-grant and the third in a row holds; the channel error that follows each counts once (row 25a)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gateway.closeSockets(1011);
      await until(() => h.grants.length === 2, "the first re-grant");
      await streaming(h);
      h.gateway.closeSockets(1000);
      await until(() => h.clock.delays().includes(2_000), "the back-off");
      expect(h.last().intelligence).toBe("retrying");
      await h.clock.advance(2_000);
      await until(() => h.grants.length === 3, "the second re-grant");
      await streaming(h);
      h.gateway.closeSockets(1011);
      await until(() => h.last().intelligence === "held", "held");
      expect(h.last().reason).toBe(BUSY);
      s.push(click(0));
      await until(async () => (await h.pending(h.trajectoryId)) === 1, "kept");
    });

    test("a frame too large (1009) re-grants and the next batch goes in halves (row 25a)", async () => {
      const h = await setup();
      const s = await h.start();
      await started(h);
      h.gate = makeGate();
      h.gateway.closeSockets(1009);
      await until(() => h.grants.length === 2, "a fresh pass");
      for (let i = 0; i < 40; i++) s.push(click(i));
      await until(
        async () => (await h.pending(h.trajectoryId)) === 40,
        "queued",
      );
      const before = h.gateway.batches.length;
      h.gate.release();
      await until(async () => (await h.pending(h.trajectoryId)) === 0, "sent");
      expect(
        h.gateway.batches.slice(before).map((batch) => batch.count),
      ).toEqual([25, 15]);
    });

    test("a socket failure followed by the channel's own errors counts once, and is torn down as core does", async () => {
      const h = await setup();
      const scripted = new ScriptedPhoenix("ok");
      h.scripted = [scripted];
      await h.start();
      await streaming(h);
      scripted.failTransport();
      await until(() => h.grants.length === 2, "one fresh pass");
      await started(h);
      expect(h.grants).toHaveLength(2);
      expect(
        h.statuses.some((entry) => entry.status.intelligence === "held"),
      ).toBe(false);
      expect(scripted.boundChannelHandlers).toBe(0);
      expect(scripted.offChannel.map(([event]) => event).sort()).toEqual([
        "expired",
        "phx_close",
        "phx_error",
      ]);
      expect(scripted.left).toBe(true);
      expect(scripted.offSocketRefs).toHaveLength(3);
      expect(scripted.disconnected).toBe(true);
      const calls = scripted.connectCalls;
      scripted.connect();
      expect(scripted.connectCalls).toBe(calls);
    });
  });

  test("a pass pointing at a disallowed gateway opens no socket", async () => {
    const h = await setup();
    const s = await h.start(
      {},
      { allowedGatewayUrls: ["wss://intelligence.example.com"] },
    );
    s.push(click(0));
    await until(() => h.last().intelligence === "parked", "parked");
    expect(h.last().reason).toBe(PASS_HINTS.GATEWAY_NOT_ALLOWED);
    expect(h.factoryCalls).toBe(0);
    expect(h.sockets).toHaveLength(0);
    await until(async () => (await h.pending(h.trajectoryId)) === 3, "kept");
  });

  test("an event with a NUL character is sanitised and accepted, an event that already has a seq is wrapped, not overwritten, and an event whose timestamp is past Intelligence's maximum is given the time it was queued", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    s.push({
      type: "CUSTOM",
      name: "input",
      timestamp: 1,
      value: { text: "a\u0000b" },
    });
    s.push({
      type: "CUSTOM",
      name: "custom",
      timestamp: 2,
      value: { seq: 42, x: 1 },
    });
    s.push({ type: "CUSTOM", name: "list", timestamp: 3, value: [1, 2] });
    s.push({
      type: "CUSTOM",
      name: "page",
      timestamp: 253_402_300_800_000,
      value: {},
    });
    s.push({ type: "CUSTOM", name: "n".repeat(250), timestamp: 4, value: {} });
    await until(() => h.stored(h.trajectoryId).length === 7, "all accepted");
    const [input, custom, list, page, long] = h.stored(h.trajectoryId).slice(2);
    expect(payloadOf(input).text).toBe("a�b");
    expect(payloadOf(custom)).toEqual({
      data: { seq: 42, x: 1 },
      seq: custom.seq,
    });
    expect(payloadOf(list)).toEqual({ data: [1, 2], seq: list.seq });
    expect((page.raw as { timestamp: number }).timestamp).toBe(h.clock.now());
    expect(long.name).toBe("n".repeat(200));
    expect(h.last().events.rejected).toBe(0);
  });

  test("a split emoji never reaches the gateway", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    s.push({
      type: "CUSTOM",
      name: "input",
      timestamp: 1,
      value: { text: "hello \ud83d" },
    });
    s.push({
      type: "CUSTOM",
      name: "click",
      timestamp: 2,
      value: { target: { text: `x\ud83d${"😀".repeat(20_000)}` } },
    });
    await until(() => h.stored(h.trajectoryId).length === 4, "both stored");
    await until(
      async () => (await h.pending(h.trajectoryId)) === 0,
      "acknowledged",
    );
    const [input, trimmed] = h.stored(h.trajectoryId).slice(2);
    expect(payloadOf(input).text).toBe("hello �");
    const text = (payloadOf(trimmed).target as { text: string }).text;
    expect(text.startsWith("x�")).toBe(true);
    expect(text.isWellFormed()).toBe(true);
    expect(text.length).toBeLessThan(400);
    expect(h.gateway.undecodable).toBe(0);
    expect(h.grants).toHaveLength(1);
  });

  test("a drop count survives a refused push and is never sent twice", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    s.push(oversize());
    h.gateway.replyNextPush({ reason: "trajectory_outbox_full" });
    s.push(click(0));
    await until(() => h.clock.delays().includes(1_000), "the back-off");
    expect(h.gateway.droppedReported(h.trajectoryId)).toBe(0);
    await h.clock.advance(1_000);
    await until(() => h.stored(h.trajectoryId).length === 3, "resent");
    expect(h.gateway.droppedReported(h.trajectoryId)).toBe(1);

    // An unanswered push may have been stored: its resend carries 0, so the total never exceeds 1.
    s.push(oversize());
    h.gateway.delayNextAck(15_000);
    s.push(click(1));
    await until(
      () => h.stored(h.trajectoryId).length === 4,
      "stored, unanswered",
    );
    await h.clock.advance(10_000);
    await until(async () => (await h.pending(h.trajectoryId)) === 0, "resent");
    expect(h.gateway.batches.at(-1)?.dropped).toBe(0);
    expect(h.gateway.droppedReported(h.trajectoryId)).toBe(2);
    expect(h.last().events.dropped).toBe(2);
  });

  test("a long recording rolls over to a linked trajectory and closes the old one", async () => {
    const h = await setup();
    const s = await h.start({}, { rolloverAt: 30 });
    for (let i = 0; i < 70; i++) s.push(click(i));
    await until(() => h.onTrajectory.length === 2, "two rollovers");
    const [second, third] = h.onTrajectory;
    await until(
      () =>
        clicks([
          ...h.stored(h.trajectoryId),
          ...h.stored(second),
          ...h.stored(third),
        ]).length === 70,
      "all 70 stored",
    );
    await until(() => h.closed.length === 2, "the first two finished");
    expect(h.rollovers).toEqual([
      { trajectoryId: second, recordingId: "rec-1" },
      { trajectoryId: third, recordingId: "rec-1" },
    ]);
    expect(h.closed).toEqual([h.trajectoryId, second]);
    const first = h.stored(h.trajectoryId);
    const middle = h.stored(second);
    const last = h.stored(third);
    expect(first).toHaveLength(31);
    expect(first.at(-1)?.name).toBe("recording.continued");
    expect(payloadOf(first.at(-1) ?? first[0]).nextTrajectoryId).toBe(second);
    expect(middle[0].name).toBe("recording.continues");
    expect(payloadOf(middle[0])).toMatchObject({
      previousTrajectoryId: h.trajectoryId,
      recordingId: "rec-1",
    });
    expect(payloadOf(middle.at(-1) ?? middle[0]).nextTrajectoryId).toBe(third);
    expect(payloadOf(last[0]).previousTrajectoryId).toBe(second);
    expect(clicks([...first, ...middle, ...last])).toEqual(range(70));
    const stopped = await s.stop("succeeded");
    expect(stopped.trajectoryIds).toEqual([h.trajectoryId, second, third]);
    await s.drained();
    expect(h.closed).toEqual([h.trajectoryId, second, third]);
  });

  test("a rollover never pushes a trajectory past Intelligence's cap, and asks OpenBot for nothing but passes", async () => {
    const h = await setup({ eventCap: 40 });
    const s = await h.start({}, { rolloverAt: 30 });
    for (let i = 0; i < 98; i++) s.push(click(i));
    await until(() => h.onTrajectory.length === 3, "three rollovers");
    const chain = [h.trajectoryId, ...h.onTrajectory];
    await until(
      () => clicks(chain.flatMap((id) => h.stored(id))).length === 98,
      "every event stored",
    );
    expect(clicks(chain.flatMap((id) => h.stored(id)))).toEqual(range(98));
    for (const id of chain) expect(h.stored(id).length).toBeLessThanOrEqual(31);
    expect(h.last().events.rejected).toBe(0);
    // Every rolled-over trajectory streams on its own pass, at once: nothing waits on OpenBot.
    expect([...new Set(h.grants)].sort()).toEqual([...chain].sort());
  });

  test("a rollover is reported for the recording it belongs to, and Start over finishes it with a failed outcome", async () => {
    const h = await setup();
    const second = crypto.randomUUID();
    const s = await h.start({}, { rolloverAt: 5, randomUUID: () => second });
    // started, outcome and three clicks fill the first trajectory; the fourth click rolls over.
    for (let i = 0; i < 4; i++) s.push(click(i));
    await until(() => h.rollovers.length === 1, "the rollover reported");
    expect(h.rollovers).toEqual([
      { trajectoryId: second, recordingId: "rec-1" },
    ]);
    const next = crypto.randomUUID();
    await s.rotate({
      recordingId: "rec-2",
      trajectoryId: next,
      title: "Try again",
    });
    await until(() => h.stored(next).length === 2, "the new recording streams");
    expect(h.closed).toEqual([h.trajectoryId, second]);
    const rolled = h.stored(second);
    expect(clicks(rolled)).toEqual([3]);
    expect(payloadOf(rolled.at(-1) ?? rolled[0])).toMatchObject({
      workflow: "File the weekly report",
      phase: "failed",
    });
  });

  test("discard clears every trajectory of the recording, a rolled-over one included, and sends nothing more", async () => {
    const h = await setup();
    const second = crypto.randomUUID();
    const s = await h.start({}, { rolloverAt: 5, randomUUID: () => second });
    await started(h);
    // The rollover's pass is refused for good (row 7): its events stay kept.
    h.passScript = [
      classifyGrantError(503, {
        code: "INTELLIGENCE_RUNTIME_REQUIRED",
        message: "Intelligence is not configured.",
      }),
    ];
    for (let i = 0; i < 4; i++) s.push(click(i));
    await until(() => h.kept.length === 1, "the rollover kept");
    expect(h.kept[0]?.trajectoryId).toBe(second);
    await until(
      async () => (await h.pending(h.trajectoryId)) === 0,
      "the first trajectory sent",
    );

    await s.discard();
    expect(await h.pending(second)).toBe(0);
    expect(await h.outbox.trajectories()).toEqual([]);
    const grants = h.grants.length;
    s.push(click(9));
    await h.clock.advance(10 * MINUTE);
    expect(h.grants).toHaveLength(grants);
    expect(h.stored(second)).toEqual([]);
  });

  test("stop queues the outcome last, keeps draining with no deadline and closes each trajectory when empty", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    h.passScript = Array.from({ length: 7 }, busyPass);
    h.gateway.dropConnections();
    await until(() => h.grants.length === 2, "the gateway is paused");
    s.push(click(0));
    s.push(click(1));
    const stopped = await s.stop("failed", { reason: "interrupted" });
    expect(stopped.trajectoryIds).toEqual([h.trajectoryId]);
    expect(await h.pending(h.trajectoryId)).toBe(3);
    let drained = false;
    void s.drained().then(() => {
      drained = true;
    });
    await h.clock.advance(90_000);
    expect(drained).toBe(false);
    expect(h.closed).toEqual([]);
    await h.clock.advance(MINUTE);
    await until(() => drained, "drained");
    expect(h.closed).toEqual([h.trajectoryId]);
    const stored = h.stored(h.trajectoryId);
    expect(stored.at(-1)?.name).toBe("outcome");
    expect(payloadOf(stored.at(-1) ?? stored[0])).toMatchObject({
      workflow: "File the weekly report",
      phase: "failed",
      reason: "interrupted",
    });
    expect(clicks(stored)).toEqual([0, 1]);
    expect(h.last().intelligence).toBe("off");
  });

  test("Start over finishes the old trajectory as failed with reason restarted, linked to the new one both ways, switches at once and resolves after the old one drained", async () => {
    const h = await setup();
    const s = await h.start();
    await started(h);
    s.push(click(0));
    const next = crypto.randomUUID();
    let atRotated: string[] = [];
    const startOverAt = h.clock.now() - 5;
    const rotating = s
      .rotate({
        recordingId: "rec-2",
        trajectoryId: next,
        title: "Try again",
        startedAt: startOverAt,
      })
      .then(() => {
        atRotated = h.stored(h.trajectoryId).map((entry) => entry.name);
      });
    s.push(click(1));
    await until(
      () => h.stored(next).length === 3,
      "the new trajectory streams at once",
    );
    await rotating;
    // Resolved only once Intelligence had every event of the old trajectory, its failed outcome included.
    expect(atRotated).toEqual([
      "recording.started",
      "outcome",
      "click",
      "recording.restarted",
      "outcome",
    ]);
    expect(h.closed).toEqual([h.trajectoryId]);
    const old = h.stored(h.trajectoryId);
    // Why it ended and where it went on, as a rollover's recording.continued names its next trajectory.
    expect(payloadOf(old.at(-2) ?? old[0])).toMatchObject({
      nextTrajectoryId: next,
      nextRecordingId: "rec-2",
    });
    expect(old.at(-1)?.name).toBe("outcome");
    expect(payloadOf(old.at(-1) ?? old[0])).toMatchObject({
      workflow: "File the weekly report",
      phase: "failed",
      reason: "restarted",
    });
    const fresh = h.stored(next);
    expect(fresh.map((entry) => entry.name)).toEqual([
      "recording.started",
      "outcome",
      "click",
    ]);
    expect(payloadOf(fresh[0])).toMatchObject({
      recordingId: "rec-2",
      previousTrajectoryId: h.trajectoryId,
    });
    expect(payloadOf(fresh[1])).toMatchObject({
      workflow: "Try again",
      phase: "started",
    });
    // Stamped when Start over began, before the new recording's first step.
    expect(
      fresh
        .slice(0, 2)
        .map((entry) => (entry.raw as { timestamp: number }).timestamp),
    ).toEqual([startOverAt, startOverAt]);
    expect(clicks(old)).toEqual([0]);
    expect(clicks(fresh)).toEqual([1]);
  });

  test("a recording that stops while its events are kept reports them again, outcome included, for the stopped recording", async () => {
    const h = await setup();
    h.gateway.refuseNextJoin("trajectories_unavailable");
    const s = await h.start();
    s.push(click(0));
    await until(() => h.last().intelligence === "parked", "parked");
    await until(async () => (await h.pending(h.trajectoryId)) === 3, "kept");
    expect(h.kept).toHaveLength(1);

    await s.stop("succeeded");
    expect(await h.pending(h.trajectoryId)).toBe(4);
    expect(h.kept).toHaveLength(2);
    expect(h.kept[1]).toEqual({
      reason: GATEWAY_OFF,
      parked: true,
      events: 4,
      recordingId: "rec-1",
      trajectoryId: h.trajectoryId,
    });
  });

  test("Start over reports the replaced recording's kept events again, outcome included", async () => {
    const h = await setup();
    h.gateway.refuseNextJoin("trajectories_unavailable");
    const s = await h.start();
    await until(() => h.last().intelligence === "parked", "parked");
    await until(async () => (await h.pending(h.trajectoryId)) === 2, "kept");
    expect(h.kept).toHaveLength(1);

    const next = crypto.randomUUID();
    let rotated = false;
    void s
      .rotate({
        recordingId: "rec-2",
        trajectoryId: next,
        title: "Try again",
      })
      .then(() => {
        rotated = true;
      });
    await until(() => h.stored(next).length === 2, "the new recording streams");
    expect(h.kept).toHaveLength(2);
    // Its start, its link to the new trajectory and its outcome.
    expect(h.kept[1]).toEqual({
      reason: GATEWAY_OFF,
      parked: true,
      events: 4,
      recordingId: "rec-1",
      trajectoryId: h.trajectoryId,
    });
    expect(await h.pending(h.trajectoryId)).toBe(4);
    expect(rotated).toBe(false);
  });

  test("events still in the outbox are sent after the offscreen document restarts", async () => {
    const h = await setup();
    h.passScript = [busyPass()];
    // The first document never gets to retry: it is gone.
    const s = await h.start({}, { timers: virtualClock().timers });
    for (let i = 0; i < 5; i++) s.push(click(i));
    await until(async () => (await h.pending(h.trajectoryId)) === 7, "queued");
    h.outbox = await openOutbox(h.dbName);
    const resumed = await h.start({ resume: true });
    resumed.push(click(5));
    await until(() => h.stored(h.trajectoryId).length === 8, "everything sent");
    const stored = h.stored(h.trajectoryId);
    expect(stored.slice(0, 2).map((entry) => entry.name)).toEqual([
      "recording.started",
      "outcome",
    ]);
    expect(clicks(stored)).toEqual(range(6));
  });

  test("an outcome without a workflow would be rejected, so an empty title is refused at construction", async () => {
    const h = await setup();
    await expect(h.start({ title: "" })).rejects.toThrow("needs a title");
    await expect(h.start({ title: "   " })).rejects.toThrow("needs a title");
    const s = await h.start();
    await expect(
      s.rotate({
        recordingId: "rec-2",
        trajectoryId: crypto.randomUUID(),
        title: "",
      }),
    ).rejects.toThrow("needs a title");
    expect(h.grants.length).toBeLessThanOrEqual(1);
  });
});