Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

52.0 KB
Raw Blame
import {
  afterAll,
  afterEach,
  beforeEach,
  describe,
  expect,
  spyOn,
  test,
} from "bun:test";
import { randomUUID } from "node:crypto";
import { and, eq, like } from "drizzle-orm";
import { createAgentProfileStore } from "../src/agents/profile-store";
import type { AgentActor } from "../src/agents/profile-types";
import { createChannelStore } from "../src/channels/routes";
import { createThreadIdentity } from "../src/channels/thread-identity";
import { createDatabase } from "../src/db/client";
import {
  agentProfiles,
  agents,
  channels,
  intelligenceChannelMappings,
  routineRuns,
  routineSweeps,
  routines,
  users,
  workItems,
} from "../src/db/schema";
import { MINIMUM_INTERVAL_MS } from "../src/routines/schedule";
import { createRoutineStore } from "../src/routines/store";
import {
  DEFAULT_GRACE_MS,
  dispatchClaimedRoutines,
  offerDueRoutines,
  ROUTINE_FIRE_KIND,
} from "../src/routines/sweep";
import { createWorkQueue, DEFAULT_MAX_ATTEMPTS } from "../src/work/queue";
import { TEST_POOL, testDatabaseUrl } from "./support/database";

/**
 * Both halves of the sweep against a real PostgreSQL and the real `work_items` queue.
 *
 * A fake queue cannot answer the only question worth asking here. The whole reason a routine fires
 * once when three replicas wake at 09:00 is a primary key on `(kind, key)` and
 * `on conflict do nothing`, which is a promise the database makes; a stub that remembered what it
 * had been offered would pass every test below while production produced three runs.
 *
 * THIS FILE OWNS `ROUTINE_FIRE_KIND`. Unlike `work-queue.integration.test.ts`, which invents a
 * per-file kind, the kind under test here is a shared constant, so the rows cannot be namespaced
 * away — the cleanup below deletes every row of that kind. A second test file that offers this kind
 * would race this one and both would flake. Put such tests here.
 *
 * NO CLAIM, LEASE OR OWNERSHIP INTERNALS ARE TESTED HERE. `for update skip locked`, leases named on
 * the database's clock and the attempt count belong to `work-queue.integration.test.ts`. What this
 * file tests is which firings the sweep decides are worth offering, and — for the consuming half —
 * whether the consumer honours the booleans the queue hands back: a `renew` that says the lease has
 * gone, a `finish` that says the item was not ours, an attempt count that has reached its cap.
 */
const databaseUrl = testDatabaseUrl();
const database = createDatabase(databaseUrl, TEST_POOL);
const profileStore = createAgentProfileStore(
  database,
  new URL("https://managed.example.test/ag-ui"),
);
const channelStore = createChannelStore(
  database,
  profileStore,
  createThreadIdentity("test-deployment"),
);
const store = createRoutineStore(database);
const queue = createWorkQueue(database);

const testPrefix = `routine-sweep-${randomUUID()}`;
const createdUserIds: string[] = [];
const createdAgentIds: string[] = [];
const createdChannelIds: string[] = [];

/** Every day at 09:00 UTC: comfortably above the floor, and one occurrence is one day. */
const DAILY = "0 9 * * *";

/**
 * The dispatch half of the options, recorded rather than dialled.
 *
 * `offerDueRoutines` never calls it — it only puts work on the queue — so a dispatch that started
 * happening in the offering phase shows up here as a recorded call rather than as nothing. What the
 * consuming phase hands it is a run id, which is the only thing `/internal/routines/run` is told.
 */
const dispatched: string[] = [];
const dispatch = async (routineRunId: string) => {
  dispatched.push(routineRunId);
};

function sweepOptions(overrides: Record<string, unknown> = {}) {
  return {
    routineStore: store,
    queue,
    dispatch,
    owner: "sweep-test",
    ...overrides,
  } as Parameters<typeof offerDueRoutines>[0];
}

beforeEach(async () => {
  dispatched.length = 0;
  await database.delete(workItems).where(eq(workItems.kind, ROUTINE_FIRE_KIND));
});

afterEach(async () => {
  // Routines first, so a failure part-way through cleanup leaves nothing pointing at rows this file
  // is about to delete.
  for (const userId of createdUserIds) {
    await database.delete(routines).where(eq(routines.ownerUserId, userId));
  }
  for (const channelId of createdChannelIds.splice(0)) {
    await database
      .delete(intelligenceChannelMappings)
      .where(eq(intelligenceChannelMappings.channelId, channelId));
    await database.delete(channels).where(eq(channels.id, channelId));
  }
  for (const agentId of createdAgentIds.splice(0)) {
    await database
      .delete(agentProfiles)
      .where(eq(agentProfiles.agentId, agentId));
    await database.delete(agents).where(eq(agents.id, agentId));
  }
  for (const userId of createdUserIds.splice(0)) {
    await database.delete(users).where(eq(users.id, userId));
  }
});

afterAll(async () => {
  await database.delete(workItems).where(eq(workItems.kind, ROUTINE_FIRE_KIND));
  await database.$client.close();
});

async function createUser(): Promise<AgentActor> {
  const id = `${testPrefix}-user-${randomUUID()}`;
  await database.insert(users).values({
    id,
    email: `${id}@example.test`,
    name: "Routine Sweep Test User",
  });
  createdUserIds.push(id);
  return { id, role: "user" };
}

async function createAgent(owner: AgentActor, name = "Expense Manager") {
  const profile = await profileStore.create(owner, {
    name,
    title: "Finance Operations",
    roleDescription: "Review receipts.",
    visibility: "private",
  });
  createdAgentIds.push(profile.id);
  return profile.id;
}

async function createChannel(owner: AgentActor, agentIds: string[]) {
  const channel = await channelStore.create(owner, agentIds);
  createdChannelIds.push(channel.id);
  return channel;
}

/** A person, a Bot, the one channel they share, and a routine on it. */
async function makeRoutine(instruction = "Summarise the day.") {
  const owner = await createUser();
  const agentId = await createAgent(owner);
  const channel = await createChannel(owner, [agentId]);
  const routine = await store.create({
    ownerUserId: owner.id,
    agentId,
    channelId: channel.id,
    instruction,
    cron: DAILY,
  });
  return { owner, agentId, channel, routine };
}

/**
 * The create path always computes a future stamp, so a due-in-the-past row is written directly.
 *
 * THE STAMPS IN THIS FILE ARE DELIBERATELY ANCIENT, and the `now` option is moved to match them.
 * `dueRoutines` is not owner-scoped and orders oldest-due first, so a test that asserts on ordering
 * or on a limit has to be sure its own rows sort ahead of anything else in the database; and the
 * grace policy is measured against `now`, so the injected clock is what makes "two minutes late" and
 * "a month late" mean anything in a row stamped in 2001.
 */
async function makeDueAt(routineId: string, nextRunAt: Date): Promise<Date> {
  await database
    .update(routines)
    .set({ nextRunAt })
    .where(eq(routines.id, routineId));
  const [row] = await database
    .select({ nextRunAt: routines.nextRunAt })
    .from(routines)
    .where(eq(routines.id, routineId))
    .limit(1);
  // Read back rather than trusting the Date we wrote: the stamp the sweep keys on is Postgres's.
  return row?.nextRunAt as Date;
}

async function readRoutine(routineId: string) {
  const [row] = await database
    .select()
    .from(routines)
    .where(eq(routines.id, routineId))
    .limit(1);
  return row;
}

/** Every queued firing of one routine, whatever minute it was keyed on. */
async function firingsFor(routineId: string) {
  return await database
    .select()
    .from(workItems)
    .where(
      and(
        eq(workItems.kind, ROUTINE_FIRE_KIND),
        like(workItems.key, `${routineId}:%`),
      ),
    );
}

/** Every run row this routine has, in-flight ones included: `status` is null until something ends. */
async function runsFor(routineId: string) {
  return await database
    .select()
    .from(routineRuns)
    .where(eq(routineRuns.routineId, routineId));
}

/**
 * One routine, due at `due`, offered by a sweep that thinks it is `at`.
 *
 * The offer is made by the real `offerDueRoutines` rather than by a hand-written insert, so what the
 * consuming half claims is the row and the payload production would give it — including
 * `scheduledFor`, which the consumer re-checks against the grace window before it fires anything.
 */
async function offerFiring(routineId: string, due: Date, at: Date) {
  await makeDueAt(routineId, due);
  await offerDueRoutines(sweepOptions({ now: () => at }));
}

/**
 * Age a queued firing by hand.
 *
 * The queue names every moment of its own in SQL, so a test cannot wait for a lease to lapse or for
 * a release delay to pass; it writes the timestamp the queue would have arrived at. That is a test
 * driving the clock, not a test reimplementing the queue: what is asserted afterwards is still the
 * queue's own answer to `claim`, `renew` and `purge`.
 */
async function backdate(
  key: string,
  values: Partial<typeof workItems.$inferInsert>,
) {
  await database
    .update(workItems)
    .set(values)
    .where(and(eq(workItems.kind, ROUTINE_FIRE_KIND), eq(workItems.key, key)));
}

test("the grace window stays under the schedule floor, as a compile/test-time fact rather than prose", () => {
  // Two consecutive occurrences of one routine must never both be inside the grace window — that
  // would make a routine's own next firing look like a re-offer of a stale one. The floor between
  // two occurrences is `MINIMUM_INTERVAL_MS`, so the grace window has to stay strictly under it.
  expect(DEFAULT_GRACE_MS).toBeLessThan(MINIMUM_INTERVAL_MS);
});

describe("offering the firings that are due", () => {
  /**
   * THE TEST THIS FILE EXISTS FOR. Three replicas wake on the same minute and read the same due
   * row; the person gets one run. Nothing in the sweep coordinates that — the offer key carries the
   * minute, so the second and third offers collide on the primary key, and the compare-and-set on
   * `next_run_at` means only one of them moves the clock.
   */
  test("two sweeps racing on one due routine offer it exactly once", async () => {
    const { routine } = await makeRoutine();
    const from = await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z"));
    const now = () => new Date("2001-01-01T09:26:00Z");

    const outcomes = await Promise.all([
      offerDueRoutines(sweepOptions({ now })),
      offerDueRoutines(sweepOptions({ now })),
    ]);

    // Both sweeps saw it as due and both offered it, which is the honest report: each did put the
    // work on the queue. What must be single is the row, and the clock.
    for (const outcome of outcomes) {
      expect(outcome.offered).toContain(routine.id);
    }
    expect(await firingsFor(routine.id)).toHaveLength(1);

    const after = await readRoutine(routine.id);
    expect(after?.nextRunAt.getTime()).toBeGreaterThan(from.getTime());
    // One occurrence on from the stamp it was given, not two: a second advance would have moved it
    // to the 3rd of January.
    expect(after?.nextRunAt.toISOString()).toBe("2001-01-02T09:00:00.000Z");
  });

  /**
   * The key format is a compatibility surface, not an implementation detail: it is the identity of a
   * firing, already written into rows in `work_items`. Asserted literally so a later change to the
   * truncation is a failing test here rather than a routine that fires twice in production.
   */
  test("the offer key is the routine and the minute it was due", async () => {
    const { routine } = await makeRoutine();
    await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z"));

    await offerDueRoutines(
      sweepOptions({ now: () => new Date("2001-01-01T09:26:00Z") }),
    );

    const [firing] = await firingsFor(routine.id);
    expect(firing?.kind).toBe(ROUTINE_FIRE_KIND);
    expect(firing?.key).toBe(`${routine.id}:2001-01-01T09:25Z`);
    expect(firing?.payload).toEqual({
      routineId: routine.id,
      scheduledFor: "2001-01-01T09:25:00.000Z",
    });
  });

  /**
   * The crash between the offer and the advance, which is why the offer comes first.
   *
   * The stamp is put back by hand to stand for the sweep that died before it could move the clock —
   * or the replica that lost the compare-and-set. Either way the next pass reads the same stamp,
   * renders the same key, and adds nothing: the firing is not lost and it is not doubled.
   */
  test("a second pass over the same stamp adds no row, before or after the run", async () => {
    const { routine } = await makeRoutine();
    const from = await makeDueAt(routine.id, new Date("2001-01-01T09:25:00Z"));
    const now = () => new Date("2001-01-01T09:26:00Z");

    await offerDueRoutines(sweepOptions({ now }));
    expect(await firingsFor(routine.id)).toHaveLength(1);

    await makeDueAt(routine.id, from);
    await offerDueRoutines(sweepOptions({ now }));
    expect(await firingsFor(routine.id)).toHaveLength(1);

    /*
     * AND AFTER THE FIRING HAS HAPPENED, which is the half that is easy to lose. `finish` marks the
     * row rather than deleting it, so a finished row still counts as a conflict
     * (`server/src/work/queue.ts:120-133`) — deleting it would hand the key back and make the
     * recovery path a duplicate-run path. The claim here is setup for that state, not a test of
     * claiming.
     */
    const [claimed] = await queue.claim({
      kind: ROUTINE_FIRE_KIND,
      owner: "sweep-test",
      leaseMs: 30_000,
    });
    expect(
      await queue.finish({
        kind: ROUTINE_FIRE_KIND,
        key: claimed?.key as string,
        owner: "sweep-test",
      }),
    ).toBe(true);

    await makeDueAt(routine.id, from);
    await offerDueRoutines(sweepOptions({ now }));

    const firings = await firingsFor(routine.id);
    expect(firings).toHaveLength(1);
    expect(firings[0]?.finishedAt).not.toBeNull();
  });

  test("a routine that is switched off, or not due yet, is not offered", async () => {
    const { owner: offOwner, routine: off } =
      await makeRoutine("Switched off.");
    await store.setEnabled(offOwner.id, off.id, false);
    await makeDueAt(off.id, new Date("2001-01-01T09:25:00Z"));

    const { routine: ahead } = await makeRoutine("Still ahead.");
    // The create path already put this in the future; nothing rounds it down.
    const aheadStamp = (await readRoutine(ahead.id))?.nextRunAt as Date;

    const { offered } = await offerDueRoutines(
      sweepOptions({ now: () => new Date("2001-01-01T09:26:00Z") }),
    );

    expect(offered).not.toContain(off.id);
    expect(offered).not.toContain(ahead.id);
    expect(await firingsFor(off.id)).toHaveLength(0);
    expect(await firingsFor(ahead.id)).toHaveLength(0);
    // And neither clock moved: a routine nobody offered is a routine nobody advanced.
    expect((await readRoutine(off.id))?.nextRunAt.getTime()).toBe(
      new Date("2001-01-01T09:25:00Z").getTime(),
    );
    expect((await readRoutine(ahead.id))?.nextRunAt.getTime()).toBe(
      aheadStamp.getTime(),
    );
  });

  test("the limit bounds one pass, and the rest wait for the next", async () => {
    const first = (await makeRoutine("First.")).routine;
    const second = (await makeRoutine("Second.")).routine;
    const third = (await makeRoutine("Third.")).routine;
    await makeDueAt(first.id, new Date("2001-01-01T09:25:00Z"));
    await makeDueAt(second.id, new Date("2001-01-01T09:26:00Z"));
    await makeDueAt(third.id, new Date("2001-01-01T09:27:00Z"));

    const { offered } = await offerDueRoutines(
      sweepOptions({ limit: 2, now: () => new Date("2001-01-01T09:28:00Z") }),
    );

    // Oldest due first, so a backlog drains in the order it built up.
    expect(offered).toEqual([first.id, second.id]);
    expect(await firingsFor(third.id)).toHaveLength(0);
    expect((await readRoutine(third.id))?.nextRunAt.getTime()).toBe(
      new Date("2001-01-01T09:27:00Z").getTime(),
    );
  });
});

/**
 * A STALE STAMP IS NOT A BACKLOG TO REPLAY.
 *
 * A routine whose stamp is a month behind must not fire once per missed occurrence when the worker
 * comes back: a person would get thirty summaries of thirty days ago. And it must not drain one
 * occurrence per pass either — that kept a fifteen-minute routine silent a further fortnight after a
 * month of downtime, because stepping through ~2,900 missed occurrences at one per five-minute sweep
 * is itself two weeks. One pass makes the clock current, silently.
 */
describe("draining a stale stamp instead of replaying it", () => {
  test("a month-old firing is caught up to current in one pass, and a two-minute-old one fires", async () => {
    const { routine: stale } = await makeRoutine("A month behind.");
    const { routine: fresh } = await makeRoutine("Two minutes late.");
    const staleFrom = await makeDueAt(
      stale.id,
      new Date("2001-01-01T09:00:00Z"),
    );
    await makeDueAt(fresh.id, new Date("2001-02-01T08:58:00Z"));

    const { offered } = await offerDueRoutines(
      sweepOptions({ now: () => new Date("2001-02-01T09:00:00Z") }),
    );

    expect(offered).toEqual([fresh.id]);
    expect(await firingsFor(stale.id)).toHaveLength(0);
    expect(await firingsFor(fresh.id)).toHaveLength(1);

    // Advanced past the whole backlog in one move: the next occurrence is computed from the pass's
    // own moment, not one day along a thirty-one-day drain, so the routine is silent for the missed
    // month and then simply current. (Whether it is still "due" is Postgres's real clock's judgement,
    // which is why this asserts the landing point rather than a second pass over a 2001 stamp.)
    const after = await readRoutine(stale.id);
    expect(after?.nextRunAt.getTime()).toBeGreaterThan(staleFrom.getTime());
    expect(after?.nextRunAt.toISOString()).toBe("2001-02-02T09:00:00.000Z");
  });

  test("the grace window is a setting, so a caller can say what counts as worth having", async () => {
    const { routine } = await makeRoutine();
    await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z"));

    // Twenty minutes late. Outside the default window, inside a thirty-minute one.
    const now = () => new Date("2001-01-01T09:20:00Z");
    await offerDueRoutines(sweepOptions({ now }));
    expect(await firingsFor(routine.id)).toHaveLength(0);

    await makeDueAt(routine.id, new Date("2001-01-01T09:00:00Z"));
    const { offered } = await offerDueRoutines(
      sweepOptions({ now, graceMs: 30 * 60_000 }),
    );
    expect(offered).toContain(routine.id);
    expect(await firingsFor(routine.id)).toHaveLength(1);
  });
});

/**
 * One poisoned routine is one person's problem, not everybody's — and not silently for ever.
 *
 * A cron the schedule module refuses makes `advanceNextRun` throw. An unguarded loop would take the
 * whole pass down with it, for every other person's routine too; and a loop that only warned would
 * read the same row as due on every pass for ever, burning one of the pass's slots while its owner
 * saw a routine that quietly stopped. So the pass survives it, says so, and switches the routine off
 * with the reason written where the routines page reads it.
 */
describe("surviving a routine that cannot be scheduled", () => {
  test("a poisoned cron is warned about, switched off with its reason on the page, and the next routine still offered", async () => {
    const { owner: poisonedOwner, routine: poisoned } =
      await makeRoutine("Unschedulable.");
    const { routine: healthy } = await makeRoutine("Perfectly fine.");
    // Only a direct write can make this row: `create` and `update` both refuse a cron the schedule
    // module cannot read, which is exactly why the bad row has to be simulated rather than created.
    await database
      .update(routines)
      .set({ cron: "not a cron" })
      .where(eq(routines.id, poisoned.id));
    // The poisoned one is due FIRST, so a pass that dies on it never reaches the healthy one.
    await makeDueAt(poisoned.id, new Date("2001-01-01T09:25:00Z"));
    await makeDueAt(healthy.id, new Date("2001-01-01T09:26:00Z"));

    // Captured as it is written rather than read off the spy afterwards, so restoring the real
    // `console.warn` cannot take the evidence with it.
    const lines: string[] = [];
    const warn = spyOn(console, "warn").mockImplementation((...args) => {
      lines.push(String(args[0]));
    });
    let offered: string[] = [];
    try {
      ({ offered } = await offerDueRoutines(
        sweepOptions({ now: () => new Date("2001-01-01T09:27:00Z") }),
      ));
    } finally {
      warn.mockRestore();
    }

    expect(offered).toContain(healthy.id);
    expect(await firingsFor(healthy.id)).toHaveLength(1);

    // Said out loud, with the routine in it: a sweep that swallowed this would look clean while one
    // routine was switched off.
    const complaint = lines.find((line) => line.includes(poisoned.id));
    expect(complaint).toBeDefined();
    expect(JSON.parse(complaint as string).routineId).toBe(poisoned.id);

    /*
     * SWITCHED OFF, NOT LEFT TO WEDGE. The clock could not move, so left enabled this row would be
     * read as due and thrown over on every pass for ever — invisible to its owner, who would see a
     * routine that simply stopped. Disabled, it stops burning a due slot, and the skipped run row is
     * the announcement: the sweep has no channel to speak in, so the page's last-run column is where
     * the owner learns why.
     */
    const after = await readRoutine(poisoned.id);
    expect(after?.enabled).toBe(false);
    expect(after?.nextRunAt.getTime()).toBe(
      new Date("2001-01-01T09:25:00Z").getTime(),
    );
    const [summary] = await store.listFor(poisonedOwner.id);
    expect(summary?.enabled).toBe(false);
    expect(summary?.lastRun?.status).toBe("skipped");

    // That single firing is the entire blast radius: pin it as exactly one `work_items` row, the
    // offer that preceded the throw, rather than leaving it inferred from the warning alone.
    expect(await firingsFor(poisoned.id)).toHaveLength(1);
  });

  /**
   * The wedge that motivated the deterministic floor: `45,55 8 * * *` used to be ACCEPTED when
   * created between the pair (the next two occurrences sampled 23h50m apart), and then the first
   * advance handed the schedule an `after` of 08:45, saw the ten-minute pair, and threw — on every
   * pass, for ever, while the routine silently never fired again. Creation now refuses it, so the
   * row is written directly to stand for the ones already in the wild.
   */
  test("a sub-floor cron already in the table is switched off instead of wedging the sweep", async () => {
    const { owner, routine } = await makeRoutine("Grandfathered in.");
    await database
      .update(routines)
      .set({ cron: "45,55 8 * * *" })
      .where(eq(routines.id, routine.id));
    await makeDueAt(routine.id, new Date("2001-01-01T08:45:00Z"));

    const warn = spyOn(console, "warn").mockImplementation(() => {});
    try {
      await offerDueRoutines(
        sweepOptions({ now: () => new Date("2001-01-01T08:46:00Z") }),
      );
    } finally {
      warn.mockRestore();
    }

    const [summary] = await store.listFor(owner.id);
    expect(summary?.enabled).toBe(false);
    expect(summary?.lastRun?.status).toBe("skipped");
    // The reason on the run row is the schedule's own sentence, so the owner (and their Bot) can
    // propose a schedule that works rather than guess at what broke.
    const runs = await runsFor(routine.id);
    expect(runs[0]?.error).toContain("15 minutes");

    // And a disabled routine is not due: the pass after this one no longer spends a slot on it.
    expect((await store.dueRoutines(50)).map((row) => row.id)).not.toContain(
      routine.id,
    );
  });
});

/**
 * The consuming half: a claimed item becomes a run, and the queue's booleans are believed.
 *
 * Every branch below is about the gap between the offer and the firing. The item was put on the
 * queue by another replica at another time, and by the time this one claims it the routine may have
 * been switched off, deleted, or the occurrence may simply have gone stale while the item sat behind
 * a backlog. The queue answers those questions with booleans — `renew`, `finish`, `release` — and a
 * consumer that treats them as formalities is a consumer that fires twice.
 */
describe("consuming a claimed firing", () => {
  const at = (moment: string) => () => new Date(moment);

  test("a claimed firing is dispatched once, finished, and never claimed again", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );

    const report = await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:26:00Z") }),
    );

    expect(report.fired).toEqual([routine.id]);
    expect(report.skipped).toEqual([]);
    // The run row is what the dispatch is told about, and it owns the outcome from here: opened with
    // no status, because null is the in-flight state.
    const runs = await runsFor(routine.id);
    expect(runs).toHaveLength(1);
    expect(runs[0]?.status).toBeNull();
    expect(dispatched).toEqual([runs[0]?.id]);

    // Marked, not deleted: the finished row is what a late replica's re-offer collides with.
    const [row] = await firingsFor(routine.id);
    expect(row?.finishedAt).not.toBeNull();
    expect(row?.claimedBy).toBeNull();

    // And the next sweep claims nothing, so the person gets one run rather than one per sweep.
    const second = await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:27:00Z") }),
    );
    expect(second.considered).toBe(0);
    expect(dispatched).toHaveLength(1);
    expect(await runsFor(routine.id)).toHaveLength(1);
  });

  test("a dispatch that throws pushes the firing out, and it comes back with its attempts grown", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );

    const report = await dispatchClaimedRoutines(
      sweepOptions({
        now: at("2001-01-01T09:26:00Z"),
        dispatch: async () => {
          throw new Error("the server answered 503");
        },
      }),
    );

    expect(report.fired).toEqual([]);
    expect(report.skipped[0]?.routineId).toBe(routine.id);
    expect(report.skipped[0]?.reason).toContain("503");

    /*
     * The run row this attempt opened stays open FOR NOW, on purpose: a dispatch that timed out may
     * be a dispatch a wedged server accepted anyway, with a detached turn coming back for this very
     * row, and only age can tell that apart from abandonment. The reaper closes it once it is older
     * than any turn could still be running — the age-scoped test further down pins that half.
     */
    const runs = await runsFor(routine.id);
    expect(runs).toHaveLength(1);
    expect(runs[0]?.status).toBeNull();

    /*
     * THE ROW, NOT THE RETURN VALUE. A consumer that reported a failure and finished the item anyway
     * would pass any assertion made on the report alone, and the firing would be gone for good.
     */
    const [row] = await firingsFor(routine.id);
    expect(row?.finishedAt).toBeNull();
    expect(row?.claimedBy).toBeNull();
    expect(row?.attempts).toBe(1);
    // The reason stays on the row, so an item that eventually runs out of attempts says why.
    expect(row?.lastError).toContain("503");
    // Pushed out rather than retried in the same pass: whatever refused this will probably refuse it
    // again in the next second.
    expect(row?.runAt.getTime()).toBeGreaterThan(Date.now() + 30_000);

    // Claimable again once the delay has passed, and the second hand-out says it is a second one.
    await backdate(row?.key as string, {
      runAt: new Date("2001-01-01T09:27:00Z"),
    });
    const [again] = await queue.claim({
      kind: ROUTINE_FIRE_KIND,
      owner: "sweep-test",
      leaseMs: 30_000,
    });
    expect(again?.key).toBe(row?.key);
    expect(again?.attempts).toBe(2);
  });

  test("a routine switched off between the offer and the firing is finished without dispatching", async () => {
    const { owner, routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    await store.setEnabled(owner.id, routine.id, false);

    const report = await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:26:00Z") }),
    );

    expect(report.fired).toEqual([]);
    expect(report.skipped[0]?.routineId).toBe(routine.id);
    expect(dispatched).toEqual([]);
    expect(await runsFor(routine.id)).toHaveLength(0);
    // Finished rather than released: re-running cannot make a switched-off routine want to fire.
    const [row] = await firingsFor(routine.id);
    expect(row?.finishedAt).not.toBeNull();
  });

  test("a routine deleted between the offer and the firing is finished without dispatching", async () => {
    const { owner, routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    // A hard delete, which is what `remove` does — and it takes the routine's runs with it.
    await store.remove(owner.id, routine.id);

    const report = await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:26:00Z") }),
    );

    expect(report.fired).toEqual([]);
    expect(report.skipped[0]?.routineId).toBe(routine.id);
    expect(dispatched).toEqual([]);
    // Finished, not released: no number of retries will make a deleted routine exist.
    const [row] = await firingsFor(routine.id);
    expect(row?.finishedAt).not.toBeNull();
  });

  /**
   * THE WINDOW IS ENFORCED AGAIN AT FIRING TIME, not only at offering time.
   *
   * The offer refused anything staler than the grace window, but the queue's own machinery can
   * outlive that window: a backlogged sweep, or five releases at a minute each, and the item is
   * claimed long after the occurrence it names. Firing it then posts "here is your morning summary"
   * in the afternoon, which is exactly what the stale-stamp policy exists to prevent.
   */
  test("a firing claimed after its window has passed is finished without dispatching, and leaves no run row", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      // Offered one minute late, well inside the window. Nothing here is a stale offer.
      new Date("2001-01-01T09:26:00Z"),
    );

    // Claimed twenty minutes after the occurrence, which is outside the default ten.
    const report = await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:45:00Z") }),
    );

    expect(report.fired).toEqual([]);
    expect(report.skipped[0]?.routineId).toBe(routine.id);
    expect(dispatched).toEqual([]);
    /*
     * NO ORPHAN. The check sits beside the deleted/disabled branch, before `insertRun`, so a skipped
     * firing leaves nothing in `routine_runs` — a run row with no outcome and nothing coming to give
     * it one would show on the routines page as a firing that started and never ended.
     */
    expect(await runsFor(routine.id)).toHaveLength(0);
    // Finished rather than released: re-delivery cannot make a past occurrence current.
    const [row] = await firingsFor(routine.id);
    expect(row?.finishedAt).not.toBeNull();

    // The window is still a setting, so a caller that wants the late firing can have it.
    await makeDueAt(routine.id, new Date("2001-02-01T09:25:00Z"));
    await offerDueRoutines(
      sweepOptions({ now: () => new Date("2001-02-01T09:26:00Z") }),
    );
    const generous = await dispatchClaimedRoutines(
      sweepOptions({
        now: at("2001-02-01T09:45:00Z"),
        graceMs: 30 * 60_000,
      }),
    );
    expect(generous.fired).toEqual([routine.id]);
  });

  /**
   * THE LESSON FROM `culler.ts:141-151`, which is why the renew comes before anything else.
   *
   * A batch is many and a lease is one. The first consumer claims, is slow, and its lease lapses;
   * another replica takes the item and runs it. If the first then dispatched what it was holding,
   * the person would get the same summary twice — and worse, the first would `finish` an item that
   * belongs to the second, taking the lease away from a run that is still going.
   */
  test("a consumer whose lease has gone neither dispatches nor finishes the item that is now somebody else's", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    const [offered] = await firingsFor(routine.id);
    const now = at("2001-01-01T09:26:00Z");

    const dispatchedBySecond: string[] = [];
    const finishedByFirst: string[] = [];
    let second: Awaited<ReturnType<typeof dispatchClaimedRoutines>> | undefined;

    /*
     * The first consumer's claim succeeds and then it stalls: the lease is written into the past and
     * a whole second consumer runs the item to completion before the first gets to its own loop.
     * Standing in for a slow pass rather than reimplementing one — everything after this point is
     * still the queue's own answer.
     */
    const stalling = {
      ...queue,
      claim: async (input: Parameters<typeof queue.claim>[0]) => {
        const claimed = await queue.claim(input);
        await backdate(offered?.key as string, {
          leaseUntil: new Date("2001-01-01T00:00:00Z"),
        });
        second = await dispatchClaimedRoutines(
          sweepOptions({
            owner: "consumer-second",
            now,
            dispatch: async (runId: string) => {
              dispatchedBySecond.push(runId);
            },
          }),
        );
        return claimed;
      },
      finish: async (input: Parameters<typeof queue.finish>[0]) => {
        finishedByFirst.push(input.key);
        return await queue.finish(input);
      },
    };

    const first = await dispatchClaimedRoutines(
      sweepOptions({
        owner: "consumer-first",
        queue: stalling,
        now,
        dispatch: async (runId: string) => {
          dispatched.push(`first:${runId}`);
        },
      }),
    );

    // The second consumer did the work, exactly once.
    expect(second?.fired).toEqual([routine.id]);
    expect(dispatchedBySecond).toHaveLength(1);
    expect(await runsFor(routine.id)).toHaveLength(1);

    // The first stopped at `renew`, which is not an error: the item is being handled, just not here.
    expect(first.considered).toBe(1);
    expect(first.fired).toEqual([]);
    expect(first.skipped[0]?.routineId).toBe(routine.id);
    expect(dispatched).toEqual([]);
    // And it never called `finish`, so it could not have taken the item off a run in flight.
    expect(finishedByFirst).toEqual([]);
  });

  /**
   * At the cap the item stops being claimed, so this loop never sees that routine again: every
   * sweep looks clean while one person's routine silently never fires. The row carries the count and
   * the reason for anybody who queries the table; the warning is for whoever reads the logs.
   */
  test("at the attempt cap the firing stops being claimed, and the giving up is said out loud", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );

    const lines: string[] = [];
    const warn = spyOn(console, "warn").mockImplementation((...args) => {
      lines.push(String(args[0]));
    });
    try {
      await dispatchClaimedRoutines(
        sweepOptions({
          now: at("2001-01-01T09:26:00Z"),
          // One go, so the first failure is also the last.
          maxAttempts: 1,
          dispatch: async () => {
            throw new Error("the server answered 503");
          },
        }),
      );
    } finally {
      warn.mockRestore();
    }

    const complaint = lines.find((line) => line.includes(routine.id));
    expect(complaint).toBeDefined();
    const said = JSON.parse(complaint as string);
    expect(said.routineId).toBe(routine.id);
    expect(said.attempts).toBe(1);
    expect(said.reason).toContain("503");

    /*
     * And it is the cap that stops it, not the release delay: the delay is put in the past first, so
     * a claim that still refuses the item is refusing it for having run out of goes.
     */
    const [row] = await firingsFor(routine.id);
    await backdate(row?.key as string, {
      runAt: new Date("2001-01-01T09:27:00Z"),
    });
    expect(
      await queue.claim({
        kind: ROUTINE_FIRE_KIND,
        owner: "sweep-test",
        leaseMs: 30_000,
        maxAttempts: 1,
      }),
    ).toEqual([]);
  });

  /**
   * NO RUN ROW STAYS OPEN FOR EVER AFTER THE CAP. `insertRun` runs before `dispatch` on every
   * attempt, so a dispatch that throws on the very last attempt leaves an open (`status` null) run
   * row: the item stops being claimed at the cap and the queue's own machinery has nothing to do
   * with `routine_runs`. Nothing closes that row at the give-up itself — a timed-out dispatch may be
   * a turn a wedged server accepted, and only age can tell — but the reaper closes it on a later
   * pass, so `listFor` (the routines page's read) stops showing "running now" for a routine that
   * never ran. At Helm defaults the old give-up branch was unreachable in time anyway (five 1-minute
   * retries against a 10-minute grace), so these rows used to leak for ever.
   */
  test("a run row leaked at the attempt cap is closed by a later pass, so the page stops reading 'running'", async () => {
    const { owner, routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );

    const warn = spyOn(console, "warn").mockImplementation(() => {});
    try {
      await dispatchClaimedRoutines(
        sweepOptions({
          now: at("2001-01-01T09:26:00Z"),
          maxAttempts: 1,
          dispatch: async () => {
            throw new Error("the server answered 503");
          },
        }),
      );

      // Old enough that no turn could still be running it; the reaper compares on the database's
      // clock, so the age is written rather than waited for.
      const [leaked] = await runsFor(routine.id);
      await database
        .update(routineRuns)
        .set({ startedAt: new Date(Date.now() - 11 * 60_000) })
        .where(eq(routineRuns.id, leaked?.id as string));

      await dispatchClaimedRoutines(
        sweepOptions({ now: at("2001-01-01T09:40:00Z") }),
      );
    } finally {
      warn.mockRestore();
    }

    // No run row for this routine is left open once the reaper has passed.
    const runs = await runsFor(routine.id);
    expect(runs.length).toBeGreaterThan(0);
    expect(runs.every((run) => run.status !== null)).toBe(true);

    // The page-facing read agrees, and it reads "skipped", not "failed": the turn never ran, so the
    // routine's own failure streak must not grow — ten flapping dispatches used to read as ten turn
    // failures, enough to trip the fatigue rule and switch a perfectly healthy routine off.
    const [summary] = await store.listFor(owner.id);
    expect(summary?.lastRun?.status).toBe("skipped");
    expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date);
    expect(await store.consecutiveFailures(routine.id)).toBe(0);
  });

  /**
   * THE NET MUST NOT CATCH THE NEIGHBOUR'S FISH. An earlier firing of the same routine can be
   * genuinely mid-turn while a later firing runs out of dispatch attempts: the dispatch call aborts
   * at 30 seconds, but the server's detached turn keeps running for minutes and finishes its row
   * itself. The old give-up cleanup closed EVERY open run of the routine, that one included, so the
   * real turn's finish then no-oped against an already-"failed" row and an honest success was
   * recorded as a failure. Cleanup is age-scoped now, and a fresh row is by definition one a turn
   * may still be running.
   */
  test("giving up leaves a genuinely in-flight run from an earlier firing untouched", async () => {
    const { routine } = await makeRoutine();
    // The earlier firing: already dispatched, its turn still running on the server, its row open.
    const inFlight = await store.insertRun(routine.id);

    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    const warn = spyOn(console, "warn").mockImplementation(() => {});
    try {
      await dispatchClaimedRoutines(
        sweepOptions({
          now: at("2001-01-01T09:26:00Z"),
          maxAttempts: 1,
          dispatch: async () => {
            throw new Error("the server answered 503");
          },
        }),
      );
    } finally {
      warn.mockRestore();
    }

    // Nothing was mislabelled at the give-up: both rows are still open, because both are young
    // enough to be turns some server is running.
    const runs = await runsFor(routine.id);
    expect(runs.find((run) => run.id === inFlight.runId)?.status).toBeNull();

    // The in-flight turn now completes, and its outcome lands rather than no-oping against a row
    // the give-up already closed — which is exactly what the old cleanup caused.
    await store.finishRun(inFlight.runId, "succeeded");
    expect(
      (await runsFor(routine.id)).find((run) => run.id === inFlight.runId)
        ?.status,
    ).toBe("succeeded");

    // And once the abandoned attempt's row is old enough that no turn could still be running it,
    // the reaper closes it — as skipped, never touching the finished run beside it.
    const attempt = runs.find((run) => run.id !== inFlight.runId);
    await database
      .update(routineRuns)
      .set({ startedAt: new Date(Date.now() - 11 * 60_000) })
      .where(eq(routineRuns.id, attempt?.id as string));
    const warnAgain = spyOn(console, "warn").mockImplementation(() => {});
    try {
      await dispatchClaimedRoutines(
        sweepOptions({ now: at("2001-01-01T09:40:00Z") }),
      );
    } finally {
      warnAgain.mockRestore();
    }
    const after = await runsFor(routine.id);
    expect(after.find((run) => run.id === attempt?.id)?.status).toBe("skipped");
    expect(after.find((run) => run.id === inFlight.runId)?.status).toBe(
      "succeeded",
    );
  });

  /**
   * A firing abandoned mid-retry must not strand its run rows. An attempt fails inside the grace
   * window and its item is released; by the time the item is claimed again the window has passed and
   * the firing is finished as stale. The grace-skip itself opens nothing and closes nothing — the
   * open row may be a turn a wedged server accepted, and only age can tell — but the reaper closes
   * it once it is old enough, so nothing about the abandoned firing reads "running now" for ever.
   */
  test("a firing abandoned by the grace window does not strand its earlier attempt's run row", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );

    // Attempt one, inside the window: the dispatch throws and the item is pushed out for retry.
    await dispatchClaimedRoutines(
      sweepOptions({
        now: at("2001-01-01T09:26:00Z"),
        dispatch: async () => {
          throw new Error("the server answered 503");
        },
      }),
    );
    // The retry delay is written into the past so the next pass can claim the item at all, and the
    // leaked row is aged past the reaper's cutoff the same way — the clock is driven, not waited on.
    const [row] = await firingsFor(routine.id);
    await backdate(row?.key as string, {
      runAt: new Date("2001-01-01T09:27:00Z"),
    });
    const [leaked] = await runsFor(routine.id);
    await database
      .update(routineRuns)
      .set({ startedAt: new Date(Date.now() - 11 * 60_000) })
      .where(eq(routineRuns.id, leaked?.id as string));

    // Attempt two, claimed twenty minutes after the occurrence: outside the window, so the firing
    // is finished without dispatching — and the same pass's reaper closes the leaked row.
    const warn = spyOn(console, "warn").mockImplementation(() => {});
    let report: Awaited<ReturnType<typeof dispatchClaimedRoutines>>;
    try {
      report = await dispatchClaimedRoutines(
        sweepOptions({ now: at("2001-01-01T09:45:00Z") }),
      );
    } finally {
      warn.mockRestore();
    }
    expect(report.fired).toEqual([]);
    expect(report.skipped[0]?.routineId).toBe(routine.id);

    // Nothing about this firing is still open: attempt one's row is closed as skipped — no turn ran,
    // so the fatigue rule must not count it — and attempt two never opened one.
    const runs = await runsFor(routine.id);
    expect(runs).toHaveLength(1);
    expect(runs[0]?.status).toBe("skipped");
    expect(await store.consecutiveFailures(routine.id)).toBe(0);
    const [finished] = await firingsFor(routine.id);
    expect(finished?.finishedAt).not.toBeNull();
  });

  /**
   * THE REAPER, for the row no retry ever comes back for. The server 202s the dispatch, the queue
   * item is finished, and then the server dies mid-turn: its run row stays open with nothing in the
   * system holding a reference to it — the page reads "running now" for a run no process is running.
   * The sweep's consuming pass is on a clock anyway, so it is the thing that mops these up.
   */
  test("a run abandoned by a dead server is closed by the next pass, as skipped, with an honest reason", async () => {
    const { owner, routine } = await makeRoutine();
    const { runId } = await store.insertRun(routine.id);
    // Aged past the reaper's cutoff (twice the server's five-minute turn timeout) by hand: the run
    // was opened by a server that died eleven minutes ago.
    await database
      .update(routineRuns)
      .set({ startedAt: new Date(Date.now() - 11 * 60_000) })
      .where(eq(routineRuns.id, runId));

    // Nothing is claimed in this pass; the reaper alone does the work.
    const warn = spyOn(console, "warn").mockImplementation(() => {});
    let report: Awaited<ReturnType<typeof dispatchClaimedRoutines>>;
    try {
      report = await dispatchClaimedRoutines(sweepOptions());
    } finally {
      warn.mockRestore();
    }
    expect(report.considered).toBe(0);

    const runs = await runsFor(routine.id);
    expect(runs[0]?.status).toBe("skipped");
    expect(runs[0]?.error).toContain("never finished this run");
    const [summary] = await store.listFor(owner.id);
    expect(summary?.lastRun?.status).toBe("skipped");
    expect(summary?.lastRun?.finishedAt).toBeInstanceOf(Date);
  });

  /**
   * BOTH KINDS OF DONE WITH, which is the half `queue.ts:262-274` documents as having been forgotten.
   *
   * A finished row has to outlive the run long enough for a late replica to collide with it, and
   * then go. An item at its attempt cap is not finished and is reaped by nothing else, so without
   * this its key stays wedged for ever and that routine can never be offered for that minute again.
   */
  test("the purge takes the finished firing and the one that gave up, once each is past the window", async () => {
    const { routine: done } = await makeRoutine("Finished cleanly.");
    const { routine: gaveUp } = await makeRoutine("Out of attempts.");
    await offerFiring(
      done.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    await dispatchClaimedRoutines(
      sweepOptions({ now: at("2001-01-01T09:26:00Z") }),
    );
    /*
     * The finished routine's clock is parked out of reach before the second offer.
     *
     * `dueRoutines` asks Postgres what is due, so a routine whose stamp the first offer advanced to
     * another day in 2001 is still due by the real clock, and the second offer would queue a second
     * firing of it. That is the offering half behaving as designed against ancient stamps; what this
     * test wants is one finished row and one that gave up, so it says which routine each pass is
     * about rather than working around the overlap afterwards.
     */
    await makeDueAt(done.id, new Date("2999-01-01T09:00:00Z"));
    await offerFiring(
      gaveUp.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    // The giving-up warning is asserted in its own test above; here it is only noise.
    const warn = spyOn(console, "warn").mockImplementation(() => {});
    try {
      await dispatchClaimedRoutines(
        sweepOptions({
          now: at("2001-01-01T09:26:00Z"),
          maxAttempts: 1,
          dispatch: async () => {
            throw new Error("the server answered 503");
          },
        }),
      );
    } finally {
      warn.mockRestore();
    }

    const window = 24 * 60 * 60 * 1000;
    // Inside the window both rows stay: a finished row too young to have been collided with is the
    // whole reason `finish` marks rather than deletes.
    expect(
      await queue.purge({
        kind: ROUTINE_FIRE_KIND,
        olderThanMs: window,
        maxAttempts: 1,
      }),
    ).toBe(0);
    expect(await firingsFor(done.id)).toHaveLength(1);
    expect(await firingsFor(gaveUp.id)).toHaveLength(1);

    const aged = new Date(Date.now() - 2 * window);
    const [doneRow] = await firingsFor(done.id);
    const [gaveUpRow] = await firingsFor(gaveUp.id);
    await backdate(doneRow?.key as string, { finishedAt: aged });
    // The one that gave up has no `finished_at` to age at all — its `updated_at` is what dates it,
    // which is precisely why it used to be reaped by nothing.
    await backdate(gaveUpRow?.key as string, { updatedAt: aged });

    expect(
      await queue.purge({
        kind: ROUTINE_FIRE_KIND,
        olderThanMs: window,
        maxAttempts: 1,
      }),
    ).toBe(2);
    expect(await firingsFor(done.id)).toHaveLength(0);
    expect(await firingsFor(gaveUp.id)).toHaveLength(0);
  });

  /**
   * The cap the consumer warns at has to be the cap the queue stops claiming at.
   *
   * A consumer with its own number warns on the wrong pass: too low and it complains every pass
   * while the item is still being retried, too high and it never complains at all — the item stops
   * being claimed and nothing anywhere says so.
   */
  test("with no cap given, the consumer gives up on the pass the queue's own default stops handing it out", async () => {
    const { routine } = await makeRoutine();
    await offerFiring(
      routine.id,
      new Date("2001-01-01T09:25:00Z"),
      new Date("2001-01-01T09:26:00Z"),
    );
    const [offered] = await firingsFor(routine.id);
    // One short of the queue's default, so this claim is the last one the queue will allow.
    await backdate(offered?.key as string, {
      attempts: DEFAULT_MAX_ATTEMPTS - 1,
    });

    const lines: string[] = [];
    const warn = spyOn(console, "warn").mockImplementation((...args) => {
      lines.push(String(args[0]));
    });
    try {
      await dispatchClaimedRoutines(
        sweepOptions({
          now: at("2001-01-01T09:26:00Z"),
          dispatch: async () => {
            throw new Error("the server answered 503");
          },
        }),
      );
    } finally {
      warn.mockRestore();
    }

    const complaint = lines.find((line) => line.includes(routine.id));
    expect(complaint).toBeDefined();
    expect(JSON.parse(complaint as string).attempts).toBe(DEFAULT_MAX_ATTEMPTS);

    await backdate(offered?.key as string, {
      runAt: new Date("2001-01-01T09:27:00Z"),
    });
    expect(
      await queue.claim({
        kind: ROUTINE_FIRE_KIND,
        owner: "sweep-test",
        leaseMs: 30_000,
      }),
    ).toEqual([]);
  });
});

describe("saying that a sweep happened at all", () => {
  test("records the pass, so a deployment with no worker is not silence", async () => {
    await database.delete(routineSweeps);

    const before = await store.lastSweptAt();
    expect(before).toBeNull();

    await offerDueRoutines(sweepOptions());

    const after = await store.lastSweptAt();
    expect(after).not.toBeNull();
    expect(Date.now() - (after as Date).getTime()).toBeLessThan(60_000);
  });

  test("keeps one row however many passes run, and names the last sweeper", async () => {
    await database.delete(routineSweeps);

    await offerDueRoutines(sweepOptions({ owner: "sweeper-one" }));
    await offerDueRoutines(sweepOptions({ owner: "sweeper-two" }));

    const rows = await database.select().from(routineSweeps);
    expect(rows).toHaveLength(1);
    expect(rows[0]?.owner).toBe("sweeper-two");
  });
});