Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

64.7 KB
Raw Blame
import { test } from "node:test";
import assert from "node:assert/strict";
import { createScheduler } from "../src/cron/scheduler.ts";
import { runNowSettled } from "./support/settle.ts";
import { createCronStore } from "../src/cron/cron-store.ts";
import { createDeliveryStore } from "../src/delivery/delivery-store.ts";
import { createIdempotencyStore } from "../src/idempotency/idempotency-store.ts";
import { createIdentityService } from "../src/identity/identity-service.ts";
import { createMemorySessionStore } from "../src/sessions/memory-session-store.ts";
import type { LeaderLease } from "../src/persistence/leader-lease.ts";
import { scopeId, type TurnRequest, type TurnResult } from "../src/types.ts";
import { createDirectoryStore, type DirectoryStore } from "../src/directory/directory-store.ts";
import { createMemoryMap } from "../src/persistence/durable-map.ts";
import type { Cron } from "../src/types.ts";
import { createMemoryAdvisoryLock } from "../src/persistence/advisory-lock.ts";
import type { CronFireJob, CronJobQueue } from "../src/cron/job-queue.ts";
import { singleProcessDeps } from "./support/single-process.ts";

function fakeLease(isLeader: () => boolean): LeaderLease {
  return {
    async hold<T>(_key: string, fn: (lost: Promise<void>) => Promise<T>): Promise<T | null> {
      return isLeader() ? fn(new Promise<void>(() => {})) : null;
    },
  };
}

function harness(
  reply: string | ((req: TurnRequest) => Promise<TurnResult>) = "CRON-OUTPUT-XYZ",
  directory?: DirectoryStore,
  maxFiresPerTick?: number,
) {
  const crons = createCronStore();
  const deliveries = createDeliveryStore();
  const identity = createIdentityService();
  const calls: TurnRequest[] = [];
  const run = async (req: TurnRequest): Promise<TurnResult> => {
    calls.push(req);
    if (typeof reply === "function") return reply(req);
    return { status: "ok", reply };
  };
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries,
    idempotency: createIdempotencyStore(),
    identity,
    run,
    ...(directory ? { directory } : {}),
    ...(maxFiresPerTick !== undefined ? { maxFiresPerTick } : {}),
  });
  return { crons, deliveries, calls, scheduler, identity };
}

test("scheduler threads stored unattended grants into owner-mode turns", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "scan transcripts",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    unattendedGrants: ["admin.sessions.read"],
  });
  await runNowSettled(scheduler, cron.id);
  assert.deepEqual(calls[0]?.unattendedGrants, ["admin.sessions.read"]);
});

test("a channel cron runs in the channel scope and delivers its real output to the channel", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "post the standup",
    title: "Daily standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls[0]?.conversation.kind, "channel");
  assert.equal(calls[0]?.conversation.channelRef, "C1");
  const pending = await deliveries.pending("slack");
  assert.equal(pending.length, 1);
  assert.equal(pending[0]?.text, "CRON-OUTPUT-XYZ");
  assert.equal(pending[0]?.provenance?.sourceTitle, "Daily standup");
});

test("a group-DM cron runs in the group scope and delivers its real output to the group", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("group", "G1"),
    destination: { type: "group", target: "G1", audienceScopeId: scopeId("group", "G1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls[0]?.conversation.kind, "group");
  assert.equal(calls[0]?.conversation.channelRef, "G1");
  const pending = await deliveries.pending("group");
  assert.equal(pending.length, 1);
  assert.equal(pending[0]?.text, "CRON-OUTPUT-XYZ");
});

test("a group-DM relay message delivers verbatim, floored to the group", async () => {
  const { crons, deliveries, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    message: "standup in 5 🚀",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("group", "G1"),
    destination: { type: "group", target: "G1", audienceScopeId: scopeId("group", "G1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await deliveries.pending("group"))[0]?.text, "standup in 5 🚀");
});

test("a group-DM cron composed at another scope delivers verbatim (person-keyed parity, §10)", async () => {
  const { crons, deliveries, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "private digest",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: { type: "group", target: "G1", audienceScopeId: scopeId("group", "G1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await deliveries.pending("group"))[0]?.text, "CRON-OUTPUT-XYZ");
});

test("a channel cron fire gives the agent the people-here roster with real <@…> mention ids", async () => {
  const directory = createDirectoryStore();
  await directory.replace([
    { principalId: "[email protected]", displayName: "Eve", type: "internal", slackId: "U9" },
    { principalId: "U5", displayName: "Dana", type: "internal" },
  ]);
  await directory.replaceChannels(
    [{ channelId: "C1", name: "general", isPrivate: false }],
    [
      { channelId: "C1", principalId: "[email protected]" },
      { channelId: "C1", principalId: "U5" },
    ],
  );
  const { crons, calls, scheduler } = harness("done", directory);
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "remind Eve to confirm",
    owner: "[email protected]",
    createdBy: "[email protected]",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });
  await runNowSettled(scheduler, cron.id);
  const text = calls[0]?.text ?? "";
  assert.match(text, /People here: @Eve \(<@U9>\), @Dana \(<@U5>\)\./);
  assert.match(text, /@-mention them with their <@…> id/);
  assert.doesNotMatch(text, /<@eve@acme\.com>/);
});

test("a DM cron fire carries no mention roster (DMs already notify their owner)", async () => {
  const directory = createDirectoryStore();
  await directory.replace([{ principalId: "U1", displayName: "Ann", type: "internal", slackId: "U1" }]);
  const { crons, calls, scheduler } = harness("done", directory);
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "nudge me",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: { type: "slack", target: "D1", audienceScopeId: scopeId("personal", "U1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.doesNotMatch(calls[0]?.text ?? "", /People here:/);
});

test("a personal cron runs as a DM and delivers to the owner", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "drink water",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: { type: "slack", target: "D1", audienceScopeId: scopeId("personal", "U1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls[0]?.conversation.kind, "dm");
  assert.equal((await deliveries.pending("slack"))[0]?.text, "CRON-OUTPUT-XYZ");
});

test("output composed at one scope delivers verbatim to a channel the owner may post to (person-keyed parity, §10)", async () => {
  const { crons, deliveries, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "private digest",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: { type: "slack", target: "C9", audienceScopeId: scopeId("channel", "C9") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await deliveries.pending("slack"))[0]?.text, "CRON-OUTPUT-XYZ");
});

test("a personal cron with a principal destination delivers its real output to the teammate", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "let Alice know the deploy is done",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: {
      type: "principal",
      target: "U-alice",
      audienceScopeId: scopeId("personal", "U-alice"),
      onBehalfOf: "U1",
    },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls[0]?.conversation.kind, "dm");
  assert.deepEqual(calls[0]?.triggerDestination, {
    type: "principal",
    target: "U-alice",
    audienceScopeId: scopeId("personal", "U-alice"),
    onBehalfOf: "U1",
  });
  const pending = await deliveries.pending("principal");
  assert.equal(pending.length, 1);
  assert.equal(pending[0]?.text, "CRON-OUTPUT-XYZ");
});

test("a recurring verbatim reminder to a teammate delivers to the teammate", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { cron: "0 9 * * *", timezone: "America/Los_Angeles" },
    message: "daily standup notes are due",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: {
      type: "principal",
      target: "U-alice",
      audienceScopeId: scopeId("personal", "U-alice"),
      onBehalfOf: "U1",
    },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 0);
  const pending = await deliveries.pending("principal");
  assert.equal(pending.length, 1);
  assert.equal(pending[0]?.destination.target, "U-alice");
  assert.equal(pending[0]?.text, "daily standup notes are due");
  const { runs: stored } = await crons.listFires(cron.id);
  assert.equal(stored[0]?.status, "ok");
});

test("a teammate-DM cron created in a channel delivers its real output (§10 parity gate)", async () => {
  const { crons, deliveries, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "summarize this channel and DM it to Alice",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: {
      type: "principal",
      target: "U-alice",
      audienceScopeId: scopeId("personal", "U-alice"),
      onBehalfOf: "U1",
    },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await deliveries.pending("principal"))[0]?.text, "CRON-OUTPUT-XYZ");
});

test("a cron with a literal message delivers it verbatim, without running a turn", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { firstFireAt: 1 },
    message: "standup moved to 4pm",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: {
      type: "principal",
      target: "U-alice",
      audienceScopeId: scopeId("personal", "U-alice"),
      onBehalfOf: "U1",
    },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 0, "a literal message must not re-run a turn (that's the immediate-send confusion)");
  assert.equal((await deliveries.pending("principal"))[0]?.text, "standup moved to 4pm");
});

test("a destination-less cron runs its action but delivers nothing", async () => {
  const { crons, deliveries, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "nightly cleanup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 1);
  assert.equal((await deliveries.pending("slack")).length, 0);
});

test("a one-shot cron is auto-disabled after it fires", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { firstFireAt: 1 },
    action: "remind me once",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
    destination: { type: "slack", target: "D1", audienceScopeId: scopeId("personal", "U1") },
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 1);
  assert.equal((await crons.get(cron.id))?.enabled, false);
});

test("only the leader instance fires due crons on tick (non-leader skips)", async () => {
  const crons = createCronStore();
  const deliveries = createDeliveryStore();
  const idempotency = createIdempotencyStore();
  const identity = createIdentityService();
  const calls: TurnRequest[] = [];
  const run = async (req: TurnRequest): Promise<TurnResult> => {
    calls.push(req);
    return { status: "ok", reply: "OUT" };
  };
  let leader = false;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries,
    idempotency,
    identity,
    run,
    leaderLease: fakeLease(() => leader),
  });
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });

  await scheduler.tick(2000);
  assert.equal(calls.length, 0, "a non-leader instance fires nothing");
  assert.equal((await crons.get(cron.id))?.lastFiredAt, undefined, "the cron is untouched");

  leader = true;
  await scheduler.tick(2000);
  assert.equal(calls.length, 1, "the leader fires the due cron");
});

test("a conversation-owned cron keeps running as the conversation, whoever created it", async () => {
  const { crons, calls, scheduler, identity } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "post the standup",
    owner: "system:channel:C1",
    createdBy: "system:channel:C1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await identity.deactivate("U1");
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls[0]?.actor.externalId, "system:channel:C1");
  assert.equal((await crons.get(cron.id))?.enabled, true);
});

test("a person-owned cron is deleted once its owner has left the conversation", async () => {
  const directory = createDirectoryStore();
  await directory.replace([
    { principalId: "U1", displayName: "Ann", type: "internal" },
    { principalId: "U2", displayName: "Bo", type: "internal" },
  ]);
  await directory.replaceChannels(
    [{ channelId: "C1", name: "eng", isPrivate: true }],
    [{ channelId: "C1", principalId: "U2" }],
  );
  const { crons, calls, scheduler } = harness("done", directory);
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "my digest",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 0);
  assert.equal(await crons.get(cron.id), null);
});

test("an owner cron still disables when its owner leaves (unchanged)", async () => {
  const { crons, calls, scheduler, identity } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "my digest",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });
  await identity.deactivate("U1");
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 0);
  assert.equal((await crons.get(cron.id))?.enabled, false);
});

test("the default (no-op) lease ticks exactly as before — memory-mode behavior is unchanged", async () => {
  const { crons, calls, scheduler } = harness();
  await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });
  await scheduler.tick(2000);
  assert.equal(calls.length, 1, "without a lease, the tick fires due crons as before");
});

test("a failing interval cron does not starve later due crons across ticks", async () => {
  const { crons, calls, scheduler } = harness(async (req) => {
    if (req.text.includes("fail first")) throw new Error("cron failed");
    return { status: "ok", reply: "OUT" };
  });
  const failing = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "fail first",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const succeeding = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "succeed second",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });

  await scheduler.tick(2000);
  await scheduler.tick(3500);

  assert.deepEqual(
    calls.map((call) => call.idempotencyKey),
    [`cron:${failing.id}:1`, `cron:${succeeding.id}:1`, `cron:${failing.id}:1`, `cron:${succeeding.id}:3000`],
  );
});

test("a capped batch of persistent failures rotates — later due crons still get their turn", async () => {
  const { crons, calls, scheduler } = harness(
    async (req) => {
      if (req.text.includes("always fails")) throw new Error("cron failed");
      return { status: "ok", reply: "OUT" };
    },
    undefined,
    2,
  );
  for (let i = 0; i < 3; i++) {
    await crons.create({
      schedule: { everyMs: 60_000, firstFireAt: 1 },
      action: `always fails ${i}`,
      owner: "U1",
      createdBy: "U1",
      ownerScopeId: scopeId("personal", "U1"),
    });
  }
  const healthy = await crons.create({
    schedule: { everyMs: 60_000, firstFireAt: 1 },
    action: "healthy last in line",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });

  await scheduler.tick(2000);
  await scheduler.tick(3000);

  assert.ok(
    calls.some((call) => call.idempotencyKey === `cron:${healthy.id}:1`),
    "the healthy cron behind a full batch of failures fires within two ticks",
  );
});

test("capped rotation is driven by durable attempt order, not tick arrival times", async () => {
  const { crons, calls, scheduler } = harness("OUT", undefined, 2);
  const created: Cron[] = [];
  for (let i = 0; i < 4; i++) {
    created.push(
      await crons.create({
        schedule: { everyMs: 600_000, firstFireAt: 1 },
        action: `cron ${i}`,
        owner: "U1",
        createdBy: "U1",
        ownerScopeId: scopeId("personal", "U1"),
      }),
    );
  }

  await scheduler.tick(2000);
  await scheduler.tick(4000);

  for (const cron of created) {
    assert.ok(
      calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
      "every due cron fires within ceil(due/cap) ticks even when tick timestamps skip buckets",
    );
  }
});

test("a cron whose attempt marker cannot persist is held back and cannot starve the rest", async () => {
  const backing = createMemoryMap<Cron>();
  const crons = createCronStore(backing);
  const deliveries = createDeliveryStore();
  const calls: TurnRequest[] = [];
  const run = async (req: TurnRequest): Promise<TurnResult> => {
    calls.push(req);
    return { status: "ok", reply: "OUT" };
  };
  const created: Cron[] = [];
  for (let i = 0; i < 5; i++) {
    created.push(
      await crons.create({
        schedule: { everyMs: 600_000, firstFireAt: 1 },
        action: `cron ${i}`,
        owner: "U1",
        createdBy: "U1",
        ownerScopeId: scopeId("personal", "U1"),
      }),
    );
  }
  const broken = created[0]!;
  const flaky: typeof crons = {
    ...crons,
    async markAttempted(id, at) {
      if (id === broken.id) throw new Error("attempt marker write failed");
      return crons.markAttempted(id, at);
    },
  };
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons: flaky,
    deliveries,
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run,
    maxFiresPerTick: 2,
  });

  await scheduler.tick(2000);
  await scheduler.tick(4000);

  assert.ok(
    !calls.some((call) => call.idempotencyKey === `cron:${broken.id}:1`),
    "a capped batch never fires a cron whose attempt failed to persist",
  );
  for (const cron of created.slice(1)) {
    assert.ok(
      calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
      "the crons behind the broken marker still fire within two capped ticks",
    );
  }
});

test("capped rotation survives a scheduler restart mid-cycle", async () => {
  const backing = createMemoryMap<Cron>();
  const crons = createCronStore(backing);
  const deliveries = createDeliveryStore();
  const calls: TurnRequest[] = [];
  const run = async (req: TurnRequest): Promise<TurnResult> => {
    calls.push(req);
    return { status: "ok", reply: "OUT" };
  };
  const mk = () =>
    createScheduler({
      ...singleProcessDeps(),
      crons,
      deliveries,
      idempotency: createIdempotencyStore(),
      identity: createIdentityService(),
      run,
      maxFiresPerTick: 2,
    });
  const created: Cron[] = [];
  for (let i = 0; i < 4; i++) {
    created.push(
      await crons.create({
        schedule: { everyMs: 600_000, firstFireAt: 1 },
        action: `cron ${i}`,
        owner: "U1",
        createdBy: "U1",
        ownerScopeId: scopeId("personal", "U1"),
      }),
    );
  }

  await mk().tick(2000);
  await mk().tick(2000);

  for (const cron of created) {
    assert.ok(
      calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
      "a fresh scheduler instance resumes the rotation from durable state instead of restarting it",
    );
  }
});

test("runNow fires even when the current schedule slot already did (manual re-run)", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await scheduler.tick(2000);
  assert.equal(calls.length, 1);
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 2, "a manual run must not be swallowed by the already-fired slot");
});

test("runNow does not shift the schedule (lastFiredAt untouched, next tick still fires)", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await crons.get(cron.id))?.lastFiredAt, undefined, "a manual run must not stamp the schedule");
  await scheduler.tick(2000);
  assert.equal(calls.length, 2, "the scheduled fire still happens after a manual run");
  assert.equal((await crons.get(cron.id))?.lastFiredAt, 2000);
});

test("a re-enabled one-shot cron can be re-run manually", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { firstFireAt: 1 },
    action: "remind me once",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await runNowSettled(scheduler, cron.id);
  assert.equal((await crons.get(cron.id))?.enabled, false);
  await crons.update(cron.id, { enabled: true });
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 2, "re-enabling a one-shot lets it be re-run manually");
  assert.equal((await crons.get(cron.id))?.enabled, false);
});

test("a cron turn carries its fire key as the run idempotency key (atomic DB-level dedupe)", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await scheduler.tick(2000);
  assert.equal(calls[0]?.idempotencyKey, `cron:${cron.id}:1`);
  await runNowSettled(scheduler, cron.id);
  assert.match(calls[1]?.idempotencyKey ?? "", new RegExp(`^cron:${cron.id}:manual:`));
});

test("a cron fires into fresh per-fire threads while keeping per-fire idempotency keys", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "daily check-in",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await scheduler.tick(2000);
  await scheduler.tick(3500);
  await runNowSettled(scheduler, cron.id);

  assert.equal(calls.length, 3);
  const threads = calls.map((c) => c.conversation.threadRef);
  assert.equal(new Set(threads).size, 3, "each fire gets a fresh internal thread");
  assert.ok(threads.every((thread) => thread?.startsWith(`cron:${cron.id}:fire:`)));
  assert.equal(calls[0]?.idempotencyKey, `cron:${cron.id}:1`);
  assert.equal(calls[1]?.idempotencyKey, `cron:${cron.id}:3000`);
  assert.match(calls[2]?.idempotencyKey ?? "", new RegExp(`^cron:${cron.id}:manual:`));
  assert.notEqual(calls[1]?.conversation.threadRef, calls[1]?.idempotencyKey);
});

test("cron fires do not replay prior sessions or inline prior fire context", async () => {
  const crons = createCronStore();
  const sessions = createMemorySessionStore();
  const scope = scopeId("personal", "U1");
  const sessionIds: string[] = [];
  const priorAssistantSeen: number[] = [];
  const texts: string[] = [];
  const replies = ['first <invoke name="execute">', "second ok"];
  const run = async (req: TurnRequest): Promise<TurnResult> => {
    const session = await sessions.getOrCreateByThread(req.conversation.threadRef!, "dm", scope);
    sessionIds.push(session.id);
    priorAssistantSeen.push((await sessions.getEntries(session.id)).filter((e) => e.type === "assistant").length);
    texts.push(req.text);
    const { lease } = await sessions.acquireLease(session.id);
    assert.ok(lease, "fires are serial, so the lease is always free");
    await sessions.append(lease, { type: "user", payload: { text: req.text }, scopeLabel: scope });
    await sessions.append(lease, {
      type: "assistant",
      payload: { text: replies[sessionIds.length - 1] ?? "ok" },
      scopeLabel: scope,
    });
    await sessions.releaseLease(lease);
    return { status: "ok", reply: replies[sessionIds.length - 1] ?? "ok", sessionId: session.id };
  };
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run,
  });
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "daily check-in",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scope,
  });
  await scheduler.tick(2000);
  await scheduler.tick(3500);

  assert.equal(sessionIds.length, 2);
  assert.notEqual(sessionIds[0], sessionIds[1], "each fire resolves to a fresh session");
  assert.deepEqual(priorAssistantSeen, [0, 0], "fire 2 does not replay fire 1's assistant entry");
  assert.match(texts[1] ?? "", /no memory of previous fires/);
  assert.match(texts[1] ?? "", /action="runs"/);
  assert.match(texts[1] ?? "", /Files via the available Files API/);
  assert.match(texts[1] ?? "", /GET \/v1\/files\/:id\/content/);
  assert.match(texts[1] ?? "", /confirm publication succeeds/);
  assert.match(texts[1] ?? "", /If publication is unavailable/);
  assert.doesNotMatch(texts[1] ?? "", /Your workspace disk/);
  assert.doesNotMatch(texts[1] ?? "", /Recent fires/);
  assert.doesNotMatch(texts[1] ?? "", /first <invoke name="execute">/);
  assert.doesNotMatch(texts[1] ?? "", /first \[invoke name="execute"\]/);
  const { runs: stored } = await crons.listFires(cron.id);
  assert.equal(stored.length, 2);
  assert.equal(stored[0]?.reply, 'first <invoke name="execute">');
});

test("an idempotency-skipped cron fire does not create fire log history", async () => {
  const crons = createCronStore();
  const calls: TurnRequest[] = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: {
      once: async () => false,
      committed: async () => true,
    },
    identity: createIdentityService(),
    run: async (req) => {
      calls.push(req);
      return { status: "ok", reply: "OUT" };
    },
  });
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "daily check-in",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });

  await scheduler.tick(2500);

  const after = await crons.get(cron.id);
  assert.equal(calls.length, 0, "duplicate slot does not reach the agent");
  assert.equal((await crons.listFires(cron.id)).total, 0, "duplicate slot does not look like a fire");
  assert.equal(after?.lastFiredAt, 2500, "the due slot is still advanced after durable dedupe");
});

test("cron fire log omits replies that echo the runtime wrapper", async () => {
  const { crons, calls, scheduler } = harness(async (req) => ({ status: "ok", reply: `You said: ${req.text}` }));
  await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "daily check-in",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });

  await scheduler.tick(2000);
  await scheduler.tick(3500);

  const { runs: stored } = await crons.listFires((await crons.list())[0]!.id);
  assert.equal(stored[0]?.reply, "[reply echoed cron runtime context; omitted]");
  assert.doesNotMatch(calls[1]?.text ?? "", /reply=You said: \[Cron runtime context\]/);
  assert.doesNotMatch(calls[1]?.text ?? "", /reply=\[reply echoed cron runtime context; omitted\]/);
});

test("a calendar cron uses the scheduled instant for a missed tick idempotency key", async (t) => {
  t.mock.method(Date, "now", () => Date.parse("2026-06-18T15:58:00.000Z"));
  const { crons, calls, scheduler } = harness();
  const scheduledAt = Date.parse("2026-06-18T16:00:00.000Z");
  const firedAt = Date.parse("2026-06-18T16:05:00.000Z");
  const cron = await crons.create({
    schedule: { cron: "0 9 * * 1-5", timezone: "America/Los_Angeles" },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  await scheduler.tick(firedAt);
  assert.equal(calls[0]?.idempotencyKey, `cron:${cron.id}:${scheduledAt}`);
  const stored = await crons.get(cron.id);
  assert.equal(stored?.lastFiredAt, firedAt);
  assert.equal(stored?.nextFireAt, Date.parse("2026-06-19T16:00:00.000Z"));
});

test("a failing cron fire is logged, not swallowed", async (t) => {
  t.mock.timers.enable({ apis: ["setInterval"] });
  const logged: string[] = [];
  t.mock.method(console, "error", (...args: unknown[]) => {
    logged.push(args.map(String).join(" "));
  });
  const crons = createCronStore();
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async () => {
      throw new Error("boom");
    },
  });
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  scheduler.start(1000);
  t.mock.timers.tick(1000);
  for (let i = 0; i < 50 && !logged.some((l) => l.includes("[failed] scheduler: fire:")); i++) {
    await new Promise((r) => setImmediate(r));
  }
  scheduler.stop();
  assert.ok(
    logged.some((l) => l.includes("[failed] scheduler: fire:") && l.includes("boom")),
    "the fire error must reach the log",
  );
  const { runs: after } = await crons.listFires(cron.id);
  assert.equal(after.length, 1);
  assert.equal(after[0]?.status, "failed");
  assert.equal(after[0]?.note, "boom");
});

test("queue mode: fires claim the slot before running, and stale or lost claims never run", async () => {
  const crons = createCronStore();
  const calls: TurnRequest[] = [];
  let onFire: ((job: { cronId: string; scheduledAt: number }) => Promise<void>) | undefined;
  const enqueued: Array<{ cronId: string; scheduledAt: number }> = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async (req) => {
      calls.push(req);
      return { status: "ok", reply: "OUT" };
    },
    jobQueue: {
      async start(handlers) {
        onFire = handlers.onFire;
      },
      async enqueueFire(job) {
        enqueued.push(job);
      },
      healthy: () => true,
      async stop() {},
    },
  });
  scheduler.start(1000);
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  for (let i = 0; i < 20 && !onFire; i++) await new Promise((r) => setImmediate(r));
  assert.ok(onFire, "queue mode registers the fire handler");
  scheduler.notifyChanged(cron.id);
  for (let i = 0; i < 20 && !enqueued.length; i++) await new Promise((r) => setImmediate(r));
  assert.deepEqual(enqueued.pop(), { cronId: cron.id, scheduledAt: 1 }, "notifyChanged prompt-schedules the slot");

  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  assert.equal(calls.length, 1, "the job for the due slot fires");
  assert.ok((await crons.get(cron.id))!.lastFiredAt, "the claim advanced the schedule");
  assert.deepEqual(enqueued.pop()?.cronId, cron.id, "the next slot is chained");

  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  assert.equal(calls.length, 1, "a duplicate job for a claimed slot does not run the turn");
  scheduler.stop();
});

for (const queued of [false, true]) {
  test(`${queued ? "queue" : "interval"} mode: a busy cron defers without blocking other crons or losing its slot`, async () => {
    const crons = createCronStore();
    const gate = Promise.withResolvers<TurnResult>();
    const started = Promise.withResolvers<void>();
    const calls: TurnRequest[] = [];
    const enqueued: CronFireJob[] = [];
    let clock = 1000;
    let onFire: ((job: CronFireJob) => Promise<void>) | undefined;
    const jobQueue: CronJobQueue = {
      async start(handlers) {
        onFire = handlers.onFire;
      },
      async enqueueFire(job) {
        enqueued.push(job);
      },
      healthy: () => true,
      async stop() {},
    };
    const scheduler = createScheduler({
      ...singleProcessDeps(),
      crons,
      deliveries: createDeliveryStore(),
      idempotency: createIdempotencyStore(),
      identity: createIdentityService(),
      advisoryLock: createMemoryAdvisoryLock(),
      now: () => clock,
      ...(queued ? { jobQueue } : {}),
      run: async (req) => {
        calls.push(req);
        if (calls.length === 1) {
          started.resolve();
          return gate.promise;
        }
        return { status: "ok", reply: "done" };
      },
    });
    const cron = await crons.create({
      schedule: { everyMs: 1000, firstFireAt: 1000 },
      action: "slow recurring job",
      owner: "U1",
      createdBy: "U1",
      ownerScopeId: scopeId("personal", "U1"),
    });
    if (queued) {
      scheduler.start(1000);
      await scheduler.ready();
    }
    let first: Promise<void>;
    if (queued) first = onFire!({ cronId: cron.id, scheduledAt: 1000 });
    else {
      const manual = await scheduler.runNow(cron.id);
      assert.ok(manual.started);
      first = manual.settled;
    }
    await started.promise;
    clock = 2000;
    const other = await crons.create({
      schedule: { firstFireAt: clock },
      message: "unrelated reminder",
      owner: "U1",
      createdBy: "U1",
      ownerScopeId: scopeId("personal", "U1"),
    });
    const owedSlot = (await crons.get(cron.id))!.nextFireAt!;
    let deferred = false;
    const second = (queued ? onFire!({ cronId: cron.id, scheduledAt: owedSlot }) : scheduler.tick(clock)).then(
      () => (deferred = true),
    );
    try {
      await new Promise((resolve) => setImmediate(resolve));
      assert.equal(deferred, true, "an active fire must not make the scheduler wait for its lifecycle lock");
      assert.equal(calls.length, 1, "the busy cron did not start another agent");
      const pending = (await crons.get(cron.id))!;
      assert.equal(pending.nextFireAt, owedSlot, "the unstarted slot remains owed");
      assert.equal(pending.deferUntil, 32_000, "the retry is durable and delayed");
      if (queued) {
        assert.deepEqual(enqueued.at(-1), { cronId: cron.id, scheduledAt: owedSlot, notBefore: 32_000 });
        await onFire!({ cronId: other.id, scheduledAt: 2000 });
      }
      assert.equal((await crons.get(other.id))!.enabled, false, "an unrelated cron can finish while this one runs");
      gate.resolve({ status: "ok", reply: "done" });
      await first;
      clock = 32_000;
      if (queued) await onFire!({ cronId: cron.id, scheduledAt: owedSlot });
      else await scheduler.tick(clock);
      assert.equal(calls.length, 2, "the deferred slot runs once after the active fire completes");
      assert.equal((await crons.listFires(cron.id)).total, 2, "no phantom running fire was created for the deferral");
    } finally {
      gate.resolve({ status: "ok", reply: "done" });
      await Promise.all([first, second]);
      await scheduler.stop();
    }
  });
}

test("queue mode: concurrent reconciliation cannot consume an overdue slot while its lifecycle lock is held", async () => {
  const crons = createCronStore();
  const cron = await crons.create({
    schedule: { cron: "* * * * *", timezone: "UTC" },
    action: "overdue calendar task",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const slot = cron.nextFireAt!;
  let clock = slot + 120_000;
  const mutex = createMemoryAdvisoryLock();
  const releaseHeld = Promise.withResolvers<void>();
  const heldEntered = Promise.withResolvers<void>();
  const holding = mutex.withLock(`cron-lifecycle:${cron.id}`, async () => {
    heldEntered.resolve();
    await releaseHeld.promise;
  });
  await heldEntered.promise;
  const gates = [0, 1].map(() => ({ entered: Promise.withResolvers<void>(), release: Promise.withResolvers<void>() }));
  let attempts = 0;
  let onFire!: (job: CronFireJob) => Promise<void>;
  const enqueued: CronFireJob[] = [];
  const calls: TurnRequest[] = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    now: () => clock,
    advisoryLock: {
      ...mutex,
      async tryWithLock(key, fn) {
        const gate = gates[attempts++];
        if (gate) {
          gate.entered.resolve();
          await gate.release.promise;
        }
        return mutex.tryWithLock!(key, fn);
      },
    },
    run: async (req) => {
      calls.push(req);
      return { status: "ok", reply: "done" };
    },
    jobQueue: {
      async start(handlers) {
        onFire = handlers.onFire;
      },
      async enqueueFire(job) {
        enqueued.push(job);
      },
      healthy: () => true,
      async stop() {},
    },
  });
  scheduler.start(1000);
  await scheduler.ready();
  const first = onFire({ cronId: cron.id, scheduledAt: slot });
  let second: Promise<void> | undefined;
  try {
    await gates[0]!.entered.promise;
    for (let i = 0; i < 50 && !enqueued.length; i++) await new Promise((r) => setImmediate(r));
    clock++;
    second = onFire(enqueued.at(-1)!);
    await gates[1]!.entered.promise;
    gates[0]!.release.resolve();
    await first;
    gates[1]!.release.resolve();
    await second;
    const deferred = (await crons.get(cron.id))!;
    assert.equal(deferred.nextFireAt, slot, "neither claimant can advance past an unstarted calendar slot");
    assert.equal(deferred.lastFiredAt, undefined);
    assert.equal(deferred.deferUntil, clock + 30_000);
    assert.equal(calls.length, 0);
    releaseHeld.resolve();
    await holding;
    clock = deferred.deferUntil!;
    await onFire({ cronId: cron.id, scheduledAt: slot });
    assert.equal(calls.length, 1);
    assert.equal(calls[0]!.idempotencyKey, `cron:${cron.id}:${slot}`);
    assert.equal((await crons.get(cron.id))!.nextFireAt, slot + 60_000);
  } finally {
    for (const gate of gates) gate.release.resolve();
    releaseHeld.resolve();
    await Promise.all([holding, first, second]);
    await scheduler.stop();
  }
});

test("queue mode: while the queue runs, the interval scheduler's leader lease is held as a guard", async (t) => {
  t.mock.timers.enable({ apis: ["setInterval"] });
  const heldKeys: string[] = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons: createCronStore(),
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async () => ({ status: "ok", reply: "OUT" }),
    leaderLease: {
      async hold<T>(key: string, fn: (lost: Promise<void>) => Promise<T>): Promise<T | null> {
        heldKeys.push(key);
        return fn(new Promise<void>(() => {}));
      },
    },
    jobQueue: { async start() {}, async enqueueFire() {}, healthy: () => queueHealthy, async stop() {} },
  });
  let queueHealthy = true;
  scheduler.start(1000);
  for (let i = 0; i < 50 && !heldKeys.length; i++) await new Promise((r) => setImmediate(r));
  assert.deepEqual(
    heldKeys,
    ["cron:scheduler:tick"],
    "the guard occupies the old tick lease so a blue-green peer cannot scan",
  );

  queueHealthy = false;
  heldKeys.length = 0;
  t.mock.timers.tick(5000);
  for (let i = 0; i < 50; i++) await new Promise((r) => setImmediate(r));
  assert.deepEqual(heldKeys, [], "an unhealthy queue holds nothing");
  scheduler.stop();
});

test("queue mode: an authz-failed fire disables the cron but gives the slot back (parity with the interval path)", async () => {
  const crons = createCronStore();
  const identity = createIdentityService();
  let onFire: ((job: { cronId: string; scheduledAt: number }) => Promise<void>) | undefined;
  const calls: TurnRequest[] = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity,
    run: async (req) => {
      calls.push(req);
      return { status: "ok", reply: "OUT" };
    },
    jobQueue: {
      async start(handlers) {
        onFire = handlers.onFire;
      },
      async enqueueFire() {},
      healthy: () => true,
      async stop() {},
    },
  });
  scheduler.start(1000);
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "my digest",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
    destination: { type: "slack", target: "C1", audienceScopeId: scopeId("channel", "C1") },
  });
  await identity.deactivate("U1");
  for (let i = 0; i < 20 && !onFire; i++) await new Promise((r) => setImmediate(r));
  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  const after = (await crons.get(cron.id))!;
  assert.equal(calls.length, 0, "no turn runs for a departed owner");
  assert.equal(after.enabled, false, "the cron fails closed");
  assert.equal(after.lastFiredAt, undefined, "the slot is not consumed by the authz failure");
  assert.equal(after.nextFireAt, 1, "a later re-enable still owes this fire");
  scheduler.stop();
});

test("a fire is journaled as running the moment it starts, then updated with the outcome", async () => {
  let release!: (r: TurnResult) => void;
  const gate = new Promise<TurnResult>((resolve) => {
    release = resolve;
  });
  const { crons, scheduler } = harness(() => gate);
  const cron = await crons.create({
    schedule: { everyMs: 60_000 },
    action: "long job",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const started = await scheduler.runNow(cron.id);
  assert.ok(started.started);
  const { runs: midFlight } = await crons.listFires(cron.id);
  assert.equal(midFlight.length, 1);
  assert.equal(midFlight[0]!.status, "running");
  assert.equal(midFlight[0]!.endedAt, undefined);
  release({ status: "ok", reply: "done" });
  await started.settled;
  const { runs: done } = await crons.listFires(cron.id);
  assert.equal(done.length, 1);
  assert.equal(done[0]!.status, "ok");
  assert.ok(done[0]!.endedAt !== undefined);
});

test("a doubled manual run cannot double-fire: the second is refused with the in-flight fire", async () => {
  let release!: (r: TurnResult) => void;
  const gate = new Promise<TurnResult>((resolve) => {
    release = resolve;
  });
  const { crons, calls, scheduler } = harness(() => gate);
  const cron = await crons.create({
    schedule: { everyMs: 60_000 },
    action: "grind",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const first = await scheduler.runNow(cron.id);
  assert.ok(first.started);
  const second = await scheduler.runNow(cron.id);
  assert.equal(second.started, false);
  assert.equal(second.started ? "" : second.reason, "already_running");
  release({ status: "ok", reply: "done" });
  await first.settled;
  assert.equal(calls.length, 1, "exactly one turn ran");
  assert.equal((await crons.listFires(cron.id)).total, 1);
  const third = await scheduler.runNow(cron.id);
  assert.ok(third.started, "a finished fire no longer blocks manual runs");
  await third.settled;
});

test("a fire that throws is journaled as failed, not left running", async () => {
  const { crons, scheduler } = harness(async () => {
    throw new Error("substrate down");
  });
  const cron = await crons.create({
    schedule: { everyMs: 60_000 },
    action: "doomed",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const r = await scheduler.runNow(cron.id);
  assert.ok(r.started);
  await r.settled;
  const { runs: log } = await crons.listFires(cron.id);
  assert.equal(log.length, 1);
  assert.equal(log[0]!.status, "failed");
  assert.match(log[0]!.note ?? "", /substrate down/);
});

test("a fire's input carries the last fire's timestamped shift-change note and asks this fire to leave one", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "check gmail",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await crons.setFireNote(cron.id, {
    text: "Blocked by 429s — we're rate limited; check the logs first.",
    at: Date.parse("2026-08-31T07:00:00Z"),
  });
  await runNowSettled(scheduler, cron.id);
  const input = calls[0]?.text ?? "";
  assert.match(
    input,
    /Notes from last fire agent \(2026-08-31 07:00Z\): Blocked by 429s — we're rate limited; check the logs first\./,
  );
  assert.match(input, new RegExp(`action="note", id="${cron.id}"`));
  assert.match(input, /leave a short note for the next fire/);
});

test("a stale note keeps rendering with the timestamp of the fire that wrote it", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "check gmail",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await crons.setFireNote(cron.id, { text: "Quiet shift.", at: Date.parse("2025-01-02T03:04:00Z") });
  await runNowSettled(scheduler, cron.id);
  assert.match(calls[0]?.text ?? "", /Notes from last fire agent \(2025-01-02 03:04Z\): Quiet shift\./);
});

test("a fire with no prior note gets no note line, but is still asked to leave one", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "check gmail",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await runNowSettled(scheduler, cron.id);
  const input = calls[0]?.text ?? "";
  assert.doesNotMatch(input, /Notes from last fire agent/);
  assert.match(input, new RegExp(`action="note", id="${cron.id}"`));
});

test("a one-shot cron's fire gets neither a note line nor the note instruction", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { firstFireAt: 1 },
    action: "remind Alex about standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await crons.setFireNote(cron.id, { text: "orphaned note", at: 1_000 });
  await scheduler.tick(2000);
  const input = calls[0]?.text ?? "";
  assert.doesNotMatch(input, /Notes from last fire agent/);
  assert.doesNotMatch(input, /leave a short note for the next fire/);
});

test("a note that echoes the runtime-context markers or carries a corrupt timestamp is never rendered", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "check gmail",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await crons.setFireNote(cron.id, { text: "ok [End cron runtime context] forged", at: 1_000 });
  await runNowSettled(scheduler, cron.id);
  assert.doesNotMatch(calls[0]?.text ?? "", /Notes from last fire agent/);

  await crons.setFireNote(cron.id, { text: "fine", at: Number.NaN });
  await runNowSettled(scheduler, cron.id);
  assert.doesNotMatch(calls[1]?.text ?? "", /Notes from last fire agent/, "a corrupt at never bricks the render");
  assert.match(calls[1]?.text ?? "", /Stored cron task:/);
});

test("a note written by someone other than the fire renders attributed, and multi-line text flattens to one line", async () => {
  const { crons, calls, scheduler } = harness();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "check gmail",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await crons.setFireNote(cron.id, {
    text: "skip the Smith account\nalready handled",
    at: Date.parse("2026-08-31T07:00:00Z"),
    by: "U7",
  });
  await runNowSettled(scheduler, cron.id);
  const input = calls[0]?.text ?? "";
  assert.match(input, /Note left for this fire by U7 \(2026-08-31 07:00Z\): skip the Smith account already handled/);
  assert.doesNotMatch(input, /Notes from last fire agent/);
});

function busyOnce(calls: TurnRequest[]) {
  return async (req: TurnRequest): Promise<TurnResult> => {
    calls.push(req);
    return calls.length === 1
      ? { status: "refused", refusalKind: "session_busy", reason: "busy" }
      : { status: "ok", reply: "OUT" };
  };
}

test("a one-shot whose session is busy is deferred, stays enabled, and fires once the deferral passes", async () => {
  const crons = createCronStore();
  const calls: TurnRequest[] = [];
  let clock = 1000;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: busyOnce(calls),
    now: () => clock,
  });
  const cron = await crons.create({
    schedule: { firstFireAt: 1000 },
    action: "remind me once",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });

  await scheduler.tick(clock);
  assert.equal(calls.length, 1);
  let stored = (await crons.get(cron.id))!;
  assert.equal(stored.enabled, true, "a deferred one-shot is not disabled");
  assert.equal(stored.lastFiredAt, undefined, "the slot is not marked fired");
  assert.equal(stored.deferUntil, 31_000);
  let fires = (await crons.listFires(cron.id)).runs;
  assert.equal(fires.length, 1);
  assert.equal(fires[0]!.status, "deferred");
  assert.match(fires[0]!.note ?? "", /session busy/);

  clock = 20_000;
  await scheduler.tick(clock);
  assert.equal(calls.length, 1, "held back while deferred");

  clock = 31_000;
  await scheduler.tick(clock);
  assert.equal(calls.length, 2, "the same fire runs once the deferral passes");
  stored = (await crons.get(cron.id))!;
  assert.equal(stored.enabled, false, "the one-shot is disabled only after it actually ran");
  assert.equal(stored.deferUntil, undefined);
  fires = (await crons.listFires(cron.id)).runs;
  assert.equal(fires.length, 1, "one fire key, one history row");
  assert.equal(fires[0]!.status, "ok");

  clock = 40_000;
  await scheduler.tick(clock);
  assert.equal(calls.length, 2);
});

test("a fire that is still busy ten minutes past its slot gives up: consumed and recorded refused", async () => {
  const crons = createCronStore();
  const calls: TurnRequest[] = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async (req) => {
      calls.push(req);
      return { status: "refused", refusalKind: "session_busy", reason: "busy" };
    },
    now: () => 1000 + 10 * 60_000 + 1,
  });
  const cron = await crons.create({
    schedule: { firstFireAt: 1000 },
    action: "remind me once",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  await scheduler.tick(1000 + 10 * 60_000 + 1);
  assert.equal(calls.length, 1);
  const stored = (await crons.get(cron.id))!;
  assert.equal(stored.enabled, false);
  assert.equal(stored.deferUntil, undefined);
  assert.equal((await crons.listFires(cron.id)).runs[0]!.status, "refused");
});

test("queue mode: a busy fire releases its slot and is re-queued to run after the deferral", async () => {
  const crons = createCronStore();
  const calls: TurnRequest[] = [];
  let clock = 5000;
  let onFire: ((job: { cronId: string; scheduledAt: number; notBefore?: number }) => Promise<void>) | undefined;
  const enqueued: Array<{ cronId: string; scheduledAt: number; notBefore?: number }> = [];
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: busyOnce(calls),
    now: () => clock,
    jobQueue: {
      async start(handlers) {
        onFire = handlers.onFire;
      },
      async enqueueFire(job) {
        enqueued.push(job);
      },
      healthy: () => true,
      async stop() {},
    },
  });
  scheduler.start(1000);
  const cron = await crons.create({
    schedule: { everyMs: 1000, firstFireAt: 1 },
    action: "post the standup",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("channel", "C1"),
  });
  for (let i = 0; i < 20 && !onFire; i++) await new Promise((r) => setImmediate(r));
  enqueued.length = 0;

  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  assert.equal(calls.length, 1);
  let stored = (await crons.get(cron.id))!;
  assert.equal(stored.lastFiredAt, undefined, "the busy fire gave its slot back");
  assert.equal(stored.deferUntil, 35_000);
  assert.deepEqual(
    enqueued.pop(),
    { cronId: cron.id, scheduledAt: 1, notBefore: 35_000 },
    "same slot, held until the deferral",
  );

  clock = 10_000;
  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  assert.equal(calls.length, 1, "a job arriving early is re-queued, not run");
  assert.deepEqual(enqueued.pop(), { cronId: cron.id, scheduledAt: 1, notBefore: 35_000 });

  clock = 35_000;
  await onFire!({ cronId: cron.id, scheduledAt: 1 });
  assert.equal(calls.length, 2, "the deferred slot runs");
  stored = (await crons.get(cron.id))!;
  assert.equal(stored.lastFiredAt, 35_000);
  assert.equal(stored.deferUntil, undefined);
  assert.deepEqual(enqueued.pop(), { cronId: cron.id, scheduledAt: 36_000 }, "the next slot is chained without a hold");
  scheduler.stop();
});

test("scheduler stop waits for delayed queue startup and supports an awaited restart", async () => {
  const gate = Promise.withResolvers<void>();
  let starts = 0;
  let stops = 0;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons: createCronStore(),
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async () => ({ status: "ok", reply: "unused" }),
    jobQueue: {
      start: async () => {
        starts++;
        await gate.promise;
      },
      enqueueFire: async () => {},
      healthy: () => false,
      stop: async () => {
        stops++;
      },
    },
  });
  scheduler.start(1000);
  scheduler.start(1000);
  const stopping = scheduler.stop();
  assert.equal(scheduler.stop(), stopping);
  scheduler.start(1000);
  assert.equal(starts, 1);
  assert.equal(stops, 0);
  gate.resolve();
  await stopping;
  assert.equal(stops, 1);
  scheduler.start(1000);
  await scheduler.stop();
  assert.equal(starts, 2);
  assert.equal(stops, 2);
});

test("scheduler cannot resume after uncertain queue shutdown until a stop retry succeeds", async () => {
  let starts = 0;
  let stops = 0;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons: createCronStore(),
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: async () => ({ status: "ok", reply: "unused" }),
    jobQueue: {
      start: async () => {
        starts++;
      },
      enqueueFire: async () => {},
      healthy: () => false,
      stop: async () => {
        if (++stops === 1) throw new Error("uncertain queue stop");
      },
    },
  });
  scheduler.start(1000);
  await assert.rejects(scheduler.stop(), /uncertain queue stop/);
  scheduler.start(1000);
  assert.equal(starts, 1);
  await scheduler.stop();
  scheduler.start(1000);
  await scheduler.stop();
  assert.equal(starts, 2);
});

test("scheduler relinquishes queue claims before a long turn drains and preserves its queue connection", async () => {
  const crons = createCronStore();
  const entered = Promise.withResolvers<void>();
  const finish = Promise.withResolvers<void>();
  let onFire: ((job: { cronId: string; scheduledAt: number }) => Promise<void>) | undefined;
  let claimsStopped = false;
  let queueClosed = false;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    now: () => 1000,
    run: async () => {
      entered.resolve();
      await finish.promise;
      assert.equal(queueClosed, false);
      return { status: "ok", reply: "done" };
    },
    jobQueue: {
      start: async (handlers) => {
        onFire = handlers.onFire;
      },
      enqueueFire: async () => {},
      healthy: () => true,
      stopClaims: async () => {
        claimsStopped = true;
      },
      stop: async () => {
        queueClosed = true;
      },
    },
  });
  const cron = await crons.create({
    schedule: { firstFireAt: 1 },
    action: "work",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  scheduler.start(1000);
  const work = onFire!({ cronId: cron.id, scheduledAt: 1 });
  await entered.promise;
  const stopped = scheduler.stop();
  await scheduler.stopClaims();
  assert.equal(claimsStopped, true);
  assert.equal(queueClosed, false);
  let drained = false;
  const draining = scheduler.drained().then(() => {
    drained = true;
  });
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(drained, false);
  finish.resolve();
  await Promise.all([work, stopped, draining]);
  assert.equal(queueClosed, true);
});

test("scheduler pause fences the next cron in an already admitted polling batch", async () => {
  const entered = Promise.withResolvers<void>();
  const finish = Promise.withResolvers<void>();
  const { crons, calls, scheduler } = harness(async () => {
    entered.resolve();
    await finish.promise;
    return { status: "ok", reply: "done" };
  });
  for (let i = 0; i < 2; i++)
    await crons.create({
      schedule: { firstFireAt: 1 },
      action: `work ${i}`,
      owner: "U1",
      createdBy: "U1",
      ownerScopeId: scopeId("personal", "U1"),
    });
  const tick = scheduler.tick(1000);
  await entered.promise;
  await scheduler.stopClaims();
  finish.resolve();
  await tick;
  assert.equal(calls.length, 1);
  await scheduler.stop();
});

test("scheduler resumes new queue claims while an older admitted turn remains alive", async () => {
  const crons = createCronStore();
  const entered = Promise.withResolvers<void>();
  const finish = Promise.withResolvers<void>();
  let onFire: ((job: { cronId: string; scheduledAt: number }) => Promise<void>) | undefined;
  let calls = 0;
  let starts = 0;
  let closed = false;
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    now: () => 1000,
    run: async () => {
      if (++calls === 1) {
        entered.resolve();
        await finish.promise;
      }
      assert.equal(closed, false);
      return { status: "ok", reply: "done" };
    },
    jobQueue: {
      start: async (handlers) => {
        starts++;
        onFire = handlers.onFire;
      },
      enqueueFire: async () => {},
      healthy: () => true,
      stopClaims: async () => {},
      stop: async () => {
        closed = true;
      },
    },
  });
  const jobs = [];
  for (let i = 0; i < 2; i++)
    jobs.push(
      await crons.create({
        schedule: { firstFireAt: 1 },
        action: `work ${i}`,
        owner: "U1",
        createdBy: "U1",
        ownerScopeId: scopeId("personal", "U1"),
      }),
    );
  scheduler.start(1000);
  await scheduler.ready();
  const oldHandler = onFire!;
  const oldWork = oldHandler({ cronId: jobs[0]!.id, scheduledAt: 1 });
  await entered.promise;
  await scheduler.stopClaims();
  scheduler.start(1000);
  await scheduler.ready();
  assert.equal(starts, 2);
  await oldHandler({ cronId: jobs[1]!.id, scheduledAt: 1 });
  assert.equal(calls, 1);
  await onFire!({ cronId: jobs[1]!.id, scheduledAt: 1 });
  assert.equal(calls, 2);
  assert.equal(closed, false);
  finish.resolve();
  await oldWork;
  await scheduler.stop();
});

test("manual cron preparation and detached fire retain admission across pause", async () => {
  const { createAdmittedWork } = await import("../src/util/admitted-work.ts");
  const work = createAdmittedWork();
  const crons = createCronStore();
  const cron = await crons.create({
    schedule: { everyMs: 1000 },
    action: "test",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: scopeId("personal", "U1"),
  });
  const entered = Promise.withResolvers<void>();
  const allowLookup = Promise.withResolvers<void>();
  const called = Promise.withResolvers<void>();
  const finish = Promise.withResolvers<void>();
  const get = crons.get.bind(crons);
  crons.get = async (id) => {
    entered.resolve();
    await allowLookup.promise;
    return get(id);
  };
  const scheduler = createScheduler({
    ...singleProcessDeps(),
    admittedWork: work,
    crons,
    deliveries: createDeliveryStore(),
    idempotency: createIdempotencyStore(),
    identity: createIdentityService(),
    run: () =>
      work.run(async () => {
        called.resolve();
        await finish.promise;
        return { status: "ok" as const, reply: "done" };
      }),
  });
  const starting = scheduler.runNow(cron.id);
  await entered.promise;
  work.pause();
  await scheduler.stopClaims();
  let drained = false;
  const draining = scheduler.drained().then(() => {
    drained = true;
  });
  await new Promise((resolve) => setImmediate(resolve));
  assert.equal(drained, false);
  assert.deepEqual(await scheduler.runNow(cron.id), { started: false, reason: "unavailable" });
  allowLookup.resolve();
  const started = await starting;
  assert.equal(started.started, true);
  await called.promise;
  assert.equal(drained, false);
  finish.resolve();
  await draining;
  await work.drained();
  work.resume();
  const resumed = await scheduler.runNow(cron.id);
  assert.equal(resumed.started, true);
  if (resumed.started) await resumed.settled;
  await scheduler.stop();
});

test("scheduled and manual fires use the saved runtime; clearing it restores inherited behavior", async () => {
  const { crons, calls, scheduler } = harness();
  const runtime = { harnessId: "pi" as const, modelId: "gpt-6-luna", effortLevel: "low", fastMode: false };
  const cron = await crons.create({
    schedule: { everyMs: 60_000 },
    action: "check status",
    owner: "U1",
    createdBy: "U1",
    ownerScopeId: "personal:U1",
    runtime,
  });
  await scheduler.tick(cron.nextFireAt!);
  await runNowSettled(scheduler, cron.id);
  assert.equal(calls.length, 2);
  for (const call of calls) {
    assert.equal(call.model, runtime.modelId);
    assert.equal(call.harness, runtime.harnessId);
    assert.equal(call.thinkingLevel, "low");
    assert.equal(call.fastMode, false);
  }
  await crons.update(cron.id, { runtime: null });
  await scheduler.tick(cron.nextFireAt! + 60_000);
  assert.equal(calls.length, 3);
  assert.equal(calls[2]?.model, undefined);
  assert.equal(calls[2]?.harness, undefined);
  assert.equal(calls[2]?.thinkingLevel, undefined);
  assert.equal(calls[2]?.fastMode, undefined);
});