Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Isolate cron fire failures so one failing cron cannot starve later due crons (#382)

Regan Bell164cc397ce7b
diff --git a/src/cron/cron-store.ts b/src/cron/cron-store.ts
index c50c00a47..8a75502df 100644
--- a/src/cron/cron-store.ts
+++ b/src/cron/cron-store.ts
@@ -43,6 +43,7 @@ export interface CronStore {
setRecipientConsent(id: string, recipientConsent: RecipientConsent): Promise<void>;
recordFire(id: string, entry: CronFireLogEntry): Promise<void>;
markFired(id: string, at: number, scheduledAt?: number): Promise<void>;
+ markAttempted(id: string, at: number): Promise<void>;
claimSlot(id: string, scheduledAt: number, at: number): Promise<boolean>;
unclaimSlot(id: string, scheduledAt: number, at: number, priorLastFiredAt: number | undefined): Promise<void>;
due(now: number): Promise<Array<Cron & { scheduledAt: number }>>;
@@ -175,6 +176,9 @@ export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cron
if (!cron || cron.lastFiredAt !== at) return;
await backing.merge(id, { lastFiredAt: priorLastFiredAt, nextFireAt: scheduledAt });
},
+ async markAttempted(id, at) {
+ await backing.merge(id, { lastAttemptAt: at });
+ },
async due(now) {
const due: Array<Cron & { scheduledAt: number }> = [];
for (const c of await backing.all()) {
diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts
index 066d79114..333bbbc4a 100644
--- a/src/cron/scheduler.ts
+++ b/src/cron/scheduler.ts
@@ -174,13 +174,28 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
const fireDue = async (t: number): Promise<void> => {
const due = await deps.crons.due(t);
- const batch = due.slice(0, maxFiresPerTick);
- if (due.length > batch.length) {
+ let batch = due;
+ if (due.length > maxFiresPerTick) {
+ const ordered = [...due].sort((a, b) => (a.lastAttemptAt ?? 0) - (b.lastAttemptAt ?? 0));
+ batch = [];
+ for (const cron of ordered) {
+ if (batch.length >= maxFiresPerTick) break;
+ try {
+ await deps.crons.markAttempted(cron.id, t);
+ batch.push(cron);
+ } catch (e) {
+ console.error("[scheduler] attempt mark failed, holding this cron back:", errMessage(e));
+ }
+ }
console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`);
}
for (const cron of batch) {
- const { authzFailed } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);
- if (!authzFailed) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
+ try {
+ const { authzFailed } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt);
+ if (!authzFailed) await deps.crons.markFired(cron.id, t, cron.scheduledAt);
+ } catch (e) {
+ console.error("[scheduler] fire failed:", errMessage(e));
+ }
}
};
diff --git a/src/types.ts b/src/types.ts
index ab5319b37..ae5968b2b 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -217,6 +217,7 @@ export interface CronFireLogEntry {
export interface Cron extends TriggerBase {
schedule: CronSchedule;
nextFireAt?: number;
+ lastAttemptAt?: number;
title?: string;
archived?: boolean;
action?: string;
diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.ts
index a6b84e068..90bc01294 100644
--- a/test/cron-scheduler.test.ts
+++ b/test/cron-scheduler.test.ts
@@ -10,6 +10,8 @@ import type { LeaderLease } from "../src/persistence/leader-lease.ts";
import { scopeId, type TurnRequest, type TurnResult } from "../src/types.ts";
import { isPollSurface, isSilentPollReply } from "../src/triggers/run-trigger.ts";
import { createDirectoryStore, type DirectoryStore } from "../src/directory/directory-store.ts";
+import { createMemoryMap } from "../src/persistence/durable-map.ts";
+import type { Cron } from "../src/types.ts";
function fakeLease(isLeader: () => boolean): LeaderLease {
return {
@@ -22,6 +24,7 @@ function fakeLease(isLeader: () => boolean): LeaderLease {
function harness(
reply: string | ((req: TurnRequest) => Promise<TurnResult>) = "CRON-OUTPUT-XYZ",
directory?: DirectoryStore,
+ maxFiresPerTick?: number,
) {
const crons = createCronStore();
const deliveries = createDeliveryStore();
@@ -42,6 +45,7 @@ function harness(
identity,
run,
...(directory ? { directory } : {}),
+ ...(maxFiresPerTick !== undefined ? { maxFiresPerTick } : {}),
});
return { crons, deliveries, calls, scheduler, identity };
}
@@ -481,6 +485,191 @@ test("the default (no-op) lease ticks exactly as before — memory-mode behavior
assert.equal(calls.length, 1, "without a lease, the tick fires due crons as before");
});
+test("a failing interval cron does not starve later due crons across ticks", async () => {
+ const { crons, calls, scheduler } = harness(async (req) => {
+ if (req.text.includes("fail first")) throw new Error("cron failed");
+ return { status: "ok", reply: "OUT" };
+ });
+ const failing = await crons.create({
+ schedule: { everyMs: 1000, firstFireAt: 1 },
+ action: "fail first",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+ const succeeding = await crons.create({
+ schedule: { everyMs: 1000, firstFireAt: 1 },
+ action: "succeed second",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+
+ await scheduler.tick(2000);
+ await scheduler.tick(3500);
+
+ assert.deepEqual(
+ calls.map((call) => call.idempotencyKey),
+ [`cron:${failing.id}:1`, `cron:${succeeding.id}:1`, `cron:${failing.id}:1`, `cron:${succeeding.id}:3000`],
+ );
+});
+
+test("a capped batch of persistent failures rotates — later due crons still get their turn", async () => {
+ const { crons, calls, scheduler } = harness(
+ async (req) => {
+ if (req.text.includes("always fails")) throw new Error("cron failed");
+ return { status: "ok", reply: "OUT" };
+ },
+ undefined,
+ 2,
+ );
+ for (let i = 0; i < 3; i++) {
+ await crons.create({
+ schedule: { everyMs: 60_000, firstFireAt: 1 },
+ action: `always fails ${i}`,
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+ }
+ const healthy = await crons.create({
+ schedule: { everyMs: 60_000, firstFireAt: 1 },
+ action: "healthy last in line",
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ });
+
+ await scheduler.tick(2000);
+ await scheduler.tick(3000);
+
+ assert.ok(
+ calls.some((call) => call.idempotencyKey === `cron:${healthy.id}:1`),
+ "the healthy cron behind a full batch of failures fires within two ticks",
+ );
+});
+
+test("capped rotation is driven by durable attempt order, not tick arrival times", async () => {
+ const { crons, calls, scheduler } = harness("OUT", undefined, 2);
+ const created: Cron[] = [];
+ for (let i = 0; i < 4; i++) {
+ created.push(
+ await crons.create({
+ schedule: { everyMs: 600_000, firstFireAt: 1 },
+ action: `cron ${i}`,
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ }),
+ );
+ }
+
+ await scheduler.tick(2000);
+ await scheduler.tick(4000);
+
+ for (const cron of created) {
+ assert.ok(
+ calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
+ "every due cron fires within ceil(due/cap) ticks even when tick timestamps skip buckets",
+ );
+ }
+});
+
+test("a cron whose attempt marker cannot persist is held back and cannot starve the rest", async () => {
+ const backing = createMemoryMap<Cron>();
+ const crons = createCronStore(backing);
+ const deliveries = createDeliveryStore();
+ const calls: TurnRequest[] = [];
+ const run = async (req: TurnRequest): Promise<TurnResult> => {
+ calls.push(req);
+ return { status: "ok", reply: "OUT" };
+ };
+ const created: Cron[] = [];
+ for (let i = 0; i < 5; i++) {
+ created.push(
+ await crons.create({
+ schedule: { everyMs: 600_000, firstFireAt: 1 },
+ action: `cron ${i}`,
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ }),
+ );
+ }
+ const broken = created[0]!;
+ const flaky: typeof crons = {
+ ...crons,
+ async markAttempted(id, at) {
+ if (id === broken.id) throw new Error("attempt marker write failed");
+ return crons.markAttempted(id, at);
+ },
+ };
+ const scheduler = createScheduler({
+ crons: flaky,
+ deliveries,
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ run,
+ maxFiresPerTick: 2,
+ });
+
+ await scheduler.tick(2000);
+ await scheduler.tick(4000);
+
+ assert.ok(
+ !calls.some((call) => call.idempotencyKey === `cron:${broken.id}:1`),
+ "a capped batch never fires a cron whose attempt failed to persist",
+ );
+ for (const cron of created.slice(1)) {
+ assert.ok(
+ calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
+ "the crons behind the broken marker still fire within two capped ticks",
+ );
+ }
+});
+
+test("capped rotation survives a scheduler restart mid-cycle", async () => {
+ const backing = createMemoryMap<Cron>();
+ const crons = createCronStore(backing);
+ const deliveries = createDeliveryStore();
+ const calls: TurnRequest[] = [];
+ const run = async (req: TurnRequest): Promise<TurnResult> => {
+ calls.push(req);
+ return { status: "ok", reply: "OUT" };
+ };
+ const mk = () =>
+ createScheduler({
+ crons,
+ deliveries,
+ idempotency: createIdempotencyStore(),
+ identity: createIdentityService(),
+ run,
+ maxFiresPerTick: 2,
+ });
+ const created: Cron[] = [];
+ for (let i = 0; i < 4; i++) {
+ created.push(
+ await crons.create({
+ schedule: { everyMs: 600_000, firstFireAt: 1 },
+ action: `cron ${i}`,
+ owner: "U1",
+ createdBy: "U1",
+ ownerScopeId: scopeId("personal", "U1"),
+ }),
+ );
+ }
+
+ await mk().tick(2000);
+ await mk().tick(2000);
+
+ for (const cron of created) {
+ assert.ok(
+ calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`),
+ "a fresh scheduler instance resumes the rotation from durable state instead of restarting it",
+ );
+ }
+});
+
test("runNow fires even when the current schedule slot already did (manual re-run)", async () => {
const { crons, calls, scheduler } = harness();
const cron = await crons.create({
@@ -724,7 +913,7 @@ test("a scopeFloor cron whose only members are non-internal fails closed", async
assert.equal((await crons.get(cron.id))?.enabled, false);
});
-test("a failing background tick is logged, not swallowed", async (t) => {
+test("a failing cron fire is logged, not swallowed", async (t) => {
t.mock.timers.enable({ apis: ["setInterval"] });
const logged: string[] = [];
t.mock.method(console, "error", (...args: unknown[]) => {
@@ -749,13 +938,13 @@ test("a failing background tick is logged, not swallowed", async (t) => {
});
scheduler.start(1000);
t.mock.timers.tick(1000);
- for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] tick failed")); i++) {
+ for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] fire failed")); i++) {
await new Promise((r) => setImmediate(r));
}
scheduler.stop();
assert.ok(
- logged.some((l) => l.includes("[scheduler] tick failed") && l.includes("boom")),
- "the tick error must reach the log",
+ logged.some((l) => l.includes("[scheduler] fire failed") && l.includes("boom")),
+ "the fire error must reach the log",
);
const after = await crons.get(cron.id);
assert.equal(after?.fireLog?.length, 1);