Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Stop sending a cron tick job every second (#2007)

Sina Matianb528b8ff0794
diff --git a/src/cron/job-queue.ts b/src/cron/job-queue.ts
index 78b674574..6e7278d43 100644
--- a/src/cron/job-queue.ts
+++ b/src/cron/job-queue.ts
@@ -1,6 +1,5 @@
import { PgBoss } from "pg-boss";
import { createPgPool, type PgPool } from "../persistence/pg-pool.ts";
-import { createSweeper, type Sweeper } from "../util/sweeper.ts";
import { errMessage } from "../util/errors.ts";
export interface CronFireJob {
@@ -11,22 +10,19 @@ export interface CronFireJob {
interface CronQueueHandlers {
onFire(job: CronFireJob): Promise<void>;
- onTick(): Promise<void>;
}
export interface CronJobQueue {
- start(handlers: CronQueueHandlers, tickIntervalMs: number): Promise<void>;
+ start(handlers: CronQueueHandlers): Promise<void>;
enqueueFire(job: CronFireJob): Promise<void>;
healthy(): boolean;
stopClaims?(): Promise<void>;
stop(): Promise<void>;
}
-const HEALTHY_SEND_MAX_AGE_MS = 30_000;
+const UNHEALTHY_AFTER_ERROR_MS = 30_000;
const FIRE_QUEUE = "cron-fire";
-const TICK_QUEUE = "cron-tick";
-const CRON_TICK_SECONDS = 5;
export function createPgBossCronQueue(
databaseUrl: string,
@@ -48,13 +44,15 @@ export function createPgBossCronQueue(
pg = null;
await previous?.close();
}
- boss.on("error", (e) => console.error("[cron-queue] pg-boss error:", errMessage(e)));
- let ticker: Sweeper | null = null;
let started = false;
let initialized = false;
- let lastSendOkAt = 0;
+ let lastErrorAt = 0;
+ boss.on("error", (e) => {
+ lastErrorAt = Date.now();
+ console.error("[cron-queue] pg-boss error:", errMessage(e));
+ });
return {
- async start(handlers, tickIntervalMs) {
+ async start(handlers) {
if (started) return;
pg ??= createPgPool(databaseUrl, []);
try {
@@ -63,7 +61,6 @@ export function createPgBossCronQueue(
initialized = true;
}
await boss.createQueue(FIRE_QUEUE, { policy: "short", notify: true });
- await boss.createQueue(TICK_QUEUE, { policy: "short", notify: true });
const localConcurrency = Math.min(32, Math.max(1, Math.trunc(fireConcurrency)));
await boss.work<CronFireJob>(
FIRE_QUEUE,
@@ -72,13 +69,9 @@ export function createPgBossCronQueue(
for (const job of jobs) await handlers.onFire(job.data);
},
);
- await boss.work(TICK_QUEUE, { pollingIntervalSeconds: 1 }, () => handlers.onTick());
} catch (e) {
if (initialized) {
- await Promise.all([
- boss.offWork(FIRE_QUEUE, { wait: false }),
- boss.offWork(TICK_QUEUE, { wait: false }),
- ]).catch(() => {});
+ await boss.offWork(FIRE_QUEUE, { wait: false }).catch(() => {});
} else {
await boss.stop({ close: true, graceful: false }).catch(() => {});
await closePool();
@@ -86,38 +79,30 @@ export function createPgBossCronQueue(
throw e;
}
started = true;
- lastSendOkAt = Date.now();
- ticker = createSweeper(
- () =>
- boss
- .send(TICK_QUEUE, {}, { singletonSeconds: CRON_TICK_SECONDS, retryLimit: 0, expireInSeconds: 60 })
- .then(() => {
- lastSendOkAt = Date.now();
- }),
- tickIntervalMs,
- { label: "cron-queue ticker", immediate: true },
- );
- ticker.start();
+ lastErrorAt = 0;
},
async enqueueFire(job) {
if (!started) return;
- await boss.send(FIRE_QUEUE, job, {
- startAfter: new Date(Math.max(job.scheduledAt, job.notBefore ?? 0)),
- singletonKey: `${job.cronId}:${job.scheduledAt}`,
- retryLimit: 0,
- });
+ await boss
+ .send(FIRE_QUEUE, job, {
+ startAfter: new Date(Math.max(job.scheduledAt, job.notBefore ?? 0)),
+ singletonKey: `${job.cronId}:${job.scheduledAt}`,
+ retryLimit: 0,
+ })
+ .catch((e: unknown) => {
+ lastErrorAt = Date.now();
+ throw e;
+ });
},
healthy() {
- return started && Date.now() - lastSendOkAt < HEALTHY_SEND_MAX_AGE_MS;
+ return started && Date.now() - lastErrorAt >= UNHEALTHY_AFTER_ERROR_MS;
},
async stopClaims() {
started = false;
- void ticker?.stop();
- await Promise.all([boss.offWork(FIRE_QUEUE, { wait: false }), boss.offWork(TICK_QUEUE, { wait: false })]);
+ await boss.offWork(FIRE_QUEUE, { wait: false });
},
async stop() {
started = false;
- await ticker?.stop();
try {
await boss.stop({ close: true, graceful: false });
} finally {
diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts
index ef8f4dc8c..24d1ba1b1 100644
--- a/src/cron/scheduler.ts
+++ b/src/cron/scheduler.ts
@@ -32,6 +32,8 @@ const STRANDED_SWEEP_INTERVAL_MS = 10 * 60_000;
const FIRE_GC_INTERVAL_MS = 6 * 60 * 60_000;
const BUSY_DEFER_MS = 30_000;
const BUSY_DEFER_MAX_LATE_MS = 10 * 60_000;
+const MAINTENANCE_INTERVAL_MS = 5000;
+const REPAIR_INTERVAL_MS = 60_000;
type FireResult = { authzFailed: boolean; deferred?: boolean };
@@ -490,6 +492,9 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
} catch (e) {
reportFailure("scheduler: reconcile", e);
}
+ }
+
+ async function maintain(): Promise<void> {
await sweepStranded(now());
await gcFires(now());
await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));
@@ -502,8 +507,15 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
? leaderLease.hold(TICK_LEASE_KEY, async (lost) => {
let lockLost = false;
void lost.then(() => (lockLost = true));
- while (!stopped && !lockLost && deps.jobQueue!.healthy())
- await Promise.race([sleep(1000, { unref: true }), lost, stopSignal.promise]);
+ let repairAt = 0;
+ while (!stopped && !lockLost && deps.jobQueue!.healthy()) {
+ if (Date.now() >= repairAt) {
+ repairAt = Date.now() + REPAIR_INTERVAL_MS;
+ await runQueueTask(reconcile);
+ }
+ await runQueueTask(maintain);
+ await Promise.race([sleep(MAINTENANCE_INTERVAL_MS, { unref: true }), lost, stopSignal.promise]);
+ }
})
: undefined,
1000,
@@ -586,13 +598,9 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {
}
const observed = epoch;
starting = deps.jobQueue
- .start(
- {
- onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),
- onTick: () => (observed === epoch ? runQueueTask(reconcile) : Promise.resolve()),
- },
- intervalMs,
- )
+ .start({
+ onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),
+ })
.then(
() => {
if (!stopped) leaseGuard.start();
diff --git a/test/cron-queue-lifecycle.test.ts b/test/cron-queue-lifecycle.test.ts
index c20710865..3f0b2eba4 100644
--- a/test/cron-queue-lifecycle.test.ts
+++ b/test/cron-queue-lifecycle.test.ts
@@ -6,6 +6,7 @@ const pools: { closed: boolean }[] = [];
let adapter: Adapter;
let failStart = false;
let failWork = false;
+let emitError: (e: Error) => void = () => {};
mock.module("../src/persistence/pg-pool.ts", {
namedExports: {
@@ -31,7 +32,9 @@ mock.module("pg-boss", {
constructor(options: { db: Adapter }) {
adapter = options.db;
}
- on() {}
+ on(_event: string, listener: (e: Error) => void) {
+ emitError = listener;
+ }
async start() {
await adapter.executeSql("SELECT 1");
if (failStart) throw new Error("startup failed");
@@ -41,26 +44,23 @@ mock.module("pg-boss", {
if (failWork) throw new Error("polling failed");
}
async offWork() {}
- async send() {
- await adapter.executeSql("SELECT 1");
- }
async stop() {}
},
},
});
const { createPgBossCronQueue } = await import("../src/cron/job-queue.ts");
-const handlers = { onFire: async () => {}, onTick: async () => {} };
+const handlers = { onFire: async () => {} };
test("late worker SQL after stop cannot reopen the queue database", async () => {
const queue = createPgBossCronQueue("postgres://unused");
- await queue.start(handlers, 60_000);
+ await queue.start(handlers);
await queue.stop();
const count = pools.length;
await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);
assert.equal(pools.length, count);
assert.ok(pools.at(-1)?.closed);
- await queue.start(handlers, 60_000);
+ await queue.start(handlers);
try {
assert.equal(pools.length, count + 1);
await adapter.executeSql("SELECT 1");
@@ -74,14 +74,14 @@ test("startup failure closes the database and permits a fresh start", async () =
const queue = createPgBossCronQueue("postgres://unused");
failStart = true;
try {
- await assert.rejects(queue.start(handlers, 60_000), /startup failed/);
+ await assert.rejects(queue.start(handlers), /startup failed/);
assert.ok(pools.at(-1)?.closed);
assert.equal(queue.healthy(), false);
await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);
} finally {
failStart = false;
}
- await queue.start(handlers, 60_000);
+ await queue.start(handlers);
try {
assert.equal(pools.at(-1)?.closed, false);
assert.ok(queue.healthy());
@@ -92,12 +92,12 @@ test("startup failure closes the database and permits a fresh start", async () =
test("resuming polling failure preserves the database for admitted callbacks", async () => {
const queue = createPgBossCronQueue("postgres://unused");
- await queue.start(handlers, 60_000);
+ await queue.start(handlers);
await queue.stopClaims!();
const count = pools.length;
failWork = true;
try {
- await assert.rejects(queue.start(handlers, 60_000), /polling failed/);
+ await assert.rejects(queue.start(handlers), /polling failed/);
assert.equal(queue.healthy(), false);
assert.equal(pools.at(-1)?.closed, false);
await adapter.executeSql("SELECT 1");
@@ -105,7 +105,7 @@ test("resuming polling failure preserves the database for admitted callbacks", a
failWork = false;
}
try {
- await queue.start(handlers, 60_000);
+ await queue.start(handlers);
assert.equal(pools.length, count);
assert.ok(queue.healthy());
} finally {
@@ -113,3 +113,15 @@ test("resuming polling failure preserves the database for admitted callbacks", a
}
assert.ok(pools.at(-1)?.closed);
});
+
+test("a pg-boss error marks the queue unhealthy without any synthetic send", async () => {
+ const queue = createPgBossCronQueue("postgres://unused");
+ await queue.start(handlers);
+ try {
+ assert.ok(queue.healthy());
+ emitError(new Error("fetch failed"));
+ assert.equal(queue.healthy(), false);
+ } finally {
+ await queue.stop();
+ }
+});
diff --git a/test/cron-queue.test.ts b/test/cron-queue.test.ts
index 95a78c8f6..8a337d1f6 100644
--- a/test/cron-queue.test.ts
+++ b/test/cron-queue.test.ts
@@ -69,6 +69,7 @@ test(
createdBy: "U1",
ownerScopeId: scopeId("personal", "U1"),
});
+ a.scheduler.notifyChanged(cron.id);
await until(() => calls.length >= 1, 30_000);
assert.equal(calls.length, 1, "the due slot starts exactly one turn");
await new Promise((r) => setTimeout(r, 16_000));
@@ -100,6 +101,7 @@ test(
createdBy: "U2",
ownerScopeId: scopeId("personal", "U2"),
});
+ a.scheduler.notifyChanged(cron.id);
const fires = () => calls.filter((c) => c.idempotencyKey?.startsWith(`cron:${cron.id}:`)).length;
await until(() => fires() >= 3, 45_000);
assert.ok(fires() >= 3, "a recurring cron chains fire jobs across instances");
@@ -107,6 +109,7 @@ test(
assert.equal(new Set(keys).size, keys.length, "no slot fires twice");
await b.crons.update(cron.id, { schedule: { everyMs: 60_000 } });
+ b.scheduler.notifyChanged(cron.id);
const at = fires();
await new Promise((r) => setTimeout(r, 6_000));
assert.equal(fires(), at, "a rescheduled cron's stale slot jobs do not fire");
@@ -117,3 +120,34 @@ test(
}
},
);
+
+test(
+ "pg-boss queue: a cron persisted without its fire job is enqueued once by startup reconciliation",
+ { skip, timeout: 120_000 },
+ async () => {
+ const calls: TurnRequest[] = [];
+ const fires = createMemoryCronFireStore();
+ const a = instance(calls, 0, fires);
+ const b = instance(calls, 0, fires);
+ const cron = await a.crons.create({
+ schedule: { firstFireAt: Date.now() - 1000 },
+ action: "survive the crash",
+ owner: "U3",
+ createdBy: "U3",
+ ownerScopeId: scopeId("personal", "U3"),
+ });
+ a.scheduler.start(1000);
+ b.scheduler.start(1000);
+ try {
+ const fired = () => calls.filter((c) => c.idempotencyKey?.startsWith(`cron:${cron.id}:`)).length;
+ await until(() => fired() >= 1, 30_000);
+ await new Promise((r) => setTimeout(r, 3_000));
+ assert.equal(fired(), 1, "the orphaned slot fires exactly once across both instances");
+ assert.equal((await a.crons.listFires(cron.id)).total, 1);
+ } finally {
+ a.scheduler.stop();
+ b.scheduler.stop();
+ await new Promise((r) => setTimeout(r, 500));
+ }
+ },
+);
diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.ts
index ea13b2521..1481400b8 100644
--- a/test/cron-scheduler.test.ts
+++ b/test/cron-scheduler.test.ts
@@ -1147,7 +1147,6 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot while
const gates = [0, 1].map(() => ({ entered: Promise.withResolvers<void>(), release: Promise.withResolvers<void>() }));
let attempts = 0;
let onFire!: (job: CronFireJob) => Promise<void>;
- let onTick!: () => Promise<void>;
const enqueued: CronFireJob[] = [];
const calls: TurnRequest[] = [];
const scheduler = createScheduler({
@@ -1174,7 +1173,6 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot while
jobQueue: {
async start(handlers) {
onFire = handlers.onFire;
- onTick = handlers.onTick;
},
async enqueueFire(job) {
enqueued.push(job);
@@ -1189,7 +1187,7 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot while
let second: Promise<void> | undefined;
try {
await gates[0]!.entered.promise;
- await onTick();
+ for (let i = 0; i < 50 && !enqueued.length; i++) await new Promise((r) => setImmediate(r));
clock++;
second = onFire(enqueued.at(-1)!);
await gates[1]!.entered.promise;
diff --git a/test/pg-pool-routing.test.ts b/test/pg-pool-routing.test.ts
index 2c16a94d1..10977e627 100644
--- a/test/pg-pool-routing.test.ts
+++ b/test/pg-pool-routing.test.ts
@@ -141,15 +141,11 @@ test(
let fired = false;
try {
await store.q("SELECT 1");
- await queue.start(
- {
- onFire: async () => {
- fired = true;
- },
- onTick: async () => {},
+ await queue.start({
+ onFire: async () => {
+ fired = true;
},
- 5_000,
- );
+ });
await queue.enqueueFire({ cronId: "pool-test", scheduledAt: Date.now() });
const deadline = Date.now() + 10_000;
while (!fired && Date.now() < deadline) await new Promise((r) => setTimeout(r, 50));
@@ -166,15 +162,11 @@ test(
await queue.stop();
assert.equal((await store.q("SELECT 42 AS answer"))[0]!.answer, 42);
fired = false;
- await queue.start(
- {
- onFire: async () => {
- fired = true;
- },
- onTick: async () => {},
+ await queue.start({
+ onFire: async () => {
+ fired = true;
},
- 5_000,
- );
+ });
await queue.enqueueFire({ cronId: "pool-test-restart", scheduledAt: Date.now() });
const restartedDeadline = Date.now() + 10_000;
while (!fired && Date.now() < restartedDeadline) await new Promise((r) => setTimeout(r, 50));
diff --git a/test/postgres-background-ownership.test.ts b/test/postgres-background-ownership.test.ts
index de43d4d9b..cf05b0134 100644
--- a/test/postgres-background-ownership.test.ts
+++ b/test/postgres-background-ownership.test.ts
@@ -51,30 +51,22 @@ test(
const newEntered = Promise.withResolvers<void>();
let oldFinished = false;
try {
- await queue.start(
- {
- onTick: async () => {},
- onFire: async () => {
- oldEntered.resolve();
- await oldFinish.promise;
- oldFinished = true;
- },
+ await queue.start({
+ onFire: async () => {
+ oldEntered.resolve();
+ await oldFinish.promise;
+ oldFinished = true;
},
- 60_000,
- );
+ });
await queue.enqueueFire({ cronId: "old", scheduledAt: Date.now() });
await oldEntered.promise;
await queue.stopClaims!();
assert.equal(oldFinished, false);
- await queue.start(
- {
- onTick: async () => {},
- onFire: async () => {
- newEntered.resolve();
- },
+ await queue.start({
+ onFire: async () => {
+ newEntered.resolve();
},
- 60_000,
- );
+ });
await queue.enqueueFire({ cronId: "new", scheduledAt: Date.now() });
await newEntered.promise;
assert.equal(oldFinished, false);