Multiplayer agent harness for work.
Stop sending a cron tick job every second (#2007)
Sina Matianb528b8ff0794
diff --git a/src/cron/job-queue.ts b/src/cron/job-queue.tsindex 78b674574..6e7278d43 100644--- a/src/cron/job-queue.ts+++ b/src/cron/job-queue.ts@@ -1,6 +1,5 @@import { PgBoss } from "pg-boss";import { createPgPool, type PgPool } from "../persistence/pg-pool.ts";-import { createSweeper, type Sweeper } from "../util/sweeper.ts";import { errMessage } from "../util/errors.ts";export interface CronFireJob {@@ -11,22 +10,19 @@ export interface CronFireJob {interface CronQueueHandlers {onFire(job: CronFireJob): Promise<void>;- onTick(): Promise<void>;}export interface CronJobQueue {- start(handlers: CronQueueHandlers, tickIntervalMs: number): Promise<void>;+ start(handlers: CronQueueHandlers): Promise<void>;enqueueFire(job: CronFireJob): Promise<void>;healthy(): boolean;stopClaims?(): Promise<void>;stop(): Promise<void>;}-const HEALTHY_SEND_MAX_AGE_MS = 30_000;+const UNHEALTHY_AFTER_ERROR_MS = 30_000;const FIRE_QUEUE = "cron-fire";-const TICK_QUEUE = "cron-tick";-const CRON_TICK_SECONDS = 5;export function createPgBossCronQueue(databaseUrl: string,@@ -48,13 +44,15 @@ export function createPgBossCronQueue(pg = null;await previous?.close();}- boss.on("error", (e) => console.error("[cron-queue] pg-boss error:", errMessage(e)));- let ticker: Sweeper | null = null;let started = false;let initialized = false;- let lastSendOkAt = 0;+ let lastErrorAt = 0;+ boss.on("error", (e) => {+ lastErrorAt = Date.now();+ console.error("[cron-queue] pg-boss error:", errMessage(e));+ });return {- async start(handlers, tickIntervalMs) {+ async start(handlers) {if (started) return;pg ??= createPgPool(databaseUrl, []);try {@@ -63,7 +61,6 @@ export function createPgBossCronQueue(initialized = true;}await boss.createQueue(FIRE_QUEUE, { policy: "short", notify: true });- await boss.createQueue(TICK_QUEUE, { policy: "short", notify: true });const localConcurrency = Math.min(32, Math.max(1, Math.trunc(fireConcurrency)));await boss.work<CronFireJob>(FIRE_QUEUE,@@ -72,13 +69,9 @@ export function createPgBossCronQueue(for (const job of jobs) await handlers.onFire(job.data);},);- await boss.work(TICK_QUEUE, { pollingIntervalSeconds: 1 }, () => handlers.onTick());} catch (e) {if (initialized) {- await Promise.all([- boss.offWork(FIRE_QUEUE, { wait: false }),- boss.offWork(TICK_QUEUE, { wait: false }),- ]).catch(() => {});+ await boss.offWork(FIRE_QUEUE, { wait: false }).catch(() => {});} else {await boss.stop({ close: true, graceful: false }).catch(() => {});await closePool();@@ -86,38 +79,30 @@ export function createPgBossCronQueue(throw e;}started = true;- lastSendOkAt = Date.now();- ticker = createSweeper(- () =>- boss- .send(TICK_QUEUE, {}, { singletonSeconds: CRON_TICK_SECONDS, retryLimit: 0, expireInSeconds: 60 })- .then(() => {- lastSendOkAt = Date.now();- }),- tickIntervalMs,- { label: "cron-queue ticker", immediate: true },- );- ticker.start();+ lastErrorAt = 0;},async enqueueFire(job) {if (!started) return;- await boss.send(FIRE_QUEUE, job, {- startAfter: new Date(Math.max(job.scheduledAt, job.notBefore ?? 0)),- singletonKey: `${job.cronId}:${job.scheduledAt}`,- retryLimit: 0,- });+ await boss+ .send(FIRE_QUEUE, job, {+ startAfter: new Date(Math.max(job.scheduledAt, job.notBefore ?? 0)),+ singletonKey: `${job.cronId}:${job.scheduledAt}`,+ retryLimit: 0,+ })+ .catch((e: unknown) => {+ lastErrorAt = Date.now();+ throw e;+ });},healthy() {- return started && Date.now() - lastSendOkAt < HEALTHY_SEND_MAX_AGE_MS;+ return started && Date.now() - lastErrorAt >= UNHEALTHY_AFTER_ERROR_MS;},async stopClaims() {started = false;- void ticker?.stop();- await Promise.all([boss.offWork(FIRE_QUEUE, { wait: false }), boss.offWork(TICK_QUEUE, { wait: false })]);+ await boss.offWork(FIRE_QUEUE, { wait: false });},async stop() {started = false;- await ticker?.stop();try {await boss.stop({ close: true, graceful: false });} finally {diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.tsindex ef8f4dc8c..24d1ba1b1 100644--- a/src/cron/scheduler.ts+++ b/src/cron/scheduler.ts@@ -32,6 +32,8 @@ const STRANDED_SWEEP_INTERVAL_MS = 10 * 60_000;const FIRE_GC_INTERVAL_MS = 6 * 60 * 60_000;const BUSY_DEFER_MS = 30_000;const BUSY_DEFER_MAX_LATE_MS = 10 * 60_000;+const MAINTENANCE_INTERVAL_MS = 5000;+const REPAIR_INTERVAL_MS = 60_000;type FireResult = { authzFailed: boolean; deferred?: boolean };@@ -490,6 +492,9 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {} catch (e) {reportFailure("scheduler: reconcile", e);}+ }++ async function maintain(): Promise<void> {await sweepStranded(now());await gcFires(now());await deps.sweepAsks?.(now()).catch(reportFailureAs("scheduler: ask sweep", undefined));@@ -502,8 +507,15 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {? leaderLease.hold(TICK_LEASE_KEY, async (lost) => {let lockLost = false;void lost.then(() => (lockLost = true));- while (!stopped && !lockLost && deps.jobQueue!.healthy())- await Promise.race([sleep(1000, { unref: true }), lost, stopSignal.promise]);+ let repairAt = 0;+ while (!stopped && !lockLost && deps.jobQueue!.healthy()) {+ if (Date.now() >= repairAt) {+ repairAt = Date.now() + REPAIR_INTERVAL_MS;+ await runQueueTask(reconcile);+ }+ await runQueueTask(maintain);+ await Promise.race([sleep(MAINTENANCE_INTERVAL_MS, { unref: true }), lost, stopSignal.promise]);+ }}): undefined,1000,@@ -586,13 +598,9 @@ export function createScheduler(deps: SchedulerDeps): Scheduler {}const observed = epoch;starting = deps.jobQueue- .start(- {- onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),- onTick: () => (observed === epoch ? runQueueTask(reconcile) : Promise.resolve()),- },- intervalMs,- )+ .start({+ onFire: (job) => (observed === epoch ? runQueueTask(() => fireJob(job)) : Promise.resolve()),+ }).then(() => {if (!stopped) leaseGuard.start();diff --git a/test/cron-queue-lifecycle.test.ts b/test/cron-queue-lifecycle.test.tsindex c20710865..3f0b2eba4 100644--- a/test/cron-queue-lifecycle.test.ts+++ b/test/cron-queue-lifecycle.test.ts@@ -6,6 +6,7 @@ const pools: { closed: boolean }[] = [];let adapter: Adapter;let failStart = false;let failWork = false;+let emitError: (e: Error) => void = () => {};mock.module("../src/persistence/pg-pool.ts", {namedExports: {@@ -31,7 +32,9 @@ mock.module("pg-boss", {constructor(options: { db: Adapter }) {adapter = options.db;}- on() {}+ on(_event: string, listener: (e: Error) => void) {+ emitError = listener;+ }async start() {await adapter.executeSql("SELECT 1");if (failStart) throw new Error("startup failed");@@ -41,26 +44,23 @@ mock.module("pg-boss", {if (failWork) throw new Error("polling failed");}async offWork() {}- async send() {- await adapter.executeSql("SELECT 1");- }async stop() {}},},});const { createPgBossCronQueue } = await import("../src/cron/job-queue.ts");-const handlers = { onFire: async () => {}, onTick: async () => {} };+const handlers = { onFire: async () => {} };test("late worker SQL after stop cannot reopen the queue database", async () => {const queue = createPgBossCronQueue("postgres://unused");- await queue.start(handlers, 60_000);+ await queue.start(handlers);await queue.stop();const count = pools.length;await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);assert.equal(pools.length, count);assert.ok(pools.at(-1)?.closed);- await queue.start(handlers, 60_000);+ await queue.start(handlers);try {assert.equal(pools.length, count + 1);await adapter.executeSql("SELECT 1");@@ -74,14 +74,14 @@ test("startup failure closes the database and permits a fresh start", async () =const queue = createPgBossCronQueue("postgres://unused");failStart = true;try {- await assert.rejects(queue.start(handlers, 60_000), /startup failed/);+ await assert.rejects(queue.start(handlers), /startup failed/);assert.ok(pools.at(-1)?.closed);assert.equal(queue.healthy(), false);await assert.rejects(adapter.executeSql("SELECT 1"), /closed/);} finally {failStart = false;}- await queue.start(handlers, 60_000);+ await queue.start(handlers);try {assert.equal(pools.at(-1)?.closed, false);assert.ok(queue.healthy());@@ -92,12 +92,12 @@ test("startup failure closes the database and permits a fresh start", async () =test("resuming polling failure preserves the database for admitted callbacks", async () => {const queue = createPgBossCronQueue("postgres://unused");- await queue.start(handlers, 60_000);+ await queue.start(handlers);await queue.stopClaims!();const count = pools.length;failWork = true;try {- await assert.rejects(queue.start(handlers, 60_000), /polling failed/);+ await assert.rejects(queue.start(handlers), /polling failed/);assert.equal(queue.healthy(), false);assert.equal(pools.at(-1)?.closed, false);await adapter.executeSql("SELECT 1");@@ -105,7 +105,7 @@ test("resuming polling failure preserves the database for admitted callbacks", afailWork = false;}try {- await queue.start(handlers, 60_000);+ await queue.start(handlers);assert.equal(pools.length, count);assert.ok(queue.healthy());} finally {@@ -113,3 +113,15 @@ test("resuming polling failure preserves the database for admitted callbacks", a}assert.ok(pools.at(-1)?.closed);});++test("a pg-boss error marks the queue unhealthy without any synthetic send", async () => {+ const queue = createPgBossCronQueue("postgres://unused");+ await queue.start(handlers);+ try {+ assert.ok(queue.healthy());+ emitError(new Error("fetch failed"));+ assert.equal(queue.healthy(), false);+ } finally {+ await queue.stop();+ }+});diff --git a/test/cron-queue.test.ts b/test/cron-queue.test.tsindex 95a78c8f6..8a337d1f6 100644--- a/test/cron-queue.test.ts+++ b/test/cron-queue.test.ts@@ -69,6 +69,7 @@ test(createdBy: "U1",ownerScopeId: scopeId("personal", "U1"),});+ a.scheduler.notifyChanged(cron.id);await until(() => calls.length >= 1, 30_000);assert.equal(calls.length, 1, "the due slot starts exactly one turn");await new Promise((r) => setTimeout(r, 16_000));@@ -100,6 +101,7 @@ test(createdBy: "U2",ownerScopeId: scopeId("personal", "U2"),});+ a.scheduler.notifyChanged(cron.id);const fires = () => calls.filter((c) => c.idempotencyKey?.startsWith(`cron:${cron.id}:`)).length;await until(() => fires() >= 3, 45_000);assert.ok(fires() >= 3, "a recurring cron chains fire jobs across instances");@@ -107,6 +109,7 @@ test(assert.equal(new Set(keys).size, keys.length, "no slot fires twice");await b.crons.update(cron.id, { schedule: { everyMs: 60_000 } });+ b.scheduler.notifyChanged(cron.id);const at = fires();await new Promise((r) => setTimeout(r, 6_000));assert.equal(fires(), at, "a rescheduled cron's stale slot jobs do not fire");@@ -117,3 +120,34 @@ test(}},);++test(+ "pg-boss queue: a cron persisted without its fire job is enqueued once by startup reconciliation",+ { skip, timeout: 120_000 },+ async () => {+ const calls: TurnRequest[] = [];+ const fires = createMemoryCronFireStore();+ const a = instance(calls, 0, fires);+ const b = instance(calls, 0, fires);+ const cron = await a.crons.create({+ schedule: { firstFireAt: Date.now() - 1000 },+ action: "survive the crash",+ owner: "U3",+ createdBy: "U3",+ ownerScopeId: scopeId("personal", "U3"),+ });+ a.scheduler.start(1000);+ b.scheduler.start(1000);+ try {+ const fired = () => calls.filter((c) => c.idempotencyKey?.startsWith(`cron:${cron.id}:`)).length;+ await until(() => fired() >= 1, 30_000);+ await new Promise((r) => setTimeout(r, 3_000));+ assert.equal(fired(), 1, "the orphaned slot fires exactly once across both instances");+ assert.equal((await a.crons.listFires(cron.id)).total, 1);+ } finally {+ a.scheduler.stop();+ b.scheduler.stop();+ await new Promise((r) => setTimeout(r, 500));+ }+ },+);diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.tsindex ea13b2521..1481400b8 100644--- a/test/cron-scheduler.test.ts+++ b/test/cron-scheduler.test.ts@@ -1147,7 +1147,6 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot whileconst gates = [0, 1].map(() => ({ entered: Promise.withResolvers<void>(), release: Promise.withResolvers<void>() }));let attempts = 0;let onFire!: (job: CronFireJob) => Promise<void>;- let onTick!: () => Promise<void>;const enqueued: CronFireJob[] = [];const calls: TurnRequest[] = [];const scheduler = createScheduler({@@ -1174,7 +1173,6 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot whilejobQueue: {async start(handlers) {onFire = handlers.onFire;- onTick = handlers.onTick;},async enqueueFire(job) {enqueued.push(job);@@ -1189,7 +1187,7 @@ test("queue mode: concurrent reconciliation cannot consume an overdue slot whilelet second: Promise<void> | undefined;try {await gates[0]!.entered.promise;- await onTick();+ for (let i = 0; i < 50 && !enqueued.length; i++) await new Promise((r) => setImmediate(r));clock++;second = onFire(enqueued.at(-1)!);await gates[1]!.entered.promise;diff --git a/test/pg-pool-routing.test.ts b/test/pg-pool-routing.test.tsindex 2c16a94d1..10977e627 100644--- a/test/pg-pool-routing.test.ts+++ b/test/pg-pool-routing.test.ts@@ -141,15 +141,11 @@ test(let fired = false;try {await store.q("SELECT 1");- await queue.start(- {- onFire: async () => {- fired = true;- },- onTick: async () => {},+ await queue.start({+ onFire: async () => {+ fired = true;},- 5_000,- );+ });await queue.enqueueFire({ cronId: "pool-test", scheduledAt: Date.now() });const deadline = Date.now() + 10_000;while (!fired && Date.now() < deadline) await new Promise((r) => setTimeout(r, 50));@@ -166,15 +162,11 @@ test(await queue.stop();assert.equal((await store.q("SELECT 42 AS answer"))[0]!.answer, 42);fired = false;- await queue.start(- {- onFire: async () => {- fired = true;- },- onTick: async () => {},+ await queue.start({+ onFire: async () => {+ fired = true;},- 5_000,- );+ });await queue.enqueueFire({ cronId: "pool-test-restart", scheduledAt: Date.now() });const restartedDeadline = Date.now() + 10_000;while (!fired && Date.now() < restartedDeadline) await new Promise((r) => setTimeout(r, 50));diff --git a/test/postgres-background-ownership.test.ts b/test/postgres-background-ownership.test.tsindex de43d4d9b..cf05b0134 100644--- a/test/postgres-background-ownership.test.ts+++ b/test/postgres-background-ownership.test.ts@@ -51,30 +51,22 @@ test(const newEntered = Promise.withResolvers<void>();let oldFinished = false;try {- await queue.start(- {- onTick: async () => {},- onFire: async () => {- oldEntered.resolve();- await oldFinish.promise;- oldFinished = true;- },+ await queue.start({+ onFire: async () => {+ oldEntered.resolve();+ await oldFinish.promise;+ oldFinished = true;},- 60_000,- );+ });await queue.enqueueFire({ cronId: "old", scheduledAt: Date.now() });await oldEntered.promise;await queue.stopClaims!();assert.equal(oldFinished, false);- await queue.start(- {- onTick: async () => {},- onFire: async () => {- newEntered.resolve();- },+ await queue.start({+ onFire: async () => {+ newEntered.resolve();},- 60_000,- );+ });await queue.enqueueFire({ cronId: "new", scheduledAt: Date.now() });await newEntered.promise;assert.equal(oldFinished, false);