Multiplayer agent harness for work.
Keep cron fire history out of scheduler scans (#614)
Joshua France72472a77ca26
diff --git a/src/api/app-messaging.ts b/src/api/app-messaging.tsindex 55717005a..86677da3d 100644--- a/src/api/app-messaging.ts+++ b/src/api/app-messaging.ts@@ -31,6 +31,7 @@ export function createMessagingMethods(App,| "createCron"| "getCron"+ | "getCronRuns"| "listCrons"| "listCronsForViewer"| "updateCron"@@ -113,6 +114,9 @@ export function createMessagingMethods(getCron(id) {return deps.crons.get(id);},+ getCronRuns(id, limit) {+ return deps.crons.getRuns(id, limit);+ },listCrons() {return deps.crons.list();},diff --git a/src/api/app-types.ts b/src/api/app-types.tsindex 0bc5d9071..7d4bcc319 100644--- a/src/api/app-types.ts+++ b/src/api/app-types.ts@@ -41,6 +41,7 @@ import type { CapabilityClaims } from "../auth/capability-token.ts";import type { ScopedConfigStore } from "../resolution/config-store.ts";import { type AdminService } from "../admin/admin-service.ts";import type { CronStore, CreateCronInput, CronPatch } from "../cron/cron-store.ts";+import type { CronFirePage } from "../cron/cron-fire-store.ts";import type { WebhookStore, CreateWebhookInput } from "../webhooks/webhook-store.ts";import type { DeliveryStore } from "../delivery/delivery-store.ts";import type {@@ -348,6 +349,7 @@ export interface App {): Promise<number>;createCron(input: CreateCronInput): Promise<Cron>;getCron(id: string): Promise<Cron | null>;+ getCronRuns(id: string, limit?: number): Promise<CronFirePage>;listCrons(): Promise<Cron[]>;listCronsForViewer(principalId: string): Promise<{ owned: Cron[]; visible: VisibleCron[] }>;updateCron(id: string, patch: CronPatch): Promise<Cron | null>;diff --git a/src/api/control-service.ts b/src/api/control-service.tsindex 62a81e531..726a5b16c 100644--- a/src/api/control-service.ts+++ b/src/api/control-service.ts@@ -564,9 +564,8 @@ export function createControlService(app: App, scheduler?: Scheduler, admin?: Adif (req.limit !== undefined && (!Number.isInteger(req.limit) || req.limit < 1)) {return { ok: false, code: "bad_request", message: "limit must be a positive integer" };}- const fireLog = cron.fireLog ?? [];- const runs = req.limit !== undefined ? fireLog.slice(-req.limit) : fireLog;- return { ok: true, cron, runs, total: fireLog.length };+ const { runs, total } = await app.getCronRuns(id, req.limit);+ return { ok: true, cron, runs, total };},async patchCron(id, req, capability) {diff --git a/src/api/routes/crons.ts b/src/api/routes/crons.tsindex 890e2bd31..ec35ef900 100644--- a/src/api/routes/crons.ts+++ b/src/api/routes/crons.ts@@ -332,7 +332,7 @@ async function runCronNow(ctx: ApiCtx): Promise<void> {}async function cronRuns(ctx: ApiCtx): Promise<void> {- const { res, url, capability } = ctx;+ const { res, url, capability, app } = ctx;const id = ctx.params.id!;const rawLimit = url.searchParams.get("limit");const limit = rawLimit === null ? undefined : Number(rawLimit);@@ -346,9 +346,8 @@ async function cronRuns(ctx: ApiCtx): Promise<void> {if (limit !== undefined && (!Number.isInteger(limit) || limit < 1)) {return sendJson(res, 400, { error: "bad_request", message: "limit must be a positive integer" });}- const fireLog = cron.fireLog ?? [];- const runs = limit !== undefined ? fireLog.slice(-limit) : fireLog;- return sendJson(res, 200, { cron: withoutFireLog(cron), runs, total: fireLog.length });+ const { runs, total } = await app.getCronRuns(id, limit);+ return sendJson(res, 200, { cron: withoutFireLog(cron), runs, total });}const CRON_PATCH_BAD_REQUEST =diff --git a/src/cron/cron-fire-store.ts b/src/cron/cron-fire-store.tsnew file mode 100644index 000000000..9336899d2--- /dev/null+++ b/src/cron/cron-fire-store.ts@@ -0,0 +1,146 @@+import type { CronFireLogEntry } from "../types.ts";+import { jsonbStringify } from "../persistence/durable-map.ts";+import type { PgPool } from "../persistence/pg-pool.ts";++export interface CronFirePage {+ runs: CronFireLogEntry[];+ total: number;+}++export interface CronFireStore {+ ready(): Promise<void>;+ drainInline?(cronId?: string): Promise<void>;+ record(cronId: string, entry: CronFireLogEntry): Promise<void>;+ import(cronId: string, entries: CronFireLogEntry[]): Promise<void>;+ list(cronId: string, limit?: number): Promise<CronFirePage>;+ delete(cronId: string): Promise<void>;+}++export function createMemoryCronFireStore(): CronFireStore {+ const byCron = new Map<string, Map<string, CronFireLogEntry>>();+ const record = (cronId: string, entry: CronFireLogEntry) => {+ let entries = byCron.get(cronId);+ if (!entries) {+ entries = new Map();+ byCron.set(cronId, entries);+ }+ entries.set(entry.fireKey, { ...entries.get(entry.fireKey), ...entry });+ };+ return {+ async ready() {},+ async record(cronId, entry) {+ record(cronId, entry);+ },+ async import(cronId, entries) {+ for (const entry of entries) record(cronId, entry);+ },+ async list(cronId, limit) {+ const all = [...(byCron.get(cronId)?.values() ?? [])].sort(+ (a, b) => a.firedAt - b.firedAt || a.fireKey.localeCompare(b.fireKey),+ );+ return { runs: limit === undefined ? all : all.slice(-limit), total: all.length };+ },+ async delete(cronId) {+ byCron.delete(cronId);+ },+ };+}++export function createPostgresCronFireStore(+ pg: PgPool,+ cronTable = "crons",+ fireTable = "cron_fire_log",+): CronFireStore {+ for (const table of [cronTable, fireTable]) {+ if (!/^[a-z_][a-z0-9_]*$/i.test(table)) throw new Error(`invalid table name: ${table}`);+ }+ const schema = [+ `CREATE TABLE IF NOT EXISTS ${fireTable} (+ cron_id TEXT NOT NULL,+ fire_key TEXT NOT NULL,+ fired_at BIGINT NOT NULL,+ json JSONB NOT NULL,+ PRIMARY KEY (cron_id, fire_key),+ FOREIGN KEY (cron_id) REFERENCES ${cronTable}(id) ON DELETE CASCADE+ )`,+ `CREATE INDEX IF NOT EXISTS ${fireTable}_cron_fired_idx ON ${fireTable} (cron_id, fired_at DESC, fire_key DESC)`,+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${fireTable}_inline_cron_idx ON ${cronTable} (id) WHERE json ? 'fireLog'`,+ ];+ let readyP: Promise<void> | undefined;+ const ready = () =>+ (readyP ??= (async () => {+ try {+ if (!pg.schema) throw new Error("cron fire history requires schema-capable Postgres storage");+ for (const statement of schema) await pg.schema(statement);+ } catch (error) {+ readyP = undefined;+ throw error;+ }+ })());+ const importEntries = async (cronId: string, entries: CronFireLogEntry[]) => {+ if (!entries.length) return;+ await ready();+ await pg.query(+ `INSERT INTO ${fireTable} (cron_id, fire_key, fired_at, json)+ SELECT $1, entry->>'fireKey', (entry->>'firedAt')::BIGINT, entry+ FROM jsonb_array_elements($2::jsonb) entry+ ON CONFLICT (cron_id, fire_key) DO UPDATE+ SET fired_at = EXCLUDED.fired_at, json = ${fireTable}.json || EXCLUDED.json`,+ [cronId, jsonbStringify(entries)],+ );+ };+ return {+ ready,+ async drainInline(cronId) {+ await ready();+ await pg.query(+ `WITH source AS (+ SELECT id, json FROM ${cronTable}+ WHERE json ? 'fireLog'${cronId === undefined ? "" : " AND id = $1"} FOR UPDATE+ ), copied AS (+ INSERT INTO ${fireTable} (cron_id, fire_key, fired_at, json)+ SELECT source.id, entry->>'fireKey', (entry->>'firedAt')::BIGINT, entry+ FROM source+ CROSS JOIN LATERAL jsonb_array_elements(+ CASE WHEN jsonb_typeof(source.json->'fireLog') = 'array' THEN source.json->'fireLog' ELSE '[]'::jsonb END+ ) AS entry+ WHERE entry ? 'fireKey' AND entry ? 'firedAt'+ ON CONFLICT (cron_id, fire_key) DO UPDATE+ SET fired_at = EXCLUDED.fired_at, json = ${fireTable}.json || EXCLUDED.json+ )+ UPDATE ${cronTable} target SET json = target.json - 'fireLog'+ FROM source WHERE target.id = source.id`,+ cronId === undefined ? [] : [cronId],+ );+ },+ async record(cronId, entry) {+ await importEntries(cronId, [entry]);+ },+ async import(cronId, entries) {+ await importEntries(cronId, entries);+ },+ async list(cronId, limit) {+ await ready();+ const rows =+ limit === undefined+ ? await pg.q(+ `SELECT json, COUNT(*) OVER()::BIGINT AS total+ FROM ${fireTable} WHERE cron_id = $1 ORDER BY fired_at, fire_key`,+ [cronId],+ )+ : await pg.q(+ `SELECT json, total FROM (+ SELECT fired_at, fire_key, json, COUNT(*) OVER()::BIGINT AS total+ FROM ${fireTable}+ WHERE cron_id = $1 ORDER BY fired_at DESC, fire_key DESC LIMIT $2+ ) recent ORDER BY fired_at, fire_key`,+ [cronId, limit],+ );+ return { runs: rows.map((row) => row.json as CronFireLogEntry), total: Number(rows[0]?.total ?? 0) };+ },+ async delete(cronId) {+ await ready();+ await pg.query(`DELETE FROM ${fireTable} WHERE cron_id = $1`, [cronId]);+ },+ };+}diff --git a/src/cron/cron-store.ts b/src/cron/cron-store.tsindex b80b1d26b..ceaffbc0d 100644--- a/src/cron/cron-store.ts+++ b/src/cron/cron-store.ts@@ -10,6 +10,7 @@ import {} from "../triggers/trigger-store.ts";import { hashId } from "../util/crypto.ts";import { advanceNextFireAt, isCalendarSchedule, normalizeSchedule, recoverNextFireAt } from "./schedule.ts";+import { createMemoryCronFireStore, type CronFirePage, type CronFireStore } from "./cron-fire-store.ts";export interface CreateCronInput extends CreateTriggerInput {schedule: Cron["schedule"];@@ -44,6 +45,7 @@ export interface CronStore {setDestination(id: string, destination: Destination | undefined): Promise<void>;setRecipientConsent(id: string, recipientConsent: RecipientConsent): Promise<void>;recordFire(id: string, entry: CronFireLogEntry): Promise<void>;+ getRuns(id: string, limit?: number): Promise<CronFirePage>;markFired(id: string, at: number, scheduledAt?: number): Promise<void>;markAttempted(id: string, at: number): Promise<void>;claimSlot(id: string, scheduledAt: number, at: number): Promise<boolean>;@@ -57,7 +59,33 @@ function normalizeTitle(title: string | undefined): string | undefined {return trimmed.length > 80 ? `${trimmed.slice(0, 79)}...` : trimmed;}-export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cron>()): CronStore {+export function createCronStore(+ backing: DurableMap<Cron> = createMemoryMap<Cron>(),+ fires: CronFireStore = createMemoryCronFireStore(),+): CronStore {+ let readyP: Promise<void> | undefined;+ const ready = () =>+ (readyP ??= (async () => {+ try {+ await backing.get("__cron_fire_log_migration__");+ await fires.ready();+ } catch (error) {+ readyP = undefined;+ throw error;+ }+ })());+ const withoutFireLog = async (cron: Cron | null): Promise<Cron | null> => {+ if (!cron) return null;+ const { fireLog, ...rest } = cron;+ if (fireLog?.length) {+ if (fires.drainInline) await fires.drainInline(cron.id);+ else {+ await fires.import(cron.id, fireLog);+ await backing.merge(cron.id, { fireLog: undefined });+ }+ }+ return rest;+ };return {async create(input) {assertNoEscalation(input);@@ -88,8 +116,16 @@ export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cron...(input.unattendedGrants ? { unattendedGrants: input.unattendedGrants } : {}),}));},- get: (id) => backing.get(id),- list: () => backing.all(),+ async get(id) {+ await ready();+ if (fires.drainInline) await fires.drainInline(id);+ return withoutFireLog(await backing.get(id));+ },+ async list() {+ await ready();+ if (fires.drainInline) await fires.drainInline();+ return (await Promise.all((await backing.all()).map(withoutFireLog))) as Cron[];+ },async update(id, patch) {const fields: Partial<Cron> = {};if (patch.title !== undefined) fields.title = normalizeTitle(patch.title);@@ -109,7 +145,9 @@ export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cronif (patch.unattendedGrants !== undefined) fields.unattendedGrants = patch.unattendedGrants;return backing.merge(id, fields);},- delete: (id) => backing.delete(id),+ async delete(id) {+ await Promise.all([backing.delete(id), fires.delete(id)]);+ },async setEnabled(id, enabled) {await backing.merge(id, { enabled, ...(enabled ? { archived: false } : {}) });},@@ -120,18 +158,15 @@ export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cronreturn setTriggerRecipientConsent(backing, id, recipientConsent);},async recordFire(id, entry) {- const addEntry = (cron: Cron): Cron => {- const byKey = new Map((cron.fireLog ?? []).map((e) => [e.fireKey, e]));- byKey.set(entry.fireKey, { ...byKey.get(entry.fireKey), ...entry });- return { ...cron, fireLog: [...byKey.values()].sort((a, b) => a.firedAt - b.firedAt) };- };- if (backing.update) {- await backing.update(id, addEntry);- return;- }- const cron = await backing.get(id);- if (!cron) return;- await backing.merge(id, { fireLog: addEntry(cron).fireLog });+ await ready();+ if (fires.drainInline) await fires.drainInline(id);+ if (!(await withoutFireLog(await backing.get(id)))) return;+ await fires.record(id, entry);+ },+ async getRuns(id, limit) {+ await ready();+ await withoutFireLog(await backing.get(id));+ return fires.list(id, limit);},async markFired(id, at, scheduledAt) {const cron = await backing.get(id);@@ -186,7 +221,10 @@ export function createCronStore(backing: DurableMap<Cron> = createMemoryMap<Cron},async due(now) {const due: Array<Cron & { scheduledAt: number }> = [];- for (const c of await backing.all()) {+ await ready();+ if (fires.drainInline) await fires.drainInline();+ for (const row of await backing.all()) {+ const c = (await withoutFireLog(row))!;if (c.archived || !c.enabled) continue;const scheduledAt = recoverNextFireAt(c.schedule, c.createdAt, c.lastFiredAt, c.nextFireAt);if (scheduledAt !== undefined && now >= scheduledAt) due.push({ ...c, nextFireAt: scheduledAt, scheduledAt });diff --git a/src/wiring.ts b/src/wiring.tsindex ae1239e01..3499722b3 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -59,6 +59,7 @@ import { createPostgresRateLimiter } from "./ratelimit/postgres-rate-limiter.ts"import { createBudgetTracker } from "./ratelimit/budget.ts";import { createPostgresBudgetTracker } from "./ratelimit/postgres-budget.ts";import { createCronStore, type CronStore } from "./cron/cron-store.ts";+import { createMemoryCronFireStore, createPostgresCronFireStore } from "./cron/cron-fire-store.ts";import { createDeliveryStore, type DeliveryStore } from "./delivery/delivery-store.ts";import { createPostgresDeliveryStore } from "./delivery/postgres-delivery-store.ts";import { wireRunResultDeliveries } from "./delivery/run-result-delivery.ts";@@ -936,7 +937,10 @@ export function buildApp(: createMemoryEnvironmentStore();const monitors = createMonitorStore(artifactMap<Monitor>("monitors"));const cronChanged: { notify?: (id: string) => void } = {};- const cronsBase = createCronStore(artifactMap<Cron>("crons"));+ const cronsBase = createCronStore(+ artifactMap<Cron>("crons"),+ pgArtifactMap ? createPostgresCronFireStore(pgArtifactMap.pool) : createMemoryCronFireStore(),+ );const crons: CronStore = {...cronsBase,async create(input) {diff --git a/test/cron-queue.test.ts b/test/cron-queue.test.tsindex a537e6d53..bf2363baf 100644--- a/test/cron-queue.test.ts+++ b/test/cron-queue.test.ts@@ -3,6 +3,7 @@ import assert from "node:assert/strict";import { createScheduler, type Scheduler } from "../src/cron/scheduler.ts";import { createPgBossCronQueue } from "../src/cron/job-queue.ts";import { createCronStore, type CronStore } from "../src/cron/cron-store.ts";+import { createPostgresCronFireStore } from "../src/cron/cron-fire-store.ts";import { createDeliveryStore } from "../src/delivery/delivery-store.ts";import { createIdempotencyStore, type IdempotencyRecord } from "../src/idempotency/idempotency-store.ts";import { createIdentityService } from "../src/identity/identity-service.ts";@@ -15,13 +16,14 @@ const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the cron queueconst SCHEMA = "pgboss_cron_queue_test";const CRONS_TABLE = "cron_queue_test_crons";const IDEM_TABLE = "cron_queue_test_idempotency";+const FIRES_TABLE = "cron_queue_test_fires";before(async () => {if (!URL) return;const pg = (await import("pg")).default;const p = new pg.Pool({ connectionString: URL });await p.query(`DROP SCHEMA IF EXISTS ${SCHEMA} CASCADE`);- await p.query(`DROP TABLE IF EXISTS ${CRONS_TABLE}, ${IDEM_TABLE}`);+ await p.query(`DROP TABLE IF EXISTS ${CRONS_TABLE}, ${IDEM_TABLE}, ${FIRES_TABLE}`);await p.end();});@@ -32,7 +34,10 @@ async function until(cond: () => boolean, ms: number): Promise<void> {function instance(calls: TurnRequest[], turnMs = 0): { scheduler: Scheduler; crons: CronStore } {const maps = createPostgresMapFactory(URL!);- const crons = createCronStore(maps.map<Cron>(CRONS_TABLE));+ const crons = createCronStore(+ maps.map<Cron>(CRONS_TABLE),+ createPostgresCronFireStore(maps.pool, CRONS_TABLE, FIRES_TABLE),+ );const run = async (req: TurnRequest): Promise<TurnResult> => {calls.push(req);if (turnMs) await new Promise((r) => setTimeout(r, turnMs));@@ -71,7 +76,7 @@ test(await new Promise((r) => setTimeout(r, 16_000));assert.equal(calls.length, 1, "no sibling or reconcile re-run while (or after) the slow turn runs");assert.equal((await b.crons.get(cron.id))?.enabled, false, "the one-shot ends disabled");- assert.equal((await b.crons.get(cron.id))?.fireLog?.length, 1, "one fire recorded");+ assert.equal((await b.crons.getRuns(cron.id)).runs.length, 1, "one fire recorded");} finally {a.scheduler.stop();b.scheduler.stop();diff --git a/test/cron-scheduler.test.ts b/test/cron-scheduler.test.tsindex cd253a8bd..511ef5dc8 100644--- a/test/cron-scheduler.test.ts+++ b/test/cron-scheduler.test.ts@@ -313,9 +313,9 @@ test("a recurring teammate-DM cron without current recipient consent is withheldassert.equal(pending.length, 1);assert.equal(pending[0]?.destination.target, "U1");assert.match(pending[0]?.text ?? "", /consent.*skipped/i);- const stored = await crons.get(cron.id);- assert.equal(stored?.fireLog?.[0]?.status, "refused");- assert.match(stored?.fireLog?.[0]?.note ?? "", /consent/);+ const stored = await crons.getRuns(cron.id);+ assert.equal(stored.runs[0]?.status, "refused");+ assert.match(stored.runs[0]?.note ?? "", /consent/);});test("a teammate-DM cron created in a channel delivers its real output (§10 parity gate)", async () => {@@ -826,9 +826,9 @@ test("cron fires do not replay prior sessions or inline prior fire context", asyassert.doesNotMatch(texts[1] ?? "", /Recent fires/);assert.doesNotMatch(texts[1] ?? "", /first <invoke name="execute">/);assert.doesNotMatch(texts[1] ?? "", /first \[invoke name="execute"\]/);- const stored = await crons.get(cron.id);- assert.equal(stored?.fireLog?.length, 2);- assert.equal(stored?.fireLog?.[0]?.reply, 'first <invoke name="execute">');+ const stored = await crons.getRuns(cron.id);+ assert.equal(stored.runs.length, 2);+ assert.equal(stored.runs[0]?.reply, 'first <invoke name="execute">');});test("an idempotency-skipped cron fire does not create fire log history", async () => {@@ -857,10 +857,10 @@ test("an idempotency-skipped cron fire does not create fire log history", asyncawait scheduler.tick(2500);- const after = await crons.get(cron.id);+ const after = await crons.getRuns(cron.id);assert.equal(calls.length, 0, "duplicate slot does not reach the agent");- assert.equal(after?.fireLog?.length ?? 0, 0, "duplicate slot does not look like a fire");- assert.equal(after?.lastFiredAt, 2500, "the due slot is still advanced after durable dedupe");+ assert.equal(after.runs.length, 0, "duplicate slot does not look like a fire");+ assert.equal((await crons.get(cron.id))?.lastFiredAt, 2500, "the due slot is still advanced after durable dedupe");});test("cron fire log omits replies that echo the runtime wrapper", async () => {@@ -876,8 +876,8 @@ test("cron fire log omits replies that echo the runtime wrapper", async () => {await scheduler.tick(2000);await scheduler.tick(3500);- const stored = await crons.get((await crons.list())[0]!.id);- assert.equal(stored?.fireLog?.[0]?.reply, "[reply echoed cron runtime context; omitted]");+ const stored = await crons.getRuns((await crons.list())[0]!.id);+ assert.equal(stored.runs[0]?.reply, "[reply echoed cron runtime context; omitted]");assert.doesNotMatch(calls[1]?.text ?? "", /reply=You said: \[Cron runtime context\]/);assert.doesNotMatch(calls[1]?.text ?? "", /reply=\[reply echoed cron runtime context; omitted\]/);});@@ -966,10 +966,10 @@ test("a failing cron fire is logged, not swallowed", async (t) => {logged.some((l) => l.includes("[scheduler] fire failed") && l.includes("boom")),"the fire error must reach the log",);- const after = await crons.get(cron.id);- assert.equal(after?.fireLog?.length, 1);- assert.equal(after?.fireLog?.[0]?.status, "failed");- assert.equal(after?.fireLog?.[0]?.note, "boom");+ const after = await crons.getRuns(cron.id);+ assert.equal(after.runs.length, 1);+ assert.equal(after.runs[0]?.status, "failed");+ assert.equal(after.runs[0]?.note, "boom");});test("queue mode: fires claim the slot before running, and stale or lost claims never run", async () => {diff --git a/test/cron-store.test.ts b/test/cron-store.test.tsindex fb79e7443..d2a3d0145 100644--- a/test/cron-store.test.ts+++ b/test/cron-store.test.ts@@ -224,13 +224,51 @@ test("recordFire appends compact durable fire log entries and replaces duplicatereply: "second updated",});- const after = await store.get(cron.id);+ const after = await store.getRuns(cron.id);assert.deepEqual(- after?.fireLog?.map((entry) => entry.fireKey),+ after.runs.map((entry) => entry.fireKey),["f1", "f2"],);- assert.equal(after?.fireLog?.[1]?.threadRef, "cron:c:fire:2b");- assert.equal(after?.fireLog?.[1]?.reply, "second updated");+ assert.equal(after.runs[1]?.threadRef, "cron:c:fire:2b");+ assert.equal(after.runs[1]?.reply, "second updated");+});++test("due scheduler scans do not load persisted fire history", async () => {+ const backing = createMemoryMap<Cron>();+ await backing.put("old", {+ ...base,+ id: "old",+ schedule: { firstFireAt: 1 },+ enabled: true,+ createdAt: 0,+ fireLog: Array.from({ length: 100 }, (_, i) => ({+ fireKey: `fire-${i}`,+ threadRef: `cron:old:fire-${i}`,+ firedAt: i,+ reply: "x".repeat(10_000),+ })),+ });++ const store = createCronStore(backing);+ const [due] = await store.due(1);+ assert.equal(due?.fireLog, undefined);+ assert.equal((await backing.get("old"))?.fireLog, undefined);+ assert.equal((await store.getRuns("old")).total, 100);+});++test("fire history remains complete while reads can page the newest entries", async () => {+ const store = createCronStore();+ const cron = await store.create({ ...base, schedule: { everyMs: 1000 } });+ for (let i = 1; i <= 3; i++) {+ await store.recordFire(cron.id, { fireKey: `f${i}`, threadRef: `cron:f${i}`, firedAt: i });+ }+ assert.deepEqual(await store.getRuns(cron.id, 2), {+ runs: [+ { fireKey: "f2", threadRef: "cron:f2", firedAt: 2 },+ { fireKey: "f3", threadRef: "cron:f3", firedAt: 3 },+ ],+ total: 3,+ });});test("create stores runAs + member snapshot for a scopeFloor cron", async () => {diff --git a/test/postgres-map.test.ts b/test/postgres-map.test.tsindex 73e6fea21..0f8384d67 100644--- a/test/postgres-map.test.ts+++ b/test/postgres-map.test.ts@@ -2,6 +2,7 @@ import { test, before } from "node:test";import assert from "node:assert/strict";import { createPostgresMapFactory } from "../src/persistence/durable-map.ts";import { createCronStore } from "../src/cron/cron-store.ts";+import { createPostgresCronFireStore } from "../src/cron/cron-fire-store.ts";import { scopeId, type Cron } from "../src/types.ts";import {createKeychain,@@ -20,7 +21,7 @@ before(async () => {const pg = (await import("pg")).default;const p = new pg.Pool({ connectionString: URL });await p.query(- "DROP TABLE IF EXISTS map_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",+ "DROP TABLE IF EXISTS map_widgets, map_crons, map_cron_fires, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",);await p.end();});@@ -68,7 +69,11 @@ test("pg map: a value persists across map instances (no per-process cache to div});test("pg map: an artifact store rides the map (a cron round-trips through Postgres)", { skip }, async () => {- const store = createCronStore(createPostgresMapFactory(URL!).map<Cron>("map_crons"));+ const writer = createPostgresMapFactory(URL!);+ const store = createCronStore(+ writer.map<Cron>("map_crons"),+ createPostgresCronFireStore(writer.pool, "map_crons", "map_cron_fires"),+ );const c = await store.create({schedule: { everyMs: 60_000 },action: "digest",@@ -77,10 +82,88 @@ test("pg map: an artifact store rides the map (a cron round-trips through PostgrcreatedBy: "U1",});await store.markFired(c.id, 123);- const reader = createCronStore(createPostgresMapFactory(URL!).map<Cron>("map_crons"));+ await store.recordFire(c.id, { fireKey: "f1", threadRef: "cron:f1", firedAt: 123, reply: "done" });+ await store.recordFire(c.id, { fireKey: "f2", threadRef: "cron:f2", firedAt: 124 });+ await store.recordFire(c.id, { fireKey: "f3", threadRef: "cron:f3", firedAt: 125 });+ const factory = createPostgresMapFactory(URL!);+ const reader = createCronStore(+ factory.map<Cron>("map_crons"),+ createPostgresCronFireStore(factory.pool, "map_crons", "map_cron_fires"),+ );const got = await reader.get(c.id);assert.equal(got?.action, "digest");assert.equal(got?.lastFiredAt, 123);+ assert.deepEqual(await reader.getRuns(c.id, 2), {+ runs: [+ { fireKey: "f2", threadRef: "cron:f2", firedAt: 124 },+ { fireKey: "f3", threadRef: "cron:f3", firedAt: 125 },+ ],+ total: 3,+ });+});++test("pg cron store migrates inline fire history without loading it in cron scans", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const backing = factory.map<Cron>("map_crons");+ await backing.put("legacy", {+ id: "legacy",+ schedule: { firstFireAt: 1 },+ enabled: true,+ createdAt: 0,+ ownerScopeId: scopeId("personal", "U1"),+ owner: "U1",+ createdBy: "U1",+ fireLog: [{ fireKey: "legacy-fire", threadRef: "cron:legacy:fire", firedAt: 1, reply: "kept" }],+ });+ const store = createCronStore(backing, createPostgresCronFireStore(factory.pool, "map_crons", "map_cron_fires"));++ assert.equal((await store.list())[0]?.fireLog, undefined);+ assert.equal((await backing.get("legacy"))?.fireLog, undefined);+ assert.deepEqual(await store.getRuns("legacy"), {+ runs: [{ fireKey: "legacy-fire", threadRef: "cron:legacy:fire", firedAt: 1, reply: "kept" }],+ total: 1,+ });+});++test("pg cron migration preserves fires written by an old worker during rollout", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const backing = factory.map<Cron>("map_crons");+ await backing.put("rolling", {+ id: "rolling",+ schedule: { firstFireAt: 1 },+ enabled: true,+ createdAt: 0,+ ownerScopeId: scopeId("personal", "U1"),+ owner: "U1",+ createdBy: "U1",+ fireLog: [{ fireKey: "before", threadRef: "cron:rolling:before", firedAt: 1 }],+ });+ const store = createCronStore(backing, createPostgresCronFireStore(factory.pool, "map_crons", "map_cron_fires"));+ const client = await (await factory.pool.pool()).connect();+ try {+ await client.query("BEGIN");+ const locked = await client.query("SELECT json FROM map_crons WHERE id = $1 FOR UPDATE", ["rolling"]);+ const json = locked.rows[0]!.json as Cron;+ json.fireLog!.push({ fireKey: "during", threadRef: "cron:rolling:during", firedAt: 2 });+ const migration = store.due(1);+ await new Promise<void>((resolve) => setImmediate(resolve));+ await client.query("UPDATE map_crons SET json = $2 WHERE id = $1", ["rolling", json]);+ await client.query("COMMIT");+ assert.equal((await migration)[0]?.fireLog, undefined);+ } finally {+ await client.query("ROLLBACK").catch(() => undefined);+ client.release();+ }++ await backing.merge("rolling", {+ fireLog: [{ fireKey: "after", threadRef: "cron:rolling:after", firedAt: 3 }],+ });+ assert.equal((await store.due(1))[0]?.fireLog, undefined);+ assert.equal((await backing.get("rolling"))?.fireLog, undefined);+ assert.deepEqual(+ (await store.getRuns("rolling")).runs.map((run) => run.fireKey),+ ["before", "during", "after"],+ );});test(