Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
fix(cron): dedup fires on a per-job key and skip a new job's first due fire
Calvin180714f7308c
diff --git a/backend/e2e-test/cron-job.spec.ts b/backend/e2e-test/cron-job.spec.tsindex d17048d702..fe3ae75970 100644--- a/backend/e2e-test/cron-job.spec.ts+++ b/backend/e2e-test/cron-job.spec.ts@@ -23,6 +23,10 @@ const KEY_PREFIX = "{cron-e2e}";const SLOT_KEYS = `${KEY_PREFIX}:slot:*`;const RUN_KEYS = (name: string) => `${KEY_PREFIX}:run:${name}:*`;const PENDING_ZSET = `${KEY_PREFIX}:pending`;+const LAST_KEY = (name: string) => `${KEY_PREFIX}:last:${name}`;++// waitForFreshMinute keeps every test clear of a boundary, so this is FAST_PATTERN's latest fire.+const currentFireMs = () => Math.floor(Date.now() / 60_000) * 60_000;const clearNamespace = async () => {const keys = await testRedis.keys(`${KEY_PREFIX}:*`);@@ -92,12 +96,29 @@ const makeFactory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>)const FAST_PATTERN = "* * * * *"; // fires every minuteconst allFactories: ReturnType<typeof makeFactory>[] = [];-const factory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>) => {+// The manager skips the latest fire of a job that has never fired, so by default every job a+// test registers is marked as having fired in the previous minute, which lets it run in the+// current one. NX keeps a value an earlier factory in the same test already wrote. The first+// enqueue tick runs enqueueIntervalMs after start(), long after this write lands.+const factory = (+ overrides?: Partial<Parameters<typeof cronJobFactory>[0]>,+ { existingJobs = true }: { existingJobs?: boolean } = {}+) => {const f = makeFactory(overrides);allFactories.push(f);- return f;+ return {+ ...f,+ register: (entry: Parameters<typeof f.register>[0]) => {+ f.register(entry);+ if (existingJobs) void testRedis.set(LAST_KEY(entry.name), String(currentFireMs() - 60_000), "NX");+ }+ };};+// For tests about a job's very first registration.+const newJobFactory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>) =>+ factory(overrides, { existingJobs: false });+// A handler that never settles on its own, simulating a wedged network call.// The resolver is parked so afterEach can release it: an abandoned handler// leaves `withTimeout`'s race unsettled, so its `finally` never runs and its@@ -493,6 +514,92 @@ describe("jitter", () => {}, 10_000);});+// ── dedup across restarts ──────────────────────────────────────────────────────++describe("dedup across restarts", () => {+ const sleep = (ms: number) =>+ new Promise((r) => {+ setTimeout(r, ms);+ });+ // Only Date is faked, so the manager's timers keep running in real time while every enqueue+ // tick sees the same distance from the fire. afterEach restores the real clock.+ const freezeClockAfterFire = (ms: number) => {+ const fire = currentFireMs();+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(fire + ms);+ return fire;+ };++ test("a restart after the run hash expires does not run the same fire again", async () => {+ const first = vi.fn().mockResolvedValue(undefined);+ const f1 = factory();+ f1.register({ name: "restart-job", pattern: FAST_PATTERN, handler: first, runHashTtlS: 3600 });+ f1.start();+ await sleep(2000);+ expect(first).toHaveBeenCalledTimes(1);+ await f1.stop();++ const runKeys = await testRedis.keys(RUN_KEYS("restart-job"));+ await testRedis.del(...runKeys);++ const second = vi.fn().mockResolvedValue(undefined);+ const f2 = factory();+ f2.register({ name: "restart-job", pattern: FAST_PATTERN, handler: second, runHashTtlS: 3600 });+ f2.start();+ await sleep(1500);++ expect(second).not.toHaveBeenCalled();+ expect(await testRedis.keys(RUN_KEYS("restart-job"))).toEqual([]);+ }, 10_000);++ test("a run hash written without a last value is recorded instead of enqueued again", async () => {+ const scheduledAt = currentFireMs();+ await testRedis.hset(+ `${KEY_PREFIX}:run:backfill-job:${scheduledAt}`,+ "name",+ "backfill-job",+ "status",+ "completed"+ );++ const handler = vi.fn().mockResolvedValue(undefined);+ const f = newJobFactory();+ f.register({ name: "backfill-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });+ f.start();+ await sleep(1500);++ expect(handler).not.toHaveBeenCalled();+ expect(await testRedis.get(LAST_KEY("backfill-job"))).toBe(String(scheduledAt));+ }, 10_000);++ test("a job that has never fired skips the fire due before it and waits for the next one", async () => {+ const fire = freezeClockAfterFire(30_000);+ const handler = vi.fn().mockResolvedValue(undefined);+ const f = newJobFactory();+ f.register({ name: "new-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });+ f.start();+ await sleep(1500);++ expect(handler).not.toHaveBeenCalled();+ expect(await testRedis.keys(RUN_KEYS("new-job"))).toEqual([]);+ expect(await testRedis.get(LAST_KEY("new-job"))).toBe(String(fire));+ }, 10_000);++ // Jobs that ran before the last key existed have no last value either. When the first deploy+ // that writes it lands just after a boundary, no pod may have enqueued that fire yet, and+ // treating the job as new would silently drop it.+ test("a job with no last value still runs a fire that only just became due", async () => {+ freezeClockAfterFire(100);+ const handler = vi.fn().mockResolvedValue(undefined);+ const f = newJobFactory();+ f.register({ name: "boundary-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });+ f.start();+ await sleep(2000);++ expect(handler).toHaveBeenCalledTimes(1);+ }, 10_000);+});+// ── graceful shutdown ──────────────────────────────────────────────────────────describe("graceful shutdown", () => {diff --git a/backend/src/lib/cron/cron-job.test.ts b/backend/src/lib/cron/cron-job.test.tsindex d990d1dbf6..5073fc4f82 100644--- a/backend/src/lib/cron/cron-job.test.ts+++ b/backend/src/lib/cron/cron-job.test.ts@@ -425,9 +425,9 @@ describe("redis cluster compatibility", () => {// Drive at least one enqueue tick (enqueueIntervalMs=100 in makeFactory).await vi.advanceTimersByTimeAsync(150);- // ENQUEUE_RUN_LUA is the only multi-key script (KEYS[1]=run, KEYS[2]=pending).- // Identify its eval call by numKeys === 2 (RELEASE_SLOT_IF_MINE_LUA uses 1).- const multiKeyEvalCalls = redis.eval.mock.calls.filter((c) => c[1] === 2);+ // ENQUEUE_RUN_LUA is the only multi-key script (KEYS[1]=run, KEYS[2]=pending, KEYS[3]=last).+ // Identify its eval call by numKeys === 3 (RELEASE_SLOT_IF_MINE_LUA uses 1).+ const multiKeyEvalCalls = redis.eval.mock.calls.filter((c) => c[1] === 3);expect(multiKeyEvalCalls.length).toBeGreaterThan(0);for (const call of multiKeyEvalCalls) {@@ -764,16 +764,16 @@ describe("derived jitter", () => {const expectedWindowMs = (intervalMs: number, maxJitterMs = MAX_JITTER_MS) =>Math.min(intervalMs * FRACTION, maxJitterMs, intervalMs - NEXT_FIRE_BUFFER_MS);- // ENQUEUE_RUN_LUA is the only two-key script:- // eval(script, 2, runKey, pendingZset, name, scheduledAt, ttl, id, enqueuedAtMs, eligibleAt)+ // ENQUEUE_RUN_LUA is the only three-key script:+ // eval(script, 3, runKey, pendingZset, lastKey, name, scheduledAt, ttl, id, enqueuedAtMs, eligibleAt, justBecameDue)const enqueuedRuns = (redis: ReturnType<typeof makeRedis>) =>redis.eval.mock.calls- .filter((c) => c[1] === 2)+ .filter((c) => c[1] === 3).map((c) => ({- name: String(c[4]),- scheduledAt: Number(c[5]),- id: String(c[7]),- eligibleAt: Number(c[9])+ name: String(c[5]),+ scheduledAt: Number(c[6]),+ id: String(c[8]),+ eligibleAt: Number(c[10])}));// Boots a manager, drives one enqueue tick, and returns what it wrote.diff --git a/backend/src/lib/cron/cron-job.ts b/backend/src/lib/cron/cron-job.tsindex d86860aab2..46594639ea 100644--- a/backend/src/lib/cron/cron-job.ts+++ b/backend/src/lib/cron/cron-job.ts@@ -61,6 +61,8 @@ const PARTICIPANT_SLOTS = 5;const PROCESS_BATCH_SIZE = 50;const NEXT_FIRE_BUFFER_MS = 1_000; // shared margin for fitsBeforeNextFire and jitterWindowMsconst ERROR_MESSAGE_MAX_LEN = 4_000;+// A fire due at most this many enqueue ticks ago counts as just due: pods may not have enqueued it yet.+const JUST_DUE_ENQUEUE_TICKS = 2;const DEFAULTS = {slotTtlMs: 60_000,@@ -98,20 +100,49 @@ const F = {NextAttemptAt: "next_attempt_at"} as const;-// ARGV[2] = unjittered scheduled_at; ARGV[6] = eligibleAt (pending zset score).+const EnqueueResult = {+ AlreadyEnqueued: 0,+ Enqueued: 1,+ SkippedFirstFire: 2+} as const;++// Dedup checks "lastKey" rather than the run hash, because the run hash expires and "lastKey" does not.const ENQUEUE_RUN_LUA = `- if redis.call('exists', KEYS[1]) == 0 then- redis.call('hset', KEYS[1],- 'name', ARGV[1],- 'scheduled_at', ARGV[2],- 'status', 'pending',- 'attempts', 0,- 'enqueued_at_ms', ARGV[5])- redis.call('expire', KEYS[1], ARGV[3])- redis.call('zadd', KEYS[2], ARGV[6], ARGV[4])- return 1+ local runKey, pendingZset, lastKey = KEYS[1], KEYS[2], KEYS[3]+ local name, scheduledAt, runHashTtlS, runId, enqueuedAtMs, eligibleAt, justBecameDue =+ ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6], ARGV[7] == '1'++ local last = tonumber(redis.call('get', lastKey))+ if last and last >= tonumber(scheduledAt) then+ return ${EnqueueResult.AlreadyEnqueued}end- return 0++ -- Backfill only, not scheduling logic. Pods without "lastKey" create the run hash but never+ -- set "lastKey". If the run hash exists, the fire is already enqueued, so just record it.+ -- Remove once every deployment sets "lastKey".+ if redis.call('exists', runKey) == 1 then+ redis.call('set', lastKey, scheduledAt)+ return ${EnqueueResult.AlreadyEnqueued}+ end++ -- No "last" means the job has never fired. Record this fire without running it, so the next+ -- fire is the first run. A fire that "justBecameDue" still runs: on the first deploy of+ -- "lastKey", existing jobs have no "last" either, and no pod may have enqueued this fire yet.+ if not last and not justBecameDue then+ redis.call('set', lastKey, scheduledAt)+ return ${EnqueueResult.SkippedFirstFire}+ end++ redis.call('hset', runKey,+ 'name', name,+ 'scheduled_at', scheduledAt,+ 'status', 'pending',+ 'attempts', 0,+ 'enqueued_at_ms', enqueuedAtMs)+ redis.call('expire', runKey, runHashTtlS)+ redis.call('zadd', pendingZset, eligibleAt, runId)+ redis.call('set', lastKey, scheduledAt)+ return ${EnqueueResult.Enqueued}`;// Release slot only if we still own it (stale stop() after TTL expiry).@@ -195,6 +226,7 @@ export const cronJobFactory = ({const SLOT_KEY = (i: number) => `${keyPrefix}:slot:${i}`;const RUN_KEY = (id: string) => `${keyPrefix}:run:${id}`;const LEASE_KEY = (id: string) => `${keyPrefix}:lease:${id}`;+ const LAST_KEY = (name: string) => `${keyPrefix}:last:${name}`;const PENDING_ZSET = `${keyPrefix}:pending`;if (!schedulingEnabled) {@@ -401,24 +433,32 @@ export const cronJobFactory = ({const id = `${entry.name}:${scheduledAt}`;const eligibleAt = eligibleAtMs(entry, scheduledAt, nextFireMs(entry.pattern));// eslint-disable-next-line no-await-in-loop- const initialized = await redis.eval(+ const result = await redis.eval(ENQUEUE_RUN_LUA,- 2,+ 3,RUN_KEY(id),PENDING_ZSET,+ LAST_KEY(entry.name),entry.name,String(scheduledAt),String(entry.runHashTtlS),id,String(Date.now()),- String(eligibleAt)+ String(eligibleAt),+ Date.now() - scheduledAt <= JUST_DUE_ENQUEUE_TICKS * enqueueIntervalMs ? "1" : "0");- if (initialized)+ if (result === EnqueueResult.Enqueued)logger.info(`cron[${entry.name}]: enqueued run [id=${id}] [scheduled_at=${new Date(scheduledAt).toISOString()}] [eligible_at=${new Date(eligibleAt).toISOString()}]`);+ if (result === EnqueueResult.SkippedFirstFire)+ logger.info(+ `cron[${entry.name}]: first registration, skipped the fire due before it, next fire is the first run [scheduled_at=${new Date(+ scheduledAt+ ).toISOString()}] [next_fire=${new Date(nextFireMs(entry.pattern)).toISOString()}]`+ );lastEnqueuedAt.set(entry.name, scheduledAt);}};