Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

fix(cron): dedup fires on a per-job key and skip a new job's first due fire

Calvin180714f7308c
diff --git a/backend/e2e-test/cron-job.spec.ts b/backend/e2e-test/cron-job.spec.ts
index d17048d702..fe3ae75970 100644
--- a/backend/e2e-test/cron-job.spec.ts
+++ b/backend/e2e-test/cron-job.spec.ts
@@ -23,6 +23,10 @@ const KEY_PREFIX = "{cron-e2e}";
const SLOT_KEYS = `${KEY_PREFIX}:slot:*`;
const RUN_KEYS = (name: string) => `${KEY_PREFIX}:run:${name}:*`;
const PENDING_ZSET = `${KEY_PREFIX}:pending`;
+const LAST_KEY = (name: string) => `${KEY_PREFIX}:last:${name}`;
+
+// waitForFreshMinute keeps every test clear of a boundary, so this is FAST_PATTERN's latest fire.
+const currentFireMs = () => Math.floor(Date.now() / 60_000) * 60_000;
const clearNamespace = async () => {
const keys = await testRedis.keys(`${KEY_PREFIX}:*`);
@@ -92,12 +96,29 @@ const makeFactory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>)
const FAST_PATTERN = "* * * * *"; // fires every minute
const allFactories: ReturnType<typeof makeFactory>[] = [];
-const factory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>) => {
+// The manager skips the latest fire of a job that has never fired, so by default every job a
+// test registers is marked as having fired in the previous minute, which lets it run in the
+// current one. NX keeps a value an earlier factory in the same test already wrote. The first
+// enqueue tick runs enqueueIntervalMs after start(), long after this write lands.
+const factory = (
+ overrides?: Partial<Parameters<typeof cronJobFactory>[0]>,
+ { existingJobs = true }: { existingJobs?: boolean } = {}
+) => {
const f = makeFactory(overrides);
allFactories.push(f);
- return f;
+ return {
+ ...f,
+ register: (entry: Parameters<typeof f.register>[0]) => {
+ f.register(entry);
+ if (existingJobs) void testRedis.set(LAST_KEY(entry.name), String(currentFireMs() - 60_000), "NX");
+ }
+ };
};
+// For tests about a job's very first registration.
+const newJobFactory = (overrides?: Partial<Parameters<typeof cronJobFactory>[0]>) =>
+ factory(overrides, { existingJobs: false });
+
// A handler that never settles on its own, simulating a wedged network call.
// The resolver is parked so afterEach can release it: an abandoned handler
// leaves `withTimeout`'s race unsettled, so its `finally` never runs and its
@@ -493,6 +514,92 @@ describe("jitter", () => {
}, 10_000);
});
+// ── dedup across restarts ──────────────────────────────────────────────────────
+
+describe("dedup across restarts", () => {
+ const sleep = (ms: number) =>
+ new Promise((r) => {
+ setTimeout(r, ms);
+ });
+ // Only Date is faked, so the manager's timers keep running in real time while every enqueue
+ // tick sees the same distance from the fire. afterEach restores the real clock.
+ const freezeClockAfterFire = (ms: number) => {
+ const fire = currentFireMs();
+ vi.useFakeTimers({ toFake: ["Date"] });
+ vi.setSystemTime(fire + ms);
+ return fire;
+ };
+
+ test("a restart after the run hash expires does not run the same fire again", async () => {
+ const first = vi.fn().mockResolvedValue(undefined);
+ const f1 = factory();
+ f1.register({ name: "restart-job", pattern: FAST_PATTERN, handler: first, runHashTtlS: 3600 });
+ f1.start();
+ await sleep(2000);
+ expect(first).toHaveBeenCalledTimes(1);
+ await f1.stop();
+
+ const runKeys = await testRedis.keys(RUN_KEYS("restart-job"));
+ await testRedis.del(...runKeys);
+
+ const second = vi.fn().mockResolvedValue(undefined);
+ const f2 = factory();
+ f2.register({ name: "restart-job", pattern: FAST_PATTERN, handler: second, runHashTtlS: 3600 });
+ f2.start();
+ await sleep(1500);
+
+ expect(second).not.toHaveBeenCalled();
+ expect(await testRedis.keys(RUN_KEYS("restart-job"))).toEqual([]);
+ }, 10_000);
+
+ test("a run hash written without a last value is recorded instead of enqueued again", async () => {
+ const scheduledAt = currentFireMs();
+ await testRedis.hset(
+ `${KEY_PREFIX}:run:backfill-job:${scheduledAt}`,
+ "name",
+ "backfill-job",
+ "status",
+ "completed"
+ );
+
+ const handler = vi.fn().mockResolvedValue(undefined);
+ const f = newJobFactory();
+ f.register({ name: "backfill-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });
+ f.start();
+ await sleep(1500);
+
+ expect(handler).not.toHaveBeenCalled();
+ expect(await testRedis.get(LAST_KEY("backfill-job"))).toBe(String(scheduledAt));
+ }, 10_000);
+
+ test("a job that has never fired skips the fire due before it and waits for the next one", async () => {
+ const fire = freezeClockAfterFire(30_000);
+ const handler = vi.fn().mockResolvedValue(undefined);
+ const f = newJobFactory();
+ f.register({ name: "new-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });
+ f.start();
+ await sleep(1500);
+
+ expect(handler).not.toHaveBeenCalled();
+ expect(await testRedis.keys(RUN_KEYS("new-job"))).toEqual([]);
+ expect(await testRedis.get(LAST_KEY("new-job"))).toBe(String(fire));
+ }, 10_000);
+
+ // Jobs that ran before the last key existed have no last value either. When the first deploy
+ // that writes it lands just after a boundary, no pod may have enqueued that fire yet, and
+ // treating the job as new would silently drop it.
+ test("a job with no last value still runs a fire that only just became due", async () => {
+ freezeClockAfterFire(100);
+ const handler = vi.fn().mockResolvedValue(undefined);
+ const f = newJobFactory();
+ f.register({ name: "boundary-job", pattern: FAST_PATTERN, handler, runHashTtlS: 3600 });
+ f.start();
+ await sleep(2000);
+
+ expect(handler).toHaveBeenCalledTimes(1);
+ }, 10_000);
+});
+
// ── graceful shutdown ──────────────────────────────────────────────────────────
describe("graceful shutdown", () => {
diff --git a/backend/src/lib/cron/cron-job.test.ts b/backend/src/lib/cron/cron-job.test.ts
index d990d1dbf6..5073fc4f82 100644
--- a/backend/src/lib/cron/cron-job.test.ts
+++ b/backend/src/lib/cron/cron-job.test.ts
@@ -425,9 +425,9 @@ describe("redis cluster compatibility", () => {
// Drive at least one enqueue tick (enqueueIntervalMs=100 in makeFactory).
await vi.advanceTimersByTimeAsync(150);
- // ENQUEUE_RUN_LUA is the only multi-key script (KEYS[1]=run, KEYS[2]=pending).
- // Identify its eval call by numKeys === 2 (RELEASE_SLOT_IF_MINE_LUA uses 1).
- const multiKeyEvalCalls = redis.eval.mock.calls.filter((c) => c[1] === 2);
+ // ENQUEUE_RUN_LUA is the only multi-key script (KEYS[1]=run, KEYS[2]=pending, KEYS[3]=last).
+ // Identify its eval call by numKeys === 3 (RELEASE_SLOT_IF_MINE_LUA uses 1).
+ const multiKeyEvalCalls = redis.eval.mock.calls.filter((c) => c[1] === 3);
expect(multiKeyEvalCalls.length).toBeGreaterThan(0);
for (const call of multiKeyEvalCalls) {
@@ -764,16 +764,16 @@ describe("derived jitter", () => {
const expectedWindowMs = (intervalMs: number, maxJitterMs = MAX_JITTER_MS) =>
Math.min(intervalMs * FRACTION, maxJitterMs, intervalMs - NEXT_FIRE_BUFFER_MS);
- // ENQUEUE_RUN_LUA is the only two-key script:
- // eval(script, 2, runKey, pendingZset, name, scheduledAt, ttl, id, enqueuedAtMs, eligibleAt)
+ // ENQUEUE_RUN_LUA is the only three-key script:
+ // eval(script, 3, runKey, pendingZset, lastKey, name, scheduledAt, ttl, id, enqueuedAtMs, eligibleAt, justBecameDue)
const enqueuedRuns = (redis: ReturnType<typeof makeRedis>) =>
redis.eval.mock.calls
- .filter((c) => c[1] === 2)
+ .filter((c) => c[1] === 3)
.map((c) => ({
- name: String(c[4]),
- scheduledAt: Number(c[5]),
- id: String(c[7]),
- eligibleAt: Number(c[9])
+ name: String(c[5]),
+ scheduledAt: Number(c[6]),
+ id: String(c[8]),
+ eligibleAt: Number(c[10])
}));
// Boots a manager, drives one enqueue tick, and returns what it wrote.
diff --git a/backend/src/lib/cron/cron-job.ts b/backend/src/lib/cron/cron-job.ts
index d86860aab2..46594639ea 100644
--- a/backend/src/lib/cron/cron-job.ts
+++ b/backend/src/lib/cron/cron-job.ts
@@ -61,6 +61,8 @@ const PARTICIPANT_SLOTS = 5;
const PROCESS_BATCH_SIZE = 50;
const NEXT_FIRE_BUFFER_MS = 1_000; // shared margin for fitsBeforeNextFire and jitterWindowMs
const ERROR_MESSAGE_MAX_LEN = 4_000;
+// A fire due at most this many enqueue ticks ago counts as just due: pods may not have enqueued it yet.
+const JUST_DUE_ENQUEUE_TICKS = 2;
const DEFAULTS = {
slotTtlMs: 60_000,
@@ -98,20 +100,49 @@ const F = {
NextAttemptAt: "next_attempt_at"
} as const;
-// ARGV[2] = unjittered scheduled_at; ARGV[6] = eligibleAt (pending zset score).
+const EnqueueResult = {
+ AlreadyEnqueued: 0,
+ Enqueued: 1,
+ SkippedFirstFire: 2
+} as const;
+
+// Dedup checks "lastKey" rather than the run hash, because the run hash expires and "lastKey" does not.
const ENQUEUE_RUN_LUA = `
- if redis.call('exists', KEYS[1]) == 0 then
- redis.call('hset', KEYS[1],
- 'name', ARGV[1],
- 'scheduled_at', ARGV[2],
- 'status', 'pending',
- 'attempts', 0,
- 'enqueued_at_ms', ARGV[5])
- redis.call('expire', KEYS[1], ARGV[3])
- redis.call('zadd', KEYS[2], ARGV[6], ARGV[4])
- return 1
+ local runKey, pendingZset, lastKey = KEYS[1], KEYS[2], KEYS[3]
+ local name, scheduledAt, runHashTtlS, runId, enqueuedAtMs, eligibleAt, justBecameDue =
+ ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6], ARGV[7] == '1'
+
+ local last = tonumber(redis.call('get', lastKey))
+ if last and last >= tonumber(scheduledAt) then
+ return ${EnqueueResult.AlreadyEnqueued}
end
- return 0
+
+ -- Backfill only, not scheduling logic. Pods without "lastKey" create the run hash but never
+ -- set "lastKey". If the run hash exists, the fire is already enqueued, so just record it.
+ -- Remove once every deployment sets "lastKey".
+ if redis.call('exists', runKey) == 1 then
+ redis.call('set', lastKey, scheduledAt)
+ return ${EnqueueResult.AlreadyEnqueued}
+ end
+
+ -- No "last" means the job has never fired. Record this fire without running it, so the next
+ -- fire is the first run. A fire that "justBecameDue" still runs: on the first deploy of
+ -- "lastKey", existing jobs have no "last" either, and no pod may have enqueued this fire yet.
+ if not last and not justBecameDue then
+ redis.call('set', lastKey, scheduledAt)
+ return ${EnqueueResult.SkippedFirstFire}
+ end
+
+ redis.call('hset', runKey,
+ 'name', name,
+ 'scheduled_at', scheduledAt,
+ 'status', 'pending',
+ 'attempts', 0,
+ 'enqueued_at_ms', enqueuedAtMs)
+ redis.call('expire', runKey, runHashTtlS)
+ redis.call('zadd', pendingZset, eligibleAt, runId)
+ redis.call('set', lastKey, scheduledAt)
+ return ${EnqueueResult.Enqueued}
`;
// Release slot only if we still own it (stale stop() after TTL expiry).
@@ -195,6 +226,7 @@ export const cronJobFactory = ({
const SLOT_KEY = (i: number) => `${keyPrefix}:slot:${i}`;
const RUN_KEY = (id: string) => `${keyPrefix}:run:${id}`;
const LEASE_KEY = (id: string) => `${keyPrefix}:lease:${id}`;
+ const LAST_KEY = (name: string) => `${keyPrefix}:last:${name}`;
const PENDING_ZSET = `${keyPrefix}:pending`;
if (!schedulingEnabled) {
@@ -401,24 +433,32 @@ export const cronJobFactory = ({
const id = `${entry.name}:${scheduledAt}`;
const eligibleAt = eligibleAtMs(entry, scheduledAt, nextFireMs(entry.pattern));
// eslint-disable-next-line no-await-in-loop
- const initialized = await redis.eval(
+ const result = await redis.eval(
ENQUEUE_RUN_LUA,
- 2,
+ 3,
RUN_KEY(id),
PENDING_ZSET,
+ LAST_KEY(entry.name),
entry.name,
String(scheduledAt),
String(entry.runHashTtlS),
id,
String(Date.now()),
- String(eligibleAt)
+ String(eligibleAt),
+ Date.now() - scheduledAt <= JUST_DUE_ENQUEUE_TICKS * enqueueIntervalMs ? "1" : "0"
);
- if (initialized)
+ if (result === EnqueueResult.Enqueued)
logger.info(
`cron[${entry.name}]: enqueued run [id=${id}] [scheduled_at=${new Date(
scheduledAt
).toISOString()}] [eligible_at=${new Date(eligibleAt).toISOString()}]`
);
+ if (result === EnqueueResult.SkippedFirstFire)
+ logger.info(
+ `cron[${entry.name}]: first registration, skipped the fire due before it, next fire is the first run [scheduled_at=${new Date(
+ scheduledAt
+ ).toISOString()}] [next_fire=${new Date(nextFireMs(entry.pattern)).toISOString()}]`
+ );
lastEnqueuedAt.set(entry.name, scheduledAt);
}
};