Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

refactor(cron): clean up comments and improve code clarity

Victor Hugo dos Santos7c0f685bb8d8
diff --git a/backend/src/lib/cron/cron-job.ts b/backend/src/lib/cron/cron-job.ts
index c1cca36197..9b602e0b96 100644
--- a/backend/src/lib/cron/cron-job.ts
+++ b/backend/src/lib/cron/cron-job.ts
@@ -6,8 +6,6 @@ import { Cluster, Redis } from "ioredis";
import { logger } from "@app/lib/logger";
import { ExecutionError, Redlock, ResourceLockedError } from "@app/lib/red-lock";
-// ── cron job name registry ────────────────────────────────────────────────────
-
export const CronJobName = {
HealthAlert: "health-alert",
KmsRootKeyCleanup: "kms-root-key-cleanup",
@@ -53,25 +51,11 @@ export const CronJobName = {
InstanceUpdateCheck: "instance-update-check"
} as const;
-// ── tuning constants ──────────────────────────────────────────────────────────
-
-// Jitter is derived from the pattern, never configured per job: how long a job may be held
-// back should follow how often it runs. A job's window is this fraction of its own interval,
-// capped by `maxJitterMs`. A fraction below 1 is what makes the offset safe by construction,
-// so it is not a free parameter:
-// - a run can never reach its own next fire, which `fitsBeforeNextFire` relies on;
-// - the first retry always fits. Worst case is 0.25 x interval + 30s backoff + 1s, under
-// `interval` for any interval above ~41s, and cron cannot fire faster than every 60s.
-// Raising this past ~0.48 would start costing short-interval jobs their retry.
-const JITTER_INTERVAL_FRACTION = 0.25;
-
+const JITTER_INTERVAL_FRACTION = 0.25; // keep below ~0.48 so 30s retry fits before next 1-min fire
const CRON_FIELD_COUNT = 5;
const PARTICIPANT_SLOTS = 5;
const PROCESS_BATCH_SIZE = 50;
-// Safety buffer in `fitsBeforeNextFire`: if the retry's nextAttemptAt is within
-// this many ms of the next scheduled fire, treat the run as final and let the
-// next fire (separate id) be the natural retry instead.
-const NEXT_FIRE_BUFFER_MS = 1_000;
+const NEXT_FIRE_BUFFER_MS = 1_000; // shared margin for fitsBeforeNextFire and jitterWindowMs
const ERROR_MESSAGE_MAX_LEN = 4_000;
const DEFAULTS = {
@@ -89,15 +73,8 @@ const DEFAULTS = {
maxConcurrentHandlers: 4
} as const;
-// ── redis schema ──────────────────────────────────────────────────────────────
-
-// Every key this module writes lives under a single Redis Cluster hash tag so
-// multi-key Lua scripts never return CROSSSLOT. A custom `keyPrefix` must keep
-// that property — see `assertHashTagged`.
const KEY_HASH_TAG = "{cron}";
-// Run-hash status values. Stored as plain strings in the hash so we don't
-// break Redis tooling, but referenced through this object to avoid drift.
const RunStatus = {
Pending: "pending",
Running: "running",
@@ -118,12 +95,7 @@ const F = {
NextAttemptAt: "next_attempt_at"
} as const;
-// ── lua scripts ───────────────────────────────────────────────────────────────
-
-// Atomic "enqueue this run hash + zset entry if no other pod beat us to it".
-// ARGV[2] is the unjittered scheduled fire, recorded on the hash as the run's
-// identity. ARGV[6] is the jittered eligibility time, and the zset is scored by
-// it so processTick's zrangebyscore holds the run back until its offset elapses.
+// ARGV[2] = unjittered scheduled_at; ARGV[6] = eligibleAt (pending zset score).
const ENQUEUE_RUN_LUA = `
if redis.call('exists', KEYS[1]) == 0 then
redis.call('hset', KEYS[1],
@@ -139,11 +111,9 @@ const ENQUEUE_RUN_LUA = `
return 0
`;
-// "DEL only if I still own this slot". Stops a stale stop() call from
-// accidentally evicting the new owner after our TTL expired.
+// Release slot only if we still own it (stale stop() after TTL expiry).
const RELEASE_SLOT_IF_MINE_LUA = `if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end`;
-// ── types ─────────────────────────────────────────────────────────────────────
type Handler = () => Promise<void>;
type CronEntry = {
@@ -162,9 +132,7 @@ class HandlerTimeoutError extends Error {
}
}
-// A prefix without a `{...}` hash tag would spread this module's keys across
-// Cluster slots and break every multi-key EVAL. Fail loudly at construction
-// rather than silently at the first enqueue tick.
+// Multi-key EVAL requires all keys in one Redis Cluster slot.
const assertHashTagged = (keyPrefix: string) => {
const start = keyPrefix.indexOf("{");
const end = keyPrefix.indexOf("}", start + 1);
@@ -175,8 +143,6 @@ const assertHashTagged = (keyPrefix: string) => {
export type TCronJobFactory = ReturnType<typeof cronJobFactory>;
-// ── factory ───────────────────────────────────────────────────────────────────
-
export const cronJobFactory = ({
redis,
redlock,
@@ -207,28 +173,13 @@ export const cronJobFactory = ({
retryBackoffBaseMs?: number;
retryBackoffMaxMs?: number;
drainTimeoutMs?: number;
- /**
- * Ceiling on the derived jitter window. Set it to 0 to disable
- * jitter, which is what development wiring does so a job runs when its pattern says.
- */
+ /** 0 disables jitter (dev wiring). */
maxJitterMs?: number;
- /**
- * Ceiling on how many handlers this pod runs at once. At capacity the pod stops claiming for
- * the tick and leaves the runs in the pending zset, so another pod or a later tick picks them
- * up; nothing is dropped.
- */
+ /** In-flight handler ceiling per pod; excess stays in the pending zset. */
maxConcurrentHandlers?: number;
- /**
- * Namespace for every Redis key this manager owns. Defaults to the production
- * `{cron}` namespace. Tests override it so a test-owned manager and the
- * server's real one can share a Redis without colliding on slot keys.
- */
+ /** Must include a Redis Cluster hash tag; tests override to avoid slot collisions. */
keyPrefix?: string;
- /**
- * Whether this pod runs cron handlers at all. When false, `register` is a no-op, so a pod that
- * never starts the timers also never holds the registry. Defaults to true so tests and any
- * caller that only wants the manager keep the previous behaviour.
- */
+ /** When false, register is a no-op (api-only pods). */
schedulingEnabled?: boolean;
}) => {
assertHashTagged(keyPrefix);
@@ -249,10 +200,7 @@ export const cronJobFactory = ({
}
const workerId = randomUUID();
- // A saturated pod defers on every tick, so logging per tick would emit a line every
- // `processIntervalMs` for as long as the backlog lasts. Throttled so a sustained backlog
- // stays visible without burying every other cron line.
- const CAPACITY_LOG_INTERVAL_MS = 60_000;
+ const CAPACITY_LOG_INTERVAL_MS = 60_000; // throttle capacity logs during sustained backlog
let lastCapacityLogAtMs = 0;
const entries = new Map<string, CronEntry>();
const lastEnqueuedAt = new Map<string, number>();
@@ -262,42 +210,21 @@ export const cronJobFactory = ({
let processTimer: ReturnType<typeof setInterval> | null = null;
let currentSlot: number | null = null;
let stopped = false;
- // Tail of the serialized slot-operation chain. Never rejects.
- let slotOp: Promise<void> = Promise.resolve();
-
- // ── helpers ────────────────────────────────────────────────────────────
+ let slotOp: Promise<void> = Promise.resolve(); // serialized slot claim chain
const prevFireMs = (pattern: string) => CronExpressionParser.parse(pattern, { tz: "UTC" }).prev().toDate().getTime();
const nextFireMs = (pattern: string) => CronExpressionParser.parse(pattern, { tz: "UTC" }).next().toDate().getTime();
- // How far past its scheduled fire a job with this interval may be held back. Proportional
- // to the interval so a frequent job is delayed proportionally less than a rare one, capped
- // so a daily job is not spread across the whole day. NEXT_FIRE_BUFFER_MS is belt-and-braces:
- // JITTER_INTERVAL_FRACTION already keeps the window well inside the interval, but the clamp
- // means the next-fire property survives someone raising the fraction.
const jitterWindowMs = (intervalMs: number) =>
Math.min(intervalMs * JITTER_INTERVAL_FRACTION, maxJitterMs, Math.max(0, intervalMs - NEXT_FIRE_BUFFER_MS));
- // Per-job offset into the jitter window, derived from a stable hash of the job
- // name. It MUST NOT be random: the run id is keyed on the scheduled fire time
- // and every pod computes a run's eligibility time independently, so a per-pod offset
- // would make pods disagree on run identity and break the enqueue dedup and the
- // lease logic. Hashing also spreads the jobs permanently, where random offsets
- // would re-collide on the next fire.
+ // Hash, not random: every pod must derive the same eligibleAt for dedup/leases.
const jitterOffsetMs = (name: string, windowMs: number) => {
if (windowMs <= 0) return 0;
- // Six bytes keeps the value inside Number.MAX_SAFE_INTEGER.
return createHash("sha256").update(name).digest().readUIntBE(0, 6) % windowMs;
};
- // The earliest a fire may be picked up: its scheduled time plus the job's
- // offset. Distinct from the run hash's `started_at`, which is when a handler
- // actually began. The window is clamped so an offset can never push a run to
- // or past its own next fire: the two would then become eligible in the wrong
- // order, and `fitsBeforeNextFire`, which measures a retry against the raw next
- // fire, would stop bounding the retry correctly. NEXT_FIRE_BUFFER_MS is the
- // same margin that check uses.
const eligibleAtMs = (entry: CronEntry, scheduledAt: number, nextFire: number) => {
const windowMs = jitterWindowMs(nextFire - scheduledAt);
return scheduledAt + jitterOffsetMs(entry.name, windowMs);
@@ -313,10 +240,7 @@ export const cronJobFactory = ({
}
};
- // Races `task` against a timeout. If the timeout wins, throws a
- // HandlerTimeoutError so executeUnderLease's catch can mark the run
- // failed-final (rather than pending-retry) and prevent another pod from
- // running the handler concurrently with the still-executing zombie.
+ // Timeout → failed-final, not retry (handler may still be running as a zombie).
const withTimeout = async (task: () => Promise<void>, timeoutMs: number) => {
let timer: ReturnType<typeof setTimeout> | undefined;
const timeout = new Promise<never>((_, reject) => {
@@ -333,8 +257,6 @@ export const cronJobFactory = ({
}
};
- // ── run-hash status writes ──────────────────────────────────────────────────
-
const markRunning = (id: string, attempt: number) =>
redis.hset(
RUN_KEY(id),
@@ -353,8 +275,6 @@ export const cronJobFactory = ({
await redis.zrem(PENDING_ZSET, id);
};
- // `err` is optional: the max-attempts-reached path writes a "failed" status
- // without a last_error.
const markFailedFinal = async (id: string, err?: unknown) => {
if (err === undefined) {
await redis.hset(RUN_KEY(id), F.Status, RunStatus.Failed, F.CompletedAt, String(Date.now()));
@@ -382,15 +302,9 @@ export const cronJobFactory = ({
F.NextAttemptAt,
String(nextAttemptAt)
);
- // zset score = next_attempt_at so processTick's zrangebyscore filter skips
- // this run until its backoff window elapses.
await redis.zadd(PENDING_ZSET, nextAttemptAt, id);
};
- // ── registration ────────────────────────────────────────────────────────────
-
- // Registers a cron entry with this pod. `runHashTtlS` controls how long each
- // run's hash lives in Redis after enqueue.
const register = ({
name,
pattern,
@@ -452,11 +366,6 @@ export const cronJobFactory = ({
logger.info(`cron[${name}]: registered (pattern="${pattern}")`);
};
- // ── slot election ───────────────────────────────────────────────────────────
-
- // Caps the number of pods that participate in cron ticking. Each pod holds
- // one of N slots via SET NX/XX with a short TTL: refreshes its own slot if
- // it still owns one, otherwise tries to claim a free slot.
const claimOrRefreshSlot = async () => {
if (currentSlot !== null) {
const ok = await redis.set(SLOT_KEY(currentSlot), workerId, "PX", slotTtlMs, "XX");
@@ -475,17 +384,7 @@ export const cronJobFactory = ({
}
};
- // Runs slot claims/refreshes strictly one at a time.
- //
- // `claimOrRefreshSlot` awaits between reading `currentSlot` and writing it,
- // so two overlapping ticks could both observe a lost slot, both null
- // `currentSlot`, and then claim two *different* slots — the pod would burn
- // two of the five participant slots and leak one on shutdown, since only the
- // last-assigned `currentSlot` is ever released.
- //
- // Chaining also gives stop() one handle to await, so it can never read
- // `currentSlot` mid-handover (skipping the release) or have a late claim land
- // after the release.
+ // Serialize slot ops: overlapping ticks could claim two slots and leak one on shutdown.
const runSlotOp = (label: string) => {
slotOp = slotOp
.then(() => (stopped ? undefined : claimOrRefreshSlot()))
@@ -493,16 +392,7 @@ export const cronJobFactory = ({
return slotOp;
};
- // ── enqueue ─────────────────────────────────────────────────────────────────
-
- // For each registered entry, computes the most recent scheduled fire time
- // and atomically inserts a run hash + pending-zset entry if not already
- // present. The Lua script makes the existence check + write a single op so
- // multiple pods racing to enqueue the same fire produce exactly one run.
- //
- // The run id and the hash's `scheduled_at` stay keyed on the unjittered fire,
- // so run identity is exactly what it was before jitter existed. Only the zset
- // score carries the offset, which is what holds the run back from processTick.
+ // Run id uses unjittered scheduledAt; pending zset score carries eligibleAt.
const enqueueDueFires = async () => {
for (const entry of entries.values()) {
const scheduledAt = prevFireMs(entry.pattern);
@@ -534,26 +424,12 @@ export const cronJobFactory = ({
}
};
- // Slow timer (~30s) that only creates run hashes for due fires. Split from
- // processing so the pod that enqueues doesn't immediately consume everything
- // it just created — leaves time for other slot-holders' process ticks to race.
const enqueueTick = async () => {
if (currentSlot === null) return;
await enqueueDueFires();
};
- // ── process ─────────────────────────────────────────────────────────────────
-
- // Executes one attempt of `entry` under an already-acquired redlock. Writes
- // the running state, races the handler against handlerTimeoutMs, and
- // branches into success / retry / final failure on completion.
- //
- // Two-knob recovery model:
- // leaseDurationMs → governs CRASH recovery. When the holder pod dies,
- // its lease key TTLs out and other pods see the run as
- // stalled via the isStalled path in processCandidate.
- // handlerTimeoutMs → bounds the LEASE for HANG recovery. On timeout we release the lease and
- // mark the run failed-final — a retry would race with the still-running zombie handler. The next scheduled fire is the natural retry.
+ // leaseDurationMs: crash recovery via stalled reclaim. handlerTimeoutMs: hang → failed-final.
const executeUnderLease = async (entry: CronEntry, id: string, attempt: number) => {
try {
await markRunning(id, attempt);
@@ -572,13 +448,8 @@ export const cronJobFactory = ({
} catch (err) {
const isHandlerTimeout = err instanceof HandlerTimeoutError;
- // Exponential backoff: 1×, 2×, 4× ... capped at retryBackoffMaxMs.
const backoffMs = Math.min(retryBackoffBaseMs * 2 ** (attempt - 1), retryBackoffMaxMs);
const nextAttemptAt = Date.now() + backoffMs;
- // If the retry wouldn't fit before the next scheduled fire, treat this
- // run as final — the upcoming fire (separate id) is the natural retry.
- // Avoids overlapping attempts of the same cron and avoids squashing
- // backoff to ~0 near the end of an interval.
const fitsBeforeNextFire = nextAttemptAt + NEXT_FIRE_BUFFER_MS < nextFireMs(entry.pattern);
const final = isHandlerTimeout || attempt >= entry.maxAttempts || !fitsBeforeNextFire;
@@ -603,10 +474,6 @@ export const cronJobFactory = ({
}
};
- // Inspects one pending zset id: validates the run hash, applies the
- // min-age and max-attempts gates, and races for the lease. Anything that
- // can be decided without taking a lock is decided here so we only ever
- // hand a "ready, attempt-eligible" run to executeUnderLease.
const processCandidate = async (id: string) => {
const data = await redis.hgetall(RUN_KEY(id));
if (!data?.name) {
@@ -618,15 +485,13 @@ export const cronJobFactory = ({
}
const entry = entries.get(data.name);
- if (!entry) return; // not handled on this pod
+ if (!entry) return;
const isPending = data.status === RunStatus.Pending;
const isStalled = data.status === RunStatus.Running && (await redis.exists(LEASE_KEY(id))) === 0;
if (!isPending && !isStalled) return;
- // Min-age delay on first pickup so the pod that enqueued doesn't
- // immediately grab everything before other slot-holders' process ticks
- // fire. Stalled runs bypass this — they're already past first pickup.
+ // Brief delay so other slot-holders can race for freshly enqueued runs.
if (isPending) {
const enqueuedAtMs = Number(data[F.EnqueuedAtMs] ?? 0);
if (Date.now() - enqueuedAtMs < minProcessAgeMs) return;
@@ -645,9 +510,6 @@ export const cronJobFactory = ({
logger.info(`cron[${data.name}]: re-claiming stalled run [id=${id}] [previous_worker=${data.worker_id}]`);
}
- // Track the in-flight run so stop() can wait for it to settle before
- // tearing down. Lock-contention rejections settle within a tick, so they
- // don't measurably delay shutdown drain.
const tracked = redlock.using([LEASE_KEY(id)], entry.leaseDurationMs ?? leaseDurationMs, () =>
executeUnderLease(entry, id, attempts + 1)
);
@@ -655,7 +517,6 @@ export const cronJobFactory = ({
try {
await tracked;
} catch (err) {
- // ExecutionError / ResourceLockedError mean another pod owns the lease — expected contention, swallow.
if (err instanceof ExecutionError || err instanceof ResourceLockedError) return;
logger.error({ err }, `cron[${data.name}]: unexpected error acquiring lease [id=${id}]`);
} finally {
@@ -663,15 +524,9 @@ export const cronJobFactory = ({
}
};
- // Fast timer (~5s) that scans the pending zset and races for runs to
- // execute. Ids are shuffled per tick so different slot-holders try
- // different ids first, spreading lease contention and distributing handler
- // load across pods.
const processTick = async () => {
if (currentSlot === null) return;
- // Filter by zset score so backed-off retries (score = next_attempt_at)
- // are skipped until their backoff window elapses.
const ids = await redis.zrangebyscore(PENDING_ZSET, "-inf", Date.now(), "LIMIT", 0, PROCESS_BATCH_SIZE);
shuffleInPlace(ids);
@@ -697,15 +552,10 @@ export const cronJobFactory = ({
}
};
- // ── lifecycle ───────────────────────────────────────────────────────────────
-
const safeTick = (label: string, fn: () => Promise<void>) => () => {
fn().catch((err: unknown) => logger.error({ err }, `cron: ${label} failed`));
};
- // Starts the slot-refresh, enqueue, and process timers, and claims a slot
- // immediately so the pod doesn't wait a full `slotRefreshMs` before
- // participating.
const start = () => {
stopped = false;
slotTimer = setInterval(() => void runSlotOp("slot refresh"), slotRefreshMs);
@@ -714,13 +564,7 @@ export const cronJobFactory = ({
void runSlotOp("initial slot claim");
};
- // Stops the timers, drains in-flight handlers, and atomically releases the
- // held slot (only if we still own it) so another pod can take over without
- // waiting for the slot TTL.
- //
- // Drain semantics wait for the currently active runs to finish so destructive handlers (rotations, deletions)
- // aren't aborted mid-execution and graceful redeploys don't leave runs
- // stuck in `status='running'` until the lease TTL elapses.
+ // Drain in-flight handlers before releasing the slot so runs are not left stuck as running.
const stop = async () => {
stopped = true;
if (slotTimer) clearInterval(slotTimer);
@@ -746,9 +590,6 @@ export const cronJobFactory = ({
}
}
- // No new slot ops can be queued (timers cleared, `stopped` set), so this is
- // the final tail. Waiting on it guarantees `currentSlot` is settled and that
- // nothing can re-create the key after the release below.
await slotOp;
if (currentSlot !== null) {