Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
refactor(cron): clean up comments and improve code clarity
Victor Hugo dos Santos7c0f685bb8d8
diff --git a/backend/src/lib/cron/cron-job.ts b/backend/src/lib/cron/cron-job.tsindex c1cca36197..9b602e0b96 100644--- a/backend/src/lib/cron/cron-job.ts+++ b/backend/src/lib/cron/cron-job.ts@@ -6,8 +6,6 @@ import { Cluster, Redis } from "ioredis";import { logger } from "@app/lib/logger";import { ExecutionError, Redlock, ResourceLockedError } from "@app/lib/red-lock";-// ── cron job name registry ────────────────────────────────────────────────────-export const CronJobName = {HealthAlert: "health-alert",KmsRootKeyCleanup: "kms-root-key-cleanup",@@ -53,25 +51,11 @@ export const CronJobName = {InstanceUpdateCheck: "instance-update-check"} as const;-// ── tuning constants ──────────────────────────────────────────────────────────--// Jitter is derived from the pattern, never configured per job: how long a job may be held-// back should follow how often it runs. A job's window is this fraction of its own interval,-// capped by `maxJitterMs`. A fraction below 1 is what makes the offset safe by construction,-// so it is not a free parameter:-// - a run can never reach its own next fire, which `fitsBeforeNextFire` relies on;-// - the first retry always fits. Worst case is 0.25 x interval + 30s backoff + 1s, under-// `interval` for any interval above ~41s, and cron cannot fire faster than every 60s.-// Raising this past ~0.48 would start costing short-interval jobs their retry.-const JITTER_INTERVAL_FRACTION = 0.25;-+const JITTER_INTERVAL_FRACTION = 0.25; // keep below ~0.48 so 30s retry fits before next 1-min fireconst CRON_FIELD_COUNT = 5;const PARTICIPANT_SLOTS = 5;const PROCESS_BATCH_SIZE = 50;-// Safety buffer in `fitsBeforeNextFire`: if the retry's nextAttemptAt is within-// this many ms of the next scheduled fire, treat the run as final and let the-// next fire (separate id) be the natural retry instead.-const NEXT_FIRE_BUFFER_MS = 1_000;+const NEXT_FIRE_BUFFER_MS = 1_000; // shared margin for fitsBeforeNextFire and jitterWindowMsconst ERROR_MESSAGE_MAX_LEN = 4_000;const DEFAULTS = {@@ -89,15 +73,8 @@ const DEFAULTS = {maxConcurrentHandlers: 4} as const;-// ── redis schema ──────────────────────────────────────────────────────────────--// Every key this module writes lives under a single Redis Cluster hash tag so-// multi-key Lua scripts never return CROSSSLOT. A custom `keyPrefix` must keep-// that property — see `assertHashTagged`.const KEY_HASH_TAG = "{cron}";-// Run-hash status values. Stored as plain strings in the hash so we don't-// break Redis tooling, but referenced through this object to avoid drift.const RunStatus = {Pending: "pending",Running: "running",@@ -118,12 +95,7 @@ const F = {NextAttemptAt: "next_attempt_at"} as const;-// ── lua scripts ───────────────────────────────────────────────────────────────--// Atomic "enqueue this run hash + zset entry if no other pod beat us to it".-// ARGV[2] is the unjittered scheduled fire, recorded on the hash as the run's-// identity. ARGV[6] is the jittered eligibility time, and the zset is scored by-// it so processTick's zrangebyscore holds the run back until its offset elapses.+// ARGV[2] = unjittered scheduled_at; ARGV[6] = eligibleAt (pending zset score).const ENQUEUE_RUN_LUA = `if redis.call('exists', KEYS[1]) == 0 thenredis.call('hset', KEYS[1],@@ -139,11 +111,9 @@ const ENQUEUE_RUN_LUA = `return 0`;-// "DEL only if I still own this slot". Stops a stale stop() call from-// accidentally evicting the new owner after our TTL expired.+// Release slot only if we still own it (stale stop() after TTL expiry).const RELEASE_SLOT_IF_MINE_LUA = `if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end`;-// ── types ─────────────────────────────────────────────────────────────────────type Handler = () => Promise<void>;type CronEntry = {@@ -162,9 +132,7 @@ class HandlerTimeoutError extends Error {}}-// A prefix without a `{...}` hash tag would spread this module's keys across-// Cluster slots and break every multi-key EVAL. Fail loudly at construction-// rather than silently at the first enqueue tick.+// Multi-key EVAL requires all keys in one Redis Cluster slot.const assertHashTagged = (keyPrefix: string) => {const start = keyPrefix.indexOf("{");const end = keyPrefix.indexOf("}", start + 1);@@ -175,8 +143,6 @@ const assertHashTagged = (keyPrefix: string) => {export type TCronJobFactory = ReturnType<typeof cronJobFactory>;-// ── factory ───────────────────────────────────────────────────────────────────-export const cronJobFactory = ({redis,redlock,@@ -207,28 +173,13 @@ export const cronJobFactory = ({retryBackoffBaseMs?: number;retryBackoffMaxMs?: number;drainTimeoutMs?: number;- /**- * Ceiling on the derived jitter window. Set it to 0 to disable- * jitter, which is what development wiring does so a job runs when its pattern says.- */+ /** 0 disables jitter (dev wiring). */maxJitterMs?: number;- /**- * Ceiling on how many handlers this pod runs at once. At capacity the pod stops claiming for- * the tick and leaves the runs in the pending zset, so another pod or a later tick picks them- * up; nothing is dropped.- */+ /** In-flight handler ceiling per pod; excess stays in the pending zset. */maxConcurrentHandlers?: number;- /**- * Namespace for every Redis key this manager owns. Defaults to the production- * `{cron}` namespace. Tests override it so a test-owned manager and the- * server's real one can share a Redis without colliding on slot keys.- */+ /** Must include a Redis Cluster hash tag; tests override to avoid slot collisions. */keyPrefix?: string;- /**- * Whether this pod runs cron handlers at all. When false, `register` is a no-op, so a pod that- * never starts the timers also never holds the registry. Defaults to true so tests and any- * caller that only wants the manager keep the previous behaviour.- */+ /** When false, register is a no-op (api-only pods). */schedulingEnabled?: boolean;}) => {assertHashTagged(keyPrefix);@@ -249,10 +200,7 @@ export const cronJobFactory = ({}const workerId = randomUUID();- // A saturated pod defers on every tick, so logging per tick would emit a line every- // `processIntervalMs` for as long as the backlog lasts. Throttled so a sustained backlog- // stays visible without burying every other cron line.- const CAPACITY_LOG_INTERVAL_MS = 60_000;+ const CAPACITY_LOG_INTERVAL_MS = 60_000; // throttle capacity logs during sustained backloglet lastCapacityLogAtMs = 0;const entries = new Map<string, CronEntry>();const lastEnqueuedAt = new Map<string, number>();@@ -262,42 +210,21 @@ export const cronJobFactory = ({let processTimer: ReturnType<typeof setInterval> | null = null;let currentSlot: number | null = null;let stopped = false;- // Tail of the serialized slot-operation chain. Never rejects.- let slotOp: Promise<void> = Promise.resolve();-- // ── helpers ────────────────────────────────────────────────────────────+ let slotOp: Promise<void> = Promise.resolve(); // serialized slot claim chainconst prevFireMs = (pattern: string) => CronExpressionParser.parse(pattern, { tz: "UTC" }).prev().toDate().getTime();const nextFireMs = (pattern: string) => CronExpressionParser.parse(pattern, { tz: "UTC" }).next().toDate().getTime();- // How far past its scheduled fire a job with this interval may be held back. Proportional- // to the interval so a frequent job is delayed proportionally less than a rare one, capped- // so a daily job is not spread across the whole day. NEXT_FIRE_BUFFER_MS is belt-and-braces:- // JITTER_INTERVAL_FRACTION already keeps the window well inside the interval, but the clamp- // means the next-fire property survives someone raising the fraction.const jitterWindowMs = (intervalMs: number) =>Math.min(intervalMs * JITTER_INTERVAL_FRACTION, maxJitterMs, Math.max(0, intervalMs - NEXT_FIRE_BUFFER_MS));- // Per-job offset into the jitter window, derived from a stable hash of the job- // name. It MUST NOT be random: the run id is keyed on the scheduled fire time- // and every pod computes a run's eligibility time independently, so a per-pod offset- // would make pods disagree on run identity and break the enqueue dedup and the- // lease logic. Hashing also spreads the jobs permanently, where random offsets- // would re-collide on the next fire.+ // Hash, not random: every pod must derive the same eligibleAt for dedup/leases.const jitterOffsetMs = (name: string, windowMs: number) => {if (windowMs <= 0) return 0;- // Six bytes keeps the value inside Number.MAX_SAFE_INTEGER.return createHash("sha256").update(name).digest().readUIntBE(0, 6) % windowMs;};- // The earliest a fire may be picked up: its scheduled time plus the job's- // offset. Distinct from the run hash's `started_at`, which is when a handler- // actually began. The window is clamped so an offset can never push a run to- // or past its own next fire: the two would then become eligible in the wrong- // order, and `fitsBeforeNextFire`, which measures a retry against the raw next- // fire, would stop bounding the retry correctly. NEXT_FIRE_BUFFER_MS is the- // same margin that check uses.const eligibleAtMs = (entry: CronEntry, scheduledAt: number, nextFire: number) => {const windowMs = jitterWindowMs(nextFire - scheduledAt);return scheduledAt + jitterOffsetMs(entry.name, windowMs);@@ -313,10 +240,7 @@ export const cronJobFactory = ({}};- // Races `task` against a timeout. If the timeout wins, throws a- // HandlerTimeoutError so executeUnderLease's catch can mark the run- // failed-final (rather than pending-retry) and prevent another pod from- // running the handler concurrently with the still-executing zombie.+ // Timeout → failed-final, not retry (handler may still be running as a zombie).const withTimeout = async (task: () => Promise<void>, timeoutMs: number) => {let timer: ReturnType<typeof setTimeout> | undefined;const timeout = new Promise<never>((_, reject) => {@@ -333,8 +257,6 @@ export const cronJobFactory = ({}};- // ── run-hash status writes ──────────────────────────────────────────────────-const markRunning = (id: string, attempt: number) =>redis.hset(RUN_KEY(id),@@ -353,8 +275,6 @@ export const cronJobFactory = ({await redis.zrem(PENDING_ZSET, id);};- // `err` is optional: the max-attempts-reached path writes a "failed" status- // without a last_error.const markFailedFinal = async (id: string, err?: unknown) => {if (err === undefined) {await redis.hset(RUN_KEY(id), F.Status, RunStatus.Failed, F.CompletedAt, String(Date.now()));@@ -382,15 +302,9 @@ export const cronJobFactory = ({F.NextAttemptAt,String(nextAttemptAt));- // zset score = next_attempt_at so processTick's zrangebyscore filter skips- // this run until its backoff window elapses.await redis.zadd(PENDING_ZSET, nextAttemptAt, id);};- // ── registration ────────────────────────────────────────────────────────────-- // Registers a cron entry with this pod. `runHashTtlS` controls how long each- // run's hash lives in Redis after enqueue.const register = ({name,pattern,@@ -452,11 +366,6 @@ export const cronJobFactory = ({logger.info(`cron[${name}]: registered (pattern="${pattern}")`);};- // ── slot election ───────────────────────────────────────────────────────────-- // Caps the number of pods that participate in cron ticking. Each pod holds- // one of N slots via SET NX/XX with a short TTL: refreshes its own slot if- // it still owns one, otherwise tries to claim a free slot.const claimOrRefreshSlot = async () => {if (currentSlot !== null) {const ok = await redis.set(SLOT_KEY(currentSlot), workerId, "PX", slotTtlMs, "XX");@@ -475,17 +384,7 @@ export const cronJobFactory = ({}};- // Runs slot claims/refreshes strictly one at a time.- //- // `claimOrRefreshSlot` awaits between reading `currentSlot` and writing it,- // so two overlapping ticks could both observe a lost slot, both null- // `currentSlot`, and then claim two *different* slots — the pod would burn- // two of the five participant slots and leak one on shutdown, since only the- // last-assigned `currentSlot` is ever released.- //- // Chaining also gives stop() one handle to await, so it can never read- // `currentSlot` mid-handover (skipping the release) or have a late claim land- // after the release.+ // Serialize slot ops: overlapping ticks could claim two slots and leak one on shutdown.const runSlotOp = (label: string) => {slotOp = slotOp.then(() => (stopped ? undefined : claimOrRefreshSlot()))@@ -493,16 +392,7 @@ export const cronJobFactory = ({return slotOp;};- // ── enqueue ─────────────────────────────────────────────────────────────────-- // For each registered entry, computes the most recent scheduled fire time- // and atomically inserts a run hash + pending-zset entry if not already- // present. The Lua script makes the existence check + write a single op so- // multiple pods racing to enqueue the same fire produce exactly one run.- //- // The run id and the hash's `scheduled_at` stay keyed on the unjittered fire,- // so run identity is exactly what it was before jitter existed. Only the zset- // score carries the offset, which is what holds the run back from processTick.+ // Run id uses unjittered scheduledAt; pending zset score carries eligibleAt.const enqueueDueFires = async () => {for (const entry of entries.values()) {const scheduledAt = prevFireMs(entry.pattern);@@ -534,26 +424,12 @@ export const cronJobFactory = ({}};- // Slow timer (~30s) that only creates run hashes for due fires. Split from- // processing so the pod that enqueues doesn't immediately consume everything- // it just created — leaves time for other slot-holders' process ticks to race.const enqueueTick = async () => {if (currentSlot === null) return;await enqueueDueFires();};- // ── process ─────────────────────────────────────────────────────────────────-- // Executes one attempt of `entry` under an already-acquired redlock. Writes- // the running state, races the handler against handlerTimeoutMs, and- // branches into success / retry / final failure on completion.- //- // Two-knob recovery model:- // leaseDurationMs → governs CRASH recovery. When the holder pod dies,- // its lease key TTLs out and other pods see the run as- // stalled via the isStalled path in processCandidate.- // handlerTimeoutMs → bounds the LEASE for HANG recovery. On timeout we release the lease and- // mark the run failed-final — a retry would race with the still-running zombie handler. The next scheduled fire is the natural retry.+ // leaseDurationMs: crash recovery via stalled reclaim. handlerTimeoutMs: hang → failed-final.const executeUnderLease = async (entry: CronEntry, id: string, attempt: number) => {try {await markRunning(id, attempt);@@ -572,13 +448,8 @@ export const cronJobFactory = ({} catch (err) {const isHandlerTimeout = err instanceof HandlerTimeoutError;- // Exponential backoff: 1×, 2×, 4× ... capped at retryBackoffMaxMs.const backoffMs = Math.min(retryBackoffBaseMs * 2 ** (attempt - 1), retryBackoffMaxMs);const nextAttemptAt = Date.now() + backoffMs;- // If the retry wouldn't fit before the next scheduled fire, treat this- // run as final — the upcoming fire (separate id) is the natural retry.- // Avoids overlapping attempts of the same cron and avoids squashing- // backoff to ~0 near the end of an interval.const fitsBeforeNextFire = nextAttemptAt + NEXT_FIRE_BUFFER_MS < nextFireMs(entry.pattern);const final = isHandlerTimeout || attempt >= entry.maxAttempts || !fitsBeforeNextFire;@@ -603,10 +474,6 @@ export const cronJobFactory = ({}};- // Inspects one pending zset id: validates the run hash, applies the- // min-age and max-attempts gates, and races for the lease. Anything that- // can be decided without taking a lock is decided here so we only ever- // hand a "ready, attempt-eligible" run to executeUnderLease.const processCandidate = async (id: string) => {const data = await redis.hgetall(RUN_KEY(id));if (!data?.name) {@@ -618,15 +485,13 @@ export const cronJobFactory = ({}const entry = entries.get(data.name);- if (!entry) return; // not handled on this pod+ if (!entry) return;const isPending = data.status === RunStatus.Pending;const isStalled = data.status === RunStatus.Running && (await redis.exists(LEASE_KEY(id))) === 0;if (!isPending && !isStalled) return;- // Min-age delay on first pickup so the pod that enqueued doesn't- // immediately grab everything before other slot-holders' process ticks- // fire. Stalled runs bypass this — they're already past first pickup.+ // Brief delay so other slot-holders can race for freshly enqueued runs.if (isPending) {const enqueuedAtMs = Number(data[F.EnqueuedAtMs] ?? 0);if (Date.now() - enqueuedAtMs < minProcessAgeMs) return;@@ -645,9 +510,6 @@ export const cronJobFactory = ({logger.info(`cron[${data.name}]: re-claiming stalled run [id=${id}] [previous_worker=${data.worker_id}]`);}- // Track the in-flight run so stop() can wait for it to settle before- // tearing down. Lock-contention rejections settle within a tick, so they- // don't measurably delay shutdown drain.const tracked = redlock.using([LEASE_KEY(id)], entry.leaseDurationMs ?? leaseDurationMs, () =>executeUnderLease(entry, id, attempts + 1));@@ -655,7 +517,6 @@ export const cronJobFactory = ({try {await tracked;} catch (err) {- // ExecutionError / ResourceLockedError mean another pod owns the lease — expected contention, swallow.if (err instanceof ExecutionError || err instanceof ResourceLockedError) return;logger.error({ err }, `cron[${data.name}]: unexpected error acquiring lease [id=${id}]`);} finally {@@ -663,15 +524,9 @@ export const cronJobFactory = ({}};- // Fast timer (~5s) that scans the pending zset and races for runs to- // execute. Ids are shuffled per tick so different slot-holders try- // different ids first, spreading lease contention and distributing handler- // load across pods.const processTick = async () => {if (currentSlot === null) return;- // Filter by zset score so backed-off retries (score = next_attempt_at)- // are skipped until their backoff window elapses.const ids = await redis.zrangebyscore(PENDING_ZSET, "-inf", Date.now(), "LIMIT", 0, PROCESS_BATCH_SIZE);shuffleInPlace(ids);@@ -697,15 +552,10 @@ export const cronJobFactory = ({}};- // ── lifecycle ───────────────────────────────────────────────────────────────-const safeTick = (label: string, fn: () => Promise<void>) => () => {fn().catch((err: unknown) => logger.error({ err }, `cron: ${label} failed`));};- // Starts the slot-refresh, enqueue, and process timers, and claims a slot- // immediately so the pod doesn't wait a full `slotRefreshMs` before- // participating.const start = () => {stopped = false;slotTimer = setInterval(() => void runSlotOp("slot refresh"), slotRefreshMs);@@ -714,13 +564,7 @@ export const cronJobFactory = ({void runSlotOp("initial slot claim");};- // Stops the timers, drains in-flight handlers, and atomically releases the- // held slot (only if we still own it) so another pod can take over without- // waiting for the slot TTL.- //- // Drain semantics wait for the currently active runs to finish so destructive handlers (rotations, deletions)- // aren't aborted mid-execution and graceful redeploys don't leave runs- // stuck in `status='running'` until the lease TTL elapses.+ // Drain in-flight handlers before releasing the slot so runs are not left stuck as running.const stop = async () => {stopped = true;if (slotTimer) clearInterval(slotTimer);@@ -746,9 +590,6 @@ export const cronJobFactory = ({}}- // No new slot ops can be queued (timers cleared, `stopped` set), so this is- // the final tail. Waiting on it guarantees `currentSlot` is settled and that- // nothing can re-create the key after the release below.await slotOp;if (currentSlot !== null) {