Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
add worker-heartbeat
Matheus Nogueirae0e02c2dcdb8
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.mdindex 719013f4ee..2a09728263 100644--- a/backend/CLAUDE.md+++ b/backend/CLAUDE.md@@ -545,6 +545,8 @@ Queue handler factories (e.g., `src/services/secret/secret-queue.ts`) follow the`QUEUE_WORKERS_ENABLED` and `QUEUE_WORKER_PROFILE` were replaced by `INFISICAL_RUN_MODES` and no longer exist.+**Worker heartbeats detect a fleet nobody deployed**, since an ungated producer means an `api`-only deployment silently queues work nothing consumes. In `src/lib/worker-heartbeat/worker-heartbeat.ts`: a `general-workers` pod `startReporting`s itself into a per-worker-type sorted set scored by its heartbeat deadline (60s beat, 5-minute TTL), and an `api` pod that doesn't run the fleet `startMonitoring`s it, logging an error while no instance reports. Observation only — never wire it into `/api/status`; a deployment missing its workers must still serve API traffic.+### Scheduled Jobs (Cron Manager)Recurring work runs through the cron manager in `src/lib/cron/cron-job.ts` (`cronJobFactory`). A single instance is constructed in `src/server/routes/index.ts` (~line 541) and injected as `cronJob` into any service that needs to schedule periodic work. The factory exposes `register`, `start`, and `stop`; `start` is called once after construction, and `stop` is invoked during graceful shutdown to drain in-flight handlers.diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.tsindex 8c914d1d98..4891bba888 100644--- a/backend/src/keystore/keystore.ts+++ b/backend/src/keystore/keystore.ts@@ -177,6 +177,12 @@ export const KeyStorePrefixes = {// UUIDs and the endpoint segments do not overlap, so one prefix serves both without collision.InsightsCache: (scopeId: string, endpoint: string) => `insights-cache:${scopeId}:${endpoint}` as const,+ // Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on+ // one slot for the multi-key upsert/delete scripts.+ WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,+ WorkerHeartbeat: (workerType: string, instanceId: string) =>+ `worker-heartbeat:{${workerType}}:${instanceId}` as const,+AdminConfig: "infisical-admin-cfg",UpdateCheckLatestVersion: "update-check-latest-version",InvalidatingCache: "invalidating-cache",@@ -270,7 +276,8 @@ export const KeyStoreTtls = {PkiAcmeNonceInSeconds: 300, // 5 minutesGatewayRelayCredentialInSeconds: 600, // 10 minutes - TURN credential lifetimeSecretReplicationSuccessInSeconds: 10,- NativeIntegrationDeprecationNoticeInSeconds: 3888000 // 45 days - outlives one monthly cycle+ NativeIntegrationDeprecationNoticeInSeconds: 3888000, // 45 days - outlives one monthly cycle+ WorkerHeartbeatInSeconds: 300 // 5 minutes - tolerates several missed 60s beats};type TDeleteItems = {diff --git a/backend/src/lib/worker-heartbeat/worker-heartbeat.test.ts b/backend/src/lib/worker-heartbeat/worker-heartbeat.test.tsnew file mode 100644index 0000000000..b4fc9f3ceb--- /dev/null+++ b/backend/src/lib/worker-heartbeat/worker-heartbeat.test.ts@@ -0,0 +1,159 @@+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";++import { inMemoryKeyStore } from "@app/keystore/memory";+import { logger } from "@app/lib/logger";+import { RunMode } from "@app/lib/types";++import { workerHeartbeatFactory } from "./worker-heartbeat";++vi.mock("@app/lib/logger", () => ({+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }+}));++const makeFactory = (keyStore = inMemoryKeyStore()) => ({+ keyStore,+ ...workerHeartbeatFactory({+ keyStore,+ reportIntervalMs: 100,+ monitorIntervalMs: 100,+ monitorInitialDelayMs: 50+ })+});++beforeEach(() => {+ vi.useFakeTimers();+ vi.mocked(logger.error).mockClear();+});++afterEach(() => {+ vi.useRealTimers();+});++describe("worker heartbeat", () => {+ test("a reporting worker shows up as an active instance", async () => {+ const worker = makeFactory();+ worker.startReporting(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(0);++ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toHaveLength(1);++ await worker.stop();+ });++ test("stop clears the instance so a graceful shutdown is visible immediately", async () => {+ const worker = makeFactory();+ worker.startReporting(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(0);++ await worker.stop();++ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual([]);+ });++ test("each reporting pod is counted separately", async () => {+ const keyStore = inMemoryKeyStore();+ const workerA = makeFactory(keyStore);+ const workerB = makeFactory(keyStore);+ workerA.startReporting(RunMode.GeneralWorkers);+ workerB.startReporting(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(0);++ expect(await workerA.getActiveInstanceIds(RunMode.GeneralWorkers)).toHaveLength(2);++ await workerA.stop();+ await workerB.stop();+ });++ test("monitoring logs an error when no worker reports", async () => {+ const api = makeFactory();+ api.startMonitoring(RunMode.GeneralWorkers);++ await vi.advanceTimersByTimeAsync(50);++ expect(logger.error).toHaveBeenCalledTimes(1);+ expect(vi.mocked(logger.error).mock.calls[0][0]).toContain('no "general-workers" instances detected');++ await api.stop();+ });++ test("monitoring waits out the initial delay before its first check", async () => {+ const api = makeFactory();+ api.startMonitoring(RunMode.GeneralWorkers);++ await vi.advanceTimersByTimeAsync(49);++ expect(logger.error).not.toHaveBeenCalled();++ await api.stop();+ });++ test("monitoring keeps quiet while a worker reports, and complains once it stops", async () => {+ const keyStore = inMemoryKeyStore();+ const api = makeFactory(keyStore);+ const worker = makeFactory(keyStore);++ worker.startReporting(RunMode.GeneralWorkers);+ api.startMonitoring(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(250);++ expect(logger.error).not.toHaveBeenCalled();++ await worker.stop();+ await vi.advanceTimersByTimeAsync(100);++ expect(logger.error).toHaveBeenCalled();++ await api.stop();+ });++ test("a worker fleet does not mask a different missing fleet", async () => {+ const keyStore = inMemoryKeyStore();+ const api = makeFactory(keyStore);+ const worker = makeFactory(keyStore);++ worker.startReporting(RunMode.SecretScanning);+ api.startMonitoring(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(50);++ expect(vi.mocked(logger.error).mock.calls[0][0]).toContain('no "general-workers" instances detected');++ await worker.stop();+ await api.stop();+ });++ test("a crashed worker's stale entry stops counting once its deadline passes", async () => {+ const keyStore = inMemoryKeyStore();+ const worker = makeFactory(keyStore);+ worker.startReporting(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(0);+ // The pod is gone: its timer never fires again and nothing removes the entry.+ await worker.stop();+ await keyStore.setIndexedItemWithExpiry({+ indexKey: "worker-heartbeat:{general-workers}",+ member: "crashed-pod",+ itemKey: "worker-heartbeat:{general-workers}:crashed-pod",+ value: "{}",+ expiryInSeconds: 300,+ indexed: true+ });++ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual(["crashed-pod"]);++ vi.setSystemTime(Date.now() + 301_000);++ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual([]);+ });++ test("a failing keystore is logged and does not throw out of the timer", async () => {+ const keyStore = inMemoryKeyStore();+ vi.spyOn(keyStore, "setIndexedItemWithExpiry").mockRejectedValue(new Error("redis down"));+ const worker = makeFactory(keyStore);++ worker.startReporting(RunMode.GeneralWorkers);+ await vi.advanceTimersByTimeAsync(0);++ expect(logger.error).toHaveBeenCalledTimes(1);++ await worker.stop();+ });+});diff --git a/backend/src/lib/worker-heartbeat/worker-heartbeat.ts b/backend/src/lib/worker-heartbeat/worker-heartbeat.tsnew file mode 100644index 0000000000..55b06327ec--- /dev/null+++ b/backend/src/lib/worker-heartbeat/worker-heartbeat.ts@@ -0,0 +1,137 @@+import { randomUUID } from "node:crypto";++import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";+import { logger } from "@app/lib/logger";+import { RunMode } from "@app/lib/types";++const DEFAULTS = {+ // Refreshed well inside KeyStoreTtls.WorkerHeartbeatInSeconds so a few missed beats (a GC pause,+ // a slow Redis round trip) don't make a healthy worker look gone.+ reportIntervalMs: 60_000,+ monitorIntervalMs: 5 * 60_000,+ // Grace before the first check: an API pod usually boots alongside its workers, and they may+ // still be running migrations when it comes up.+ monitorInitialDelayMs: 60_000+} as const;++const missingWorkerError = (workerType: RunMode) =>+ `no "${workerType}" instances detected. Background jobs (secret syncs, rotations, and others) will be queued but never executed. Add "${workerType}" to the "INFISICAL_RUN_MODES" environment variable of this instance, or deploy a separate instance with that run mode.`;++type TWorkerHeartbeatKeyStore = Pick<+ TKeyStoreFactory,+ "setIndexedItemWithExpiry" | "sortedSetRangeByScore" | "deleteIndexedItems"+>;++export type TWorkerHeartbeatFactory = ReturnType<typeof workerHeartbeatFactory>;++/**+ * Tracks which worker fleets are alive so an API-only deployment can tell the user when nothing is+ * consuming its queues. Each reporting pod keeps a member in a per-worker-type sorted set scored by+ * its heartbeat deadline, plus a small payload key with the same TTL; readers only count members+ * whose deadline hasn't passed, so a dead pod drops out on its own.+ */+export const workerHeartbeatFactory = ({+ keyStore,+ reportIntervalMs = DEFAULTS.reportIntervalMs,+ monitorIntervalMs = DEFAULTS.monitorIntervalMs,+ monitorInitialDelayMs = DEFAULTS.monitorInitialDelayMs+}: {+ keyStore: TWorkerHeartbeatKeyStore;+ reportIntervalMs?: number;+ monitorIntervalMs?: number;+ monitorInitialDelayMs?: number;+}) => {+ const instanceId = randomUUID();+ const reportedTypes = new Set<RunMode>();+ const monitoredTypes = new Set<RunMode>();+ let reportTimer: ReturnType<typeof setInterval> | null = null;+ let monitorTimer: ReturnType<typeof setInterval> | null = null;+ let monitorDelayTimer: ReturnType<typeof setTimeout> | null = null;++ const report = async (workerType: RunMode) =>+ keyStore.setIndexedItemWithExpiry({+ indexKey: KeyStorePrefixes.WorkerHeartbeatIndex(workerType),+ member: instanceId,+ itemKey: KeyStorePrefixes.WorkerHeartbeat(workerType, instanceId),+ value: JSON.stringify({ instanceId, reportedAt: new Date().toISOString() }),+ expiryInSeconds: KeyStoreTtls.WorkerHeartbeatInSeconds,+ indexed: true+ });++ const reportAll = async () => {+ for (const workerType of reportedTypes) {+ try {+ // eslint-disable-next-line no-await-in-loop+ await report(workerType);+ } catch (err) {+ logger.error({ err }, `worker heartbeat: failed to report "${workerType}" heartbeat`);+ }+ }+ };++ const getActiveInstanceIds = async (workerType: RunMode) =>+ keyStore.sortedSetRangeByScore(KeyStorePrefixes.WorkerHeartbeatIndex(workerType), Date.now(), "+inf");++ const checkAll = async () => {+ for (const workerType of monitoredTypes) {+ try {+ // eslint-disable-next-line no-await-in-loop+ const instanceIds = await getActiveInstanceIds(workerType);+ if (!instanceIds.length) logger.error(missingWorkerError(workerType));+ } catch (err) {+ logger.error({ err }, `worker heartbeat: failed to check for "${workerType}" instances`);+ }+ }+ };++ /**+ * Marks this pod as a live instance of `workerType`. Call once per fleet the pod consumes.+ */+ const startReporting = (workerType: RunMode) => {+ reportedTypes.add(workerType);+ void reportAll();+ if (reportTimer) return;+ reportTimer = setInterval(() => void reportAll(), reportIntervalMs);+ };++ /**+ * Logs an error whenever no instance of `workerType` is reporting. Only observes — a deployment+ * missing its workers still serves API traffic, so this must never fail a health check.+ */+ const startMonitoring = (workerType: RunMode) => {+ monitoredTypes.add(workerType);+ if (monitorTimer || monitorDelayTimer) return;+ monitorDelayTimer = setTimeout(() => {+ monitorDelayTimer = null;+ void checkAll();+ monitorTimer = setInterval(() => void checkAll(), monitorIntervalMs);+ }, monitorInitialDelayMs);+ };++ // Drops this pod's members so a graceful shutdown is visible immediately instead of after the TTL.+ const stop = async () => {+ if (reportTimer) clearInterval(reportTimer);+ if (monitorTimer) clearInterval(monitorTimer);+ if (monitorDelayTimer) clearTimeout(monitorDelayTimer);+ reportTimer = null;+ monitorTimer = null;+ monitorDelayTimer = null;++ for (const workerType of reportedTypes) {+ try {+ // eslint-disable-next-line no-await-in-loop+ await keyStore.deleteIndexedItems({+ indexKey: KeyStorePrefixes.WorkerHeartbeatIndex(workerType),+ members: [instanceId],+ itemKeys: [KeyStorePrefixes.WorkerHeartbeat(workerType, instanceId)]+ });+ } catch (err) {+ logger.error({ err }, `worker heartbeat: failed to clear "${workerType}" heartbeat`);+ }+ }+ reportedTypes.clear();+ monitoredTypes.clear();+ };++ return { startReporting, startMonitoring, getActiveInstanceIds, stop };+};diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.tsindex 9f2f5999ec..46f6afa8e9 100644--- a/backend/src/server/routes/index.ts+++ b/backend/src/server/routes/index.ts@@ -204,6 +204,8 @@ import { BadRequestError } from "@app/lib/errors";import { initGatewayLoadTracker } from "@app/lib/gateway-v2/gateway-load-tracker";import { logger } from "@app/lib/logger";import { Redlock } from "@app/lib/red-lock";+import { RunMode } from "@app/lib/types";+import { workerHeartbeatFactory } from "@app/lib/worker-heartbeat/worker-heartbeat";import { TQueueServiceFactory } from "@app/queue";import { readLimit } from "@app/server/config/rateLimiter";import { registerSecretScanningV2Webhooks } from "@app/server/plugins/secret-scanner-v2";@@ -589,6 +591,17 @@ export const registerRoutes = async (cronJob.start();}+ // An api-only deployment with no general-workers pod queues background work nobody consumes, so+ // the fleet reports itself in Redis and every api pod complains when the reports stop.+ const workerHeartbeat = workerHeartbeatFactory({ keyStore });+ if (envConfig.isGeneralWorkerRunModeEnabled) {+ workerHeartbeat.startReporting(RunMode.GeneralWorkers);+ }++ if (envConfig.isApiRunModeEnabled) {+ workerHeartbeat.startMonitoring(RunMode.GeneralWorkers);+ }+// Reached from the gateway proxy layer in src/lib, which has no DI, so it is installed as a module// singleton rather than threaded through every call site.const gatewayLoadTracker = initGatewayLoadTracker(keyStore);@@ -4398,6 +4411,7 @@ export const registerRoutes = async (gatewayLoadTracker.shutdown();cronJobs.forEach((job) => job.stop());await cronJob.stop();+ await workerHeartbeat.stop();await telemetryService.flushAll();await eventBusService.close();await projectEventsSSEService.close();