Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

add worker-heartbeat

Matheus Nogueirae0e02c2dcdb8
diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md
index 719013f4ee..2a09728263 100644
--- a/backend/CLAUDE.md
+++ b/backend/CLAUDE.md
@@ -545,6 +545,8 @@ Queue handler factories (e.g., `src/services/secret/secret-queue.ts`) follow the
`QUEUE_WORKERS_ENABLED` and `QUEUE_WORKER_PROFILE` were replaced by `INFISICAL_RUN_MODES` and no longer exist.
+**Worker heartbeats detect a fleet nobody deployed**, since an ungated producer means an `api`-only deployment silently queues work nothing consumes. In `src/lib/worker-heartbeat/worker-heartbeat.ts`: a `general-workers` pod `startReporting`s itself into a per-worker-type sorted set scored by its heartbeat deadline (60s beat, 5-minute TTL), and an `api` pod that doesn't run the fleet `startMonitoring`s it, logging an error while no instance reports. Observation only — never wire it into `/api/status`; a deployment missing its workers must still serve API traffic.
+
### Scheduled Jobs (Cron Manager)
Recurring work runs through the cron manager in `src/lib/cron/cron-job.ts` (`cronJobFactory`). A single instance is constructed in `src/server/routes/index.ts` (~line 541) and injected as `cronJob` into any service that needs to schedule periodic work. The factory exposes `register`, `start`, and `stop`; `start` is called once after construction, and `stop` is invoked during graceful shutdown to drain in-flight handlers.
diff --git a/backend/src/keystore/keystore.ts b/backend/src/keystore/keystore.ts
index 8c914d1d98..4891bba888 100644
--- a/backend/src/keystore/keystore.ts
+++ b/backend/src/keystore/keystore.ts
@@ -177,6 +177,12 @@ export const KeyStorePrefixes = {
// UUIDs and the endpoint segments do not overlap, so one prefix serves both without collision.
InsightsCache: (scopeId: string, endpoint: string) => `insights-cache:${scopeId}:${endpoint}` as const,
+ // Braces are a Redis Cluster hash tag: the index zset and every member's payload key must land on
+ // one slot for the multi-key upsert/delete scripts.
+ WorkerHeartbeatIndex: (workerType: string) => `worker-heartbeat:{${workerType}}` as const,
+ WorkerHeartbeat: (workerType: string, instanceId: string) =>
+ `worker-heartbeat:{${workerType}}:${instanceId}` as const,
+
AdminConfig: "infisical-admin-cfg",
UpdateCheckLatestVersion: "update-check-latest-version",
InvalidatingCache: "invalidating-cache",
@@ -270,7 +276,8 @@ export const KeyStoreTtls = {
PkiAcmeNonceInSeconds: 300, // 5 minutes
GatewayRelayCredentialInSeconds: 600, // 10 minutes - TURN credential lifetime
SecretReplicationSuccessInSeconds: 10,
- NativeIntegrationDeprecationNoticeInSeconds: 3888000 // 45 days - outlives one monthly cycle
+ NativeIntegrationDeprecationNoticeInSeconds: 3888000, // 45 days - outlives one monthly cycle
+ WorkerHeartbeatInSeconds: 300 // 5 minutes - tolerates several missed 60s beats
};
type TDeleteItems = {
diff --git a/backend/src/lib/worker-heartbeat/worker-heartbeat.test.ts b/backend/src/lib/worker-heartbeat/worker-heartbeat.test.ts
new file mode 100644
index 0000000000..b4fc9f3ceb
--- /dev/null
+++ b/backend/src/lib/worker-heartbeat/worker-heartbeat.test.ts
@@ -0,0 +1,159 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+
+import { inMemoryKeyStore } from "@app/keystore/memory";
+import { logger } from "@app/lib/logger";
+import { RunMode } from "@app/lib/types";
+
+import { workerHeartbeatFactory } from "./worker-heartbeat";
+
+vi.mock("@app/lib/logger", () => ({
+ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }
+}));
+
+const makeFactory = (keyStore = inMemoryKeyStore()) => ({
+ keyStore,
+ ...workerHeartbeatFactory({
+ keyStore,
+ reportIntervalMs: 100,
+ monitorIntervalMs: 100,
+ monitorInitialDelayMs: 50
+ })
+});
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.mocked(logger.error).mockClear();
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+});
+
+describe("worker heartbeat", () => {
+ test("a reporting worker shows up as an active instance", async () => {
+ const worker = makeFactory();
+ worker.startReporting(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toHaveLength(1);
+
+ await worker.stop();
+ });
+
+ test("stop clears the instance so a graceful shutdown is visible immediately", async () => {
+ const worker = makeFactory();
+ worker.startReporting(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(0);
+
+ await worker.stop();
+
+ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual([]);
+ });
+
+ test("each reporting pod is counted separately", async () => {
+ const keyStore = inMemoryKeyStore();
+ const workerA = makeFactory(keyStore);
+ const workerB = makeFactory(keyStore);
+ workerA.startReporting(RunMode.GeneralWorkers);
+ workerB.startReporting(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(await workerA.getActiveInstanceIds(RunMode.GeneralWorkers)).toHaveLength(2);
+
+ await workerA.stop();
+ await workerB.stop();
+ });
+
+ test("monitoring logs an error when no worker reports", async () => {
+ const api = makeFactory();
+ api.startMonitoring(RunMode.GeneralWorkers);
+
+ await vi.advanceTimersByTimeAsync(50);
+
+ expect(logger.error).toHaveBeenCalledTimes(1);
+ expect(vi.mocked(logger.error).mock.calls[0][0]).toContain('no "general-workers" instances detected');
+
+ await api.stop();
+ });
+
+ test("monitoring waits out the initial delay before its first check", async () => {
+ const api = makeFactory();
+ api.startMonitoring(RunMode.GeneralWorkers);
+
+ await vi.advanceTimersByTimeAsync(49);
+
+ expect(logger.error).not.toHaveBeenCalled();
+
+ await api.stop();
+ });
+
+ test("monitoring keeps quiet while a worker reports, and complains once it stops", async () => {
+ const keyStore = inMemoryKeyStore();
+ const api = makeFactory(keyStore);
+ const worker = makeFactory(keyStore);
+
+ worker.startReporting(RunMode.GeneralWorkers);
+ api.startMonitoring(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(250);
+
+ expect(logger.error).not.toHaveBeenCalled();
+
+ await worker.stop();
+ await vi.advanceTimersByTimeAsync(100);
+
+ expect(logger.error).toHaveBeenCalled();
+
+ await api.stop();
+ });
+
+ test("a worker fleet does not mask a different missing fleet", async () => {
+ const keyStore = inMemoryKeyStore();
+ const api = makeFactory(keyStore);
+ const worker = makeFactory(keyStore);
+
+ worker.startReporting(RunMode.SecretScanning);
+ api.startMonitoring(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(50);
+
+ expect(vi.mocked(logger.error).mock.calls[0][0]).toContain('no "general-workers" instances detected');
+
+ await worker.stop();
+ await api.stop();
+ });
+
+ test("a crashed worker's stale entry stops counting once its deadline passes", async () => {
+ const keyStore = inMemoryKeyStore();
+ const worker = makeFactory(keyStore);
+ worker.startReporting(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(0);
+ // The pod is gone: its timer never fires again and nothing removes the entry.
+ await worker.stop();
+ await keyStore.setIndexedItemWithExpiry({
+ indexKey: "worker-heartbeat:{general-workers}",
+ member: "crashed-pod",
+ itemKey: "worker-heartbeat:{general-workers}:crashed-pod",
+ value: "{}",
+ expiryInSeconds: 300,
+ indexed: true
+ });
+
+ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual(["crashed-pod"]);
+
+ vi.setSystemTime(Date.now() + 301_000);
+
+ expect(await worker.getActiveInstanceIds(RunMode.GeneralWorkers)).toEqual([]);
+ });
+
+ test("a failing keystore is logged and does not throw out of the timer", async () => {
+ const keyStore = inMemoryKeyStore();
+ vi.spyOn(keyStore, "setIndexedItemWithExpiry").mockRejectedValue(new Error("redis down"));
+ const worker = makeFactory(keyStore);
+
+ worker.startReporting(RunMode.GeneralWorkers);
+ await vi.advanceTimersByTimeAsync(0);
+
+ expect(logger.error).toHaveBeenCalledTimes(1);
+
+ await worker.stop();
+ });
+});
diff --git a/backend/src/lib/worker-heartbeat/worker-heartbeat.ts b/backend/src/lib/worker-heartbeat/worker-heartbeat.ts
new file mode 100644
index 0000000000..55b06327ec
--- /dev/null
+++ b/backend/src/lib/worker-heartbeat/worker-heartbeat.ts
@@ -0,0 +1,137 @@
+import { randomUUID } from "node:crypto";
+
+import { KeyStorePrefixes, KeyStoreTtls, TKeyStoreFactory } from "@app/keystore/keystore";
+import { logger } from "@app/lib/logger";
+import { RunMode } from "@app/lib/types";
+
+const DEFAULTS = {
+ // Refreshed well inside KeyStoreTtls.WorkerHeartbeatInSeconds so a few missed beats (a GC pause,
+ // a slow Redis round trip) don't make a healthy worker look gone.
+ reportIntervalMs: 60_000,
+ monitorIntervalMs: 5 * 60_000,
+ // Grace before the first check: an API pod usually boots alongside its workers, and they may
+ // still be running migrations when it comes up.
+ monitorInitialDelayMs: 60_000
+} as const;
+
+const missingWorkerError = (workerType: RunMode) =>
+ `no "${workerType}" instances detected. Background jobs (secret syncs, rotations, and others) will be queued but never executed. Add "${workerType}" to the "INFISICAL_RUN_MODES" environment variable of this instance, or deploy a separate instance with that run mode.`;
+
+type TWorkerHeartbeatKeyStore = Pick<
+ TKeyStoreFactory,
+ "setIndexedItemWithExpiry" | "sortedSetRangeByScore" | "deleteIndexedItems"
+>;
+
+export type TWorkerHeartbeatFactory = ReturnType<typeof workerHeartbeatFactory>;
+
+/**
+ * Tracks which worker fleets are alive so an API-only deployment can tell the user when nothing is
+ * consuming its queues. Each reporting pod keeps a member in a per-worker-type sorted set scored by
+ * its heartbeat deadline, plus a small payload key with the same TTL; readers only count members
+ * whose deadline hasn't passed, so a dead pod drops out on its own.
+ */
+export const workerHeartbeatFactory = ({
+ keyStore,
+ reportIntervalMs = DEFAULTS.reportIntervalMs,
+ monitorIntervalMs = DEFAULTS.monitorIntervalMs,
+ monitorInitialDelayMs = DEFAULTS.monitorInitialDelayMs
+}: {
+ keyStore: TWorkerHeartbeatKeyStore;
+ reportIntervalMs?: number;
+ monitorIntervalMs?: number;
+ monitorInitialDelayMs?: number;
+}) => {
+ const instanceId = randomUUID();
+ const reportedTypes = new Set<RunMode>();
+ const monitoredTypes = new Set<RunMode>();
+ let reportTimer: ReturnType<typeof setInterval> | null = null;
+ let monitorTimer: ReturnType<typeof setInterval> | null = null;
+ let monitorDelayTimer: ReturnType<typeof setTimeout> | null = null;
+
+ const report = async (workerType: RunMode) =>
+ keyStore.setIndexedItemWithExpiry({
+ indexKey: KeyStorePrefixes.WorkerHeartbeatIndex(workerType),
+ member: instanceId,
+ itemKey: KeyStorePrefixes.WorkerHeartbeat(workerType, instanceId),
+ value: JSON.stringify({ instanceId, reportedAt: new Date().toISOString() }),
+ expiryInSeconds: KeyStoreTtls.WorkerHeartbeatInSeconds,
+ indexed: true
+ });
+
+ const reportAll = async () => {
+ for (const workerType of reportedTypes) {
+ try {
+ // eslint-disable-next-line no-await-in-loop
+ await report(workerType);
+ } catch (err) {
+ logger.error({ err }, `worker heartbeat: failed to report "${workerType}" heartbeat`);
+ }
+ }
+ };
+
+ const getActiveInstanceIds = async (workerType: RunMode) =>
+ keyStore.sortedSetRangeByScore(KeyStorePrefixes.WorkerHeartbeatIndex(workerType), Date.now(), "+inf");
+
+ const checkAll = async () => {
+ for (const workerType of monitoredTypes) {
+ try {
+ // eslint-disable-next-line no-await-in-loop
+ const instanceIds = await getActiveInstanceIds(workerType);
+ if (!instanceIds.length) logger.error(missingWorkerError(workerType));
+ } catch (err) {
+ logger.error({ err }, `worker heartbeat: failed to check for "${workerType}" instances`);
+ }
+ }
+ };
+
+ /**
+ * Marks this pod as a live instance of `workerType`. Call once per fleet the pod consumes.
+ */
+ const startReporting = (workerType: RunMode) => {
+ reportedTypes.add(workerType);
+ void reportAll();
+ if (reportTimer) return;
+ reportTimer = setInterval(() => void reportAll(), reportIntervalMs);
+ };
+
+ /**
+ * Logs an error whenever no instance of `workerType` is reporting. Only observes — a deployment
+ * missing its workers still serves API traffic, so this must never fail a health check.
+ */
+ const startMonitoring = (workerType: RunMode) => {
+ monitoredTypes.add(workerType);
+ if (monitorTimer || monitorDelayTimer) return;
+ monitorDelayTimer = setTimeout(() => {
+ monitorDelayTimer = null;
+ void checkAll();
+ monitorTimer = setInterval(() => void checkAll(), monitorIntervalMs);
+ }, monitorInitialDelayMs);
+ };
+
+ // Drops this pod's members so a graceful shutdown is visible immediately instead of after the TTL.
+ const stop = async () => {
+ if (reportTimer) clearInterval(reportTimer);
+ if (monitorTimer) clearInterval(monitorTimer);
+ if (monitorDelayTimer) clearTimeout(monitorDelayTimer);
+ reportTimer = null;
+ monitorTimer = null;
+ monitorDelayTimer = null;
+
+ for (const workerType of reportedTypes) {
+ try {
+ // eslint-disable-next-line no-await-in-loop
+ await keyStore.deleteIndexedItems({
+ indexKey: KeyStorePrefixes.WorkerHeartbeatIndex(workerType),
+ members: [instanceId],
+ itemKeys: [KeyStorePrefixes.WorkerHeartbeat(workerType, instanceId)]
+ });
+ } catch (err) {
+ logger.error({ err }, `worker heartbeat: failed to clear "${workerType}" heartbeat`);
+ }
+ }
+ reportedTypes.clear();
+ monitoredTypes.clear();
+ };
+
+ return { startReporting, startMonitoring, getActiveInstanceIds, stop };
+};
diff --git a/backend/src/server/routes/index.ts b/backend/src/server/routes/index.ts
index 9f2f5999ec..46f6afa8e9 100644
--- a/backend/src/server/routes/index.ts
+++ b/backend/src/server/routes/index.ts
@@ -204,6 +204,8 @@ import { BadRequestError } from "@app/lib/errors";
import { initGatewayLoadTracker } from "@app/lib/gateway-v2/gateway-load-tracker";
import { logger } from "@app/lib/logger";
import { Redlock } from "@app/lib/red-lock";
+import { RunMode } from "@app/lib/types";
+import { workerHeartbeatFactory } from "@app/lib/worker-heartbeat/worker-heartbeat";
import { TQueueServiceFactory } from "@app/queue";
import { readLimit } from "@app/server/config/rateLimiter";
import { registerSecretScanningV2Webhooks } from "@app/server/plugins/secret-scanner-v2";
@@ -589,6 +591,17 @@ export const registerRoutes = async (
cronJob.start();
}
+ // An api-only deployment with no general-workers pod queues background work nobody consumes, so
+ // the fleet reports itself in Redis and every api pod complains when the reports stop.
+ const workerHeartbeat = workerHeartbeatFactory({ keyStore });
+ if (envConfig.isGeneralWorkerRunModeEnabled) {
+ workerHeartbeat.startReporting(RunMode.GeneralWorkers);
+ }
+
+ if (envConfig.isApiRunModeEnabled) {
+ workerHeartbeat.startMonitoring(RunMode.GeneralWorkers);
+ }
+
// Reached from the gateway proxy layer in src/lib, which has no DI, so it is installed as a module
// singleton rather than threaded through every call site.
const gatewayLoadTracker = initGatewayLoadTracker(keyStore);
@@ -4398,6 +4411,7 @@ export const registerRoutes = async (
gatewayLoadTracker.shutdown();
cronJobs.forEach((job) => job.stop());
await cronJob.stop();
+ await workerHeartbeat.stop();
await telemetryService.flushAll();
await eventBusService.close();
await projectEventsSSEService.close();