Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
chore(queue): share one throttled error logger across all workers
Vlad Ligaibc241598ee30
diff --git a/backend/src/lib/config/redis.test.ts b/backend/src/lib/config/redis.test.tsindex 3816b73181..8232d0b1ad 100644--- a/backend/src/lib/config/redis.test.ts+++ b/backend/src/lib/config/redis.test.ts@@ -21,18 +21,18 @@ describe("createThrottledRedisErrorLogger", () => {vi.useRealTimers();});- test("logs one line per error key per window and counts the repeats it held back", () => {- const { log } = createThrottledRedisErrorLogger("queue-worker:audit-log", "Queue worker error");- log(refused());- log(refused());- log(refused());+ test("callers sharing a logger share one limit per error key, and their fields are logged", () => {+ const { log } = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");+ log(refused(), undefined, { queue: "audit-log" });+ log(refused(), undefined, { queue: "secret-sync" });+ log(refused(), undefined, { queue: "audit-log" });expect(logger.error).toHaveBeenCalledTimes(1);vi.advanceTimersByTime(10_000);- log(refused());+ log(refused(), undefined, { queue: "secret-sync" });expect(logger.error).toHaveBeenCalledTimes(2);expect(loggedLine(1)).toBe(- "Queue worker error [client=queue-worker:audit-log] [error=ECONNREFUSED] [suppressedCount=2] [message=connect ECONNREFUSED 10.0.0.1:6379]"+ "Queue worker error [client=queue-workers] [error=ECONNREFUSED] [suppressedCount=2] [queue=secret-sync] [message=connect ECONNREFUSED 10.0.0.1:6379]");});diff --git a/backend/src/lib/config/redis.ts b/backend/src/lib/config/redis.tsindex 351e405e68..3179cdf379 100644--- a/backend/src/lib/config/redis.ts+++ b/backend/src/lib/config/redis.ts@@ -36,12 +36,13 @@ const getRedisErrorKey = (err: Error) => {/*** Logs at most one line per error key (and node) every REDIS_ERROR_LOG_THROTTLE_MS, with a count of the- * repeats it held back. `reset` clears the window, for a client that has just reconnected.+ * repeats it held back. `fields` are logged but not throttled on, so callers sharing one logger share+ * one limit. `reset` clears the window, for a client that has just reconnected.*/export const createThrottledRedisErrorLogger = (name: string, label = "Redis connection error") => {const throttleByErrorKey = new Map<string, { lastLoggedAt: number; suppressedCount: number }>();- const log = (err: Error, nodeAddress?: string) => {+ const log = (err: Error, nodeAddress?: string, fields: Record<string, string> = {}) => {const errorKey = getRedisErrorKey(err);// Nodes are bucketed separately so one failing node can't absorb another's line. Still bounded,// unlike keying on err.message, since a cluster has a fixed node count.@@ -56,9 +57,12 @@ export const createThrottledRedisErrorLogger = (name: string, label = "Redis conconst suppressedCount = throttle?.suppressedCount ?? 0;throttleByErrorKey.set(throttleKey, { lastLoggedAt: now, suppressedCount: 0 });+ const fieldTags = Object.entries(fields)+ .map(([key, value]) => ` [${key}=${value}]`)+ .join("");logger.error(- { err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount },- `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`+ { err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount, ...fields },+ `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}]${fieldTags} [message=${err.message}]`);};diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.tsindex 42ed15c3d9..35ca391bfa 100644--- a/backend/src/queue/queue-service.ts+++ b/backend/src/queue/queue-service.ts@@ -747,6 +747,9 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFaRecord<QueueName, Worker<TQueueJobTypes[QueueName]["payload"], void, TQueueJobTypes[QueueName]["name"]>>> = {};+ // One logger for every worker, so an outage hitting all of them still logs one line per error per window.+ const workerErrorLogger = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");+// Observable gauge for queue depth. The SDK invokes the callback on each export (every 30s for OTLP// push, on each scrape for Prometheus). Iterates only initialized queues in queueContainer; one// snapshot covers all ~30 named queues. Failures are swallowed because metrics must never crash the app.@@ -949,8 +952,7 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFa// The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its// errors (and run-loop errors) through this event.- const workerErrorLogger = createThrottledRedisErrorLogger(`queue-worker:${name}`, "Queue worker error");- worker.on("error", (err) => workerErrorLogger.log(err));+ worker.on("error", (err) => workerErrorLogger.log(err, undefined, { queue: name }));workerContainer[name] = worker;};