Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

chore(queue): share one throttled error logger across all workers

Vlad Ligaibc241598ee30
diff --git a/backend/src/lib/config/redis.test.ts b/backend/src/lib/config/redis.test.ts
index 3816b73181..8232d0b1ad 100644
--- a/backend/src/lib/config/redis.test.ts
+++ b/backend/src/lib/config/redis.test.ts
@@ -21,18 +21,18 @@ describe("createThrottledRedisErrorLogger", () => {
vi.useRealTimers();
});
- test("logs one line per error key per window and counts the repeats it held back", () => {
- const { log } = createThrottledRedisErrorLogger("queue-worker:audit-log", "Queue worker error");
- log(refused());
- log(refused());
- log(refused());
+ test("callers sharing a logger share one limit per error key, and their fields are logged", () => {
+ const { log } = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");
+ log(refused(), undefined, { queue: "audit-log" });
+ log(refused(), undefined, { queue: "secret-sync" });
+ log(refused(), undefined, { queue: "audit-log" });
expect(logger.error).toHaveBeenCalledTimes(1);
vi.advanceTimersByTime(10_000);
- log(refused());
+ log(refused(), undefined, { queue: "secret-sync" });
expect(logger.error).toHaveBeenCalledTimes(2);
expect(loggedLine(1)).toBe(
- "Queue worker error [client=queue-worker:audit-log] [error=ECONNREFUSED] [suppressedCount=2] [message=connect ECONNREFUSED 10.0.0.1:6379]"
+ "Queue worker error [client=queue-workers] [error=ECONNREFUSED] [suppressedCount=2] [queue=secret-sync] [message=connect ECONNREFUSED 10.0.0.1:6379]"
);
});
diff --git a/backend/src/lib/config/redis.ts b/backend/src/lib/config/redis.ts
index 351e405e68..3179cdf379 100644
--- a/backend/src/lib/config/redis.ts
+++ b/backend/src/lib/config/redis.ts
@@ -36,12 +36,13 @@ const getRedisErrorKey = (err: Error) => {
/**
* Logs at most one line per error key (and node) every REDIS_ERROR_LOG_THROTTLE_MS, with a count of the
- * repeats it held back. `reset` clears the window, for a client that has just reconnected.
+ * repeats it held back. `fields` are logged but not throttled on, so callers sharing one logger share
+ * one limit. `reset` clears the window, for a client that has just reconnected.
*/
export const createThrottledRedisErrorLogger = (name: string, label = "Redis connection error") => {
const throttleByErrorKey = new Map<string, { lastLoggedAt: number; suppressedCount: number }>();
- const log = (err: Error, nodeAddress?: string) => {
+ const log = (err: Error, nodeAddress?: string, fields: Record<string, string> = {}) => {
const errorKey = getRedisErrorKey(err);
// Nodes are bucketed separately so one failing node can't absorb another's line. Still bounded,
// unlike keying on err.message, since a cluster has a fixed node count.
@@ -56,9 +57,12 @@ export const createThrottledRedisErrorLogger = (name: string, label = "Redis con
const suppressedCount = throttle?.suppressedCount ?? 0;
throttleByErrorKey.set(throttleKey, { lastLoggedAt: now, suppressedCount: 0 });
+ const fieldTags = Object.entries(fields)
+ .map(([key, value]) => ` [${key}=${value}]`)
+ .join("");
logger.error(
- { err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount },
- `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`
+ { err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount, ...fields },
+ `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}]${fieldTags} [message=${err.message}]`
);
};
diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts
index 42ed15c3d9..35ca391bfa 100644
--- a/backend/src/queue/queue-service.ts
+++ b/backend/src/queue/queue-service.ts
@@ -747,6 +747,9 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFa
Record<QueueName, Worker<TQueueJobTypes[QueueName]["payload"], void, TQueueJobTypes[QueueName]["name"]>>
> = {};
+ // One logger for every worker, so an outage hitting all of them still logs one line per error per window.
+ const workerErrorLogger = createThrottledRedisErrorLogger("queue-workers", "Queue worker error");
+
// Observable gauge for queue depth. The SDK invokes the callback on each export (every 30s for OTLP
// push, on each scrape for Prometheus). Iterates only initialized queues in queueContainer; one
// snapshot covers all ~30 named queues. Failures are swallowed because metrics must never crash the app.
@@ -949,8 +952,7 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFa
// The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its
// errors (and run-loop errors) through this event.
- const workerErrorLogger = createThrottledRedisErrorLogger(`queue-worker:${name}`, "Queue worker error");
- worker.on("error", (err) => workerErrorLogger.log(err));
+ worker.on("error", (err) => workerErrorLogger.log(err, undefined, { queue: name }));
workerContainer[name] = worker;
};