Skip to content

Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.

chore(deps): log bullmq worker errors through the throttled Redis error logger

Vlad Ligai49a8d0f290e8
diff --git a/backend/src/lib/config/redis.test.ts b/backend/src/lib/config/redis.test.ts
new file mode 100644
index 0000000000..3816b73181
--- /dev/null
+++ b/backend/src/lib/config/redis.test.ts
@@ -0,0 +1,47 @@
+import { EventEmitter } from "node:events";
+
+import type { Redis } from "ioredis";
+
+import { logger } from "@app/lib/logger";
+
+import { attachConnectionLogging, createThrottledRedisErrorLogger } from "./redis";
+
+vi.mock("@app/lib/logger", () => ({ logger: { error: vi.fn(), info: vi.fn() } }));
+
+const refused = () => Object.assign(new Error("connect ECONNREFUSED 10.0.0.1:6379"), { code: "ECONNREFUSED" });
+const loggedLine = (call: number) => vi.mocked(logger.error).mock.calls[call]?.[1] as unknown;
+
+describe("createThrottledRedisErrorLogger", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.mocked(logger.error).mockClear();
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ test("logs one line per error key per window and counts the repeats it held back", () => {
+ const { log } = createThrottledRedisErrorLogger("queue-worker:audit-log", "Queue worker error");
+ log(refused());
+ log(refused());
+ log(refused());
+ expect(logger.error).toHaveBeenCalledTimes(1);
+
+ vi.advanceTimersByTime(10_000);
+ log(refused());
+ expect(logger.error).toHaveBeenCalledTimes(2);
+ expect(loggedLine(1)).toBe(
+ "Queue worker error [client=queue-worker:audit-log] [error=ECONNREFUSED] [suppressedCount=2] [message=connect ECONNREFUSED 10.0.0.1:6379]"
+ );
+ });
+
+ test("attachConnectionLogging keeps its connection error line", () => {
+ const client = new EventEmitter() as unknown as Redis;
+ attachConnectionLogging(client, "main");
+ client.emit("error", refused());
+ expect(loggedLine(0)).toBe(
+ "Redis connection error [client=main] [error=ECONNREFUSED] [suppressedCount=0] [message=connect ECONNREFUSED 10.0.0.1:6379]"
+ );
+ });
+});
diff --git a/backend/src/lib/config/redis.ts b/backend/src/lib/config/redis.ts
index d7808e3df9..351e405e68 100644
--- a/backend/src/lib/config/redis.ts
+++ b/backend/src/lib/config/redis.ts
@@ -35,14 +35,13 @@ const getRedisErrorKey = (err: Error) => {
};
/**
- * Attaches throttled, named connection logging to a client. Call this on any client not built by
- * `buildRedisFromConfig`. Specifically `duplicate()`d clients, since `duplicate()` copies options but not listeners.
+ * Logs at most one line per error key (and node) every REDIS_ERROR_LOG_THROTTLE_MS, with a count of the
+ * repeats it held back. `reset` clears the window, for a client that has just reconnected.
*/
-export const attachConnectionLogging = (client: Redis | Cluster, name: string) => {
+export const createThrottledRedisErrorLogger = (name: string, label = "Redis connection error") => {
const throttleByErrorKey = new Map<string, { lastLoggedAt: number; suppressedCount: number }>();
- let isDown = false;
- const logConnectionError = (err: Error, nodeAddress?: string) => {
+ const log = (err: Error, nodeAddress?: string) => {
const errorKey = getRedisErrorKey(err);
// Nodes are bucketed separately so one failing node can't absorb another's line. Still bounded,
// unlike keying on err.message, since a cluster has a fixed node count.
@@ -59,10 +58,21 @@ export const attachConnectionLogging = (client: Redis | Cluster, name: string) =
throttleByErrorKey.set(throttleKey, { lastLoggedAt: now, suppressedCount: 0 });
logger.error(
{ err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount },
- `Redis connection error [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`
+ `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`
);
};
+ return { log, reset: () => throttleByErrorKey.clear() };
+};
+
+/**
+ * Attaches throttled, named connection logging to a client. Call this on any client not built by
+ * `buildRedisFromConfig`. Specifically `duplicate()`d clients, since `duplicate()` copies options but not listeners.
+ */
+export const attachConnectionLogging = (client: Redis | Cluster, name: string) => {
+ const { log: logConnectionError, reset } = createThrottledRedisErrorLogger(name);
+ let isDown = false;
+
client.on("error", (err: Error) => logConnectionError(err));
if (client instanceof Cluster) {
@@ -75,7 +85,7 @@ export const attachConnectionLogging = (client: Redis | Cluster, name: string) =
client.on("ready", () => {
// Must clear, otherwise stale timestamps would throttle the first error of the next outage.
- throttleByErrorKey.clear();
+ reset();
// Skip the initial connect; only a recovery is worth a line.
if (!isDown) return;
diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.ts
index 829ad846a0..42ed15c3d9 100644
--- a/backend/src/queue/queue-service.ts
+++ b/backend/src/queue/queue-service.ts
@@ -28,7 +28,7 @@ import {
TQueueSecretScanningSendNotification
} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";
import { getConfig } from "@app/lib/config/env";
-import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis";
+import { buildRedisFromConfig, createThrottledRedisErrorLogger, TRedisConfigKeys } from "@app/lib/config/redis";
import { crypto } from "@app/lib/crypto";
import { classifyError } from "@app/lib/errors/classify";
import { logger } from "@app/lib/logger";
@@ -947,6 +947,11 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFa
logger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job picked up by a worker");
});
+ // The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its
+ // errors (and run-loop errors) through this event.
+ const workerErrorLogger = createThrottledRedisErrorLogger(`queue-worker:${name}`, "Queue worker error");
+ worker.on("error", (err) => workerErrorLogger.log(err));
+
workerContainer[name] = worker;
};