Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
chore(deps): log bullmq worker errors through the throttled Redis error logger
Vlad Ligai49a8d0f290e8
diff --git a/backend/src/lib/config/redis.test.ts b/backend/src/lib/config/redis.test.tsnew file mode 100644index 0000000000..3816b73181--- /dev/null+++ b/backend/src/lib/config/redis.test.ts@@ -0,0 +1,47 @@+import { EventEmitter } from "node:events";++import type { Redis } from "ioredis";++import { logger } from "@app/lib/logger";++import { attachConnectionLogging, createThrottledRedisErrorLogger } from "./redis";++vi.mock("@app/lib/logger", () => ({ logger: { error: vi.fn(), info: vi.fn() } }));++const refused = () => Object.assign(new Error("connect ECONNREFUSED 10.0.0.1:6379"), { code: "ECONNREFUSED" });+const loggedLine = (call: number) => vi.mocked(logger.error).mock.calls[call]?.[1] as unknown;++describe("createThrottledRedisErrorLogger", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ vi.mocked(logger.error).mockClear();+ });++ afterEach(() => {+ vi.useRealTimers();+ });++ test("logs one line per error key per window and counts the repeats it held back", () => {+ const { log } = createThrottledRedisErrorLogger("queue-worker:audit-log", "Queue worker error");+ log(refused());+ log(refused());+ log(refused());+ expect(logger.error).toHaveBeenCalledTimes(1);++ vi.advanceTimersByTime(10_000);+ log(refused());+ expect(logger.error).toHaveBeenCalledTimes(2);+ expect(loggedLine(1)).toBe(+ "Queue worker error [client=queue-worker:audit-log] [error=ECONNREFUSED] [suppressedCount=2] [message=connect ECONNREFUSED 10.0.0.1:6379]"+ );+ });++ test("attachConnectionLogging keeps its connection error line", () => {+ const client = new EventEmitter() as unknown as Redis;+ attachConnectionLogging(client, "main");+ client.emit("error", refused());+ expect(loggedLine(0)).toBe(+ "Redis connection error [client=main] [error=ECONNREFUSED] [suppressedCount=0] [message=connect ECONNREFUSED 10.0.0.1:6379]"+ );+ });+});diff --git a/backend/src/lib/config/redis.ts b/backend/src/lib/config/redis.tsindex d7808e3df9..351e405e68 100644--- a/backend/src/lib/config/redis.ts+++ b/backend/src/lib/config/redis.ts@@ -35,14 +35,13 @@ const getRedisErrorKey = (err: Error) => {};/**- * Attaches throttled, named connection logging to a client. Call this on any client not built by- * `buildRedisFromConfig`. Specifically `duplicate()`d clients, since `duplicate()` copies options but not listeners.+ * Logs at most one line per error key (and node) every REDIS_ERROR_LOG_THROTTLE_MS, with a count of the+ * repeats it held back. `reset` clears the window, for a client that has just reconnected.*/-export const attachConnectionLogging = (client: Redis | Cluster, name: string) => {+export const createThrottledRedisErrorLogger = (name: string, label = "Redis connection error") => {const throttleByErrorKey = new Map<string, { lastLoggedAt: number; suppressedCount: number }>();- let isDown = false;- const logConnectionError = (err: Error, nodeAddress?: string) => {+ const log = (err: Error, nodeAddress?: string) => {const errorKey = getRedisErrorKey(err);// Nodes are bucketed separately so one failing node can't absorb another's line. Still bounded,// unlike keying on err.message, since a cluster has a fixed node count.@@ -59,10 +58,21 @@ export const attachConnectionLogging = (client: Redis | Cluster, name: string) =throttleByErrorKey.set(throttleKey, { lastLoggedAt: now, suppressedCount: 0 });logger.error({ err, redisClient: name, redisErrorKey: errorKey, redisNode: nodeAddress, suppressedCount },- `Redis connection error [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`+ `${label} [client=${name}] [error=${errorKey}]${nodeAddress ? ` [node=${nodeAddress}]` : ""} [suppressedCount=${suppressedCount}] [message=${err.message}]`);};+ return { log, reset: () => throttleByErrorKey.clear() };+};++/**+ * Attaches throttled, named connection logging to a client. Call this on any client not built by+ * `buildRedisFromConfig`. Specifically `duplicate()`d clients, since `duplicate()` copies options but not listeners.+ */+export const attachConnectionLogging = (client: Redis | Cluster, name: string) => {+ const { log: logConnectionError, reset } = createThrottledRedisErrorLogger(name);+ let isDown = false;+client.on("error", (err: Error) => logConnectionError(err));if (client instanceof Cluster) {@@ -75,7 +85,7 @@ export const attachConnectionLogging = (client: Redis | Cluster, name: string) =client.on("ready", () => {// Must clear, otherwise stale timestamps would throttle the first error of the next outage.- throttleByErrorKey.clear();+ reset();// Skip the initial connect; only a recovery is worth a line.if (!isDown) return;diff --git a/backend/src/queue/queue-service.ts b/backend/src/queue/queue-service.tsindex 829ad846a0..42ed15c3d9 100644--- a/backend/src/queue/queue-service.ts+++ b/backend/src/queue/queue-service.ts@@ -28,7 +28,7 @@ import {TQueueSecretScanningSendNotification} from "@app/ee/services/secret-scanning-v2/secret-scanning-v2-types";import { getConfig } from "@app/lib/config/env";-import { buildRedisFromConfig, TRedisConfigKeys } from "@app/lib/config/redis";+import { buildRedisFromConfig, createThrottledRedisErrorLogger, TRedisConfigKeys } from "@app/lib/config/redis";import { crypto } from "@app/lib/crypto";import { classifyError } from "@app/lib/errors/classify";import { logger } from "@app/lib/logger";@@ -947,6 +947,11 @@ export const queueServiceFactory = (redisCfg: TRedisConfigKeys): TQueueServiceFalogger.debug({ queue: name, job: job.name, jobId: job.id }, "Queue job picked up by a worker");});+ // The worker's own blocking connection is a duplicate with no listeners, and BullMQ only reports its+ // errors (and run-loop errors) through this event.+ const workerErrorLogger = createThrottledRedisErrorLogger(`queue-worker:${name}`, "Queue worker error");+ worker.on("error", (err) => workerErrorLogger.log(err));+workerContainer[name] = worker;};