Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

chore(shared): log the node-level cause behind Redis connection errors (#16200)

Niklas Semmlerb6162c697bac
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.ts
index 2c2f0c5698..b79ab4ae73 100644
--- a/packages/shared/src/server/redis/redis.test.ts
+++ b/packages/shared/src/server/redis/redis.test.ts
@@ -11,6 +11,10 @@ vi.mock("../../env", () => ({
import { env } from "../../env";
import { safeMultiGet, scanKeys } from "./redis";
+import {
+ buildRedisErrorContext,
+ formatRedisErrorMessage,
+} from "./redisErrorContext";
type ScanCall = [string, "MATCH", string, "COUNT", number];
@@ -121,3 +125,139 @@ describe("safeMultiGet", () => {
await expect(safeMultiGet(client, [])).resolves.toEqual([]);
});
});
+
+const clusterAllFailedError = (lastNodeError: unknown) =>
+ Object.assign(new Error("Failed to refresh slots cache."), {
+ name: "ClusterAllFailedError",
+ lastNodeError,
+ });
+
+describe("buildRedisErrorContext", () => {
+ it("lifts lastNodeError out of a ClusterAllFailedError", () => {
+ const context = buildRedisErrorContext(
+ clusterAllFailedError(new Error("timeout")),
+ "10.0.1.5:6379",
+ );
+
+ expect(context.failureMode).toBe("timeout");
+ expect(context.error.message).toBe("Failed to refresh slots cache.");
+ expect(context.lastNodeError).toMatchObject({
+ message: "timeout",
+ node: "10.0.1.5:6379",
+ });
+ });
+
+ it("keeps the socket-level fields of the node error", () => {
+ const nodeError = Object.assign(
+ new Error("connect ECONNREFUSED 10.0.1.5:6379"),
+ {
+ code: "ECONNREFUSED",
+ errno: -61,
+ syscall: "connect",
+ address: "10.0.1.5",
+ port: 6379,
+ },
+ );
+
+ const context = buildRedisErrorContext(clusterAllFailedError(nodeError));
+
+ expect(context.failureMode).toBe("connection-refused");
+ expect(context.lastNodeError).toMatchObject({
+ code: "ECONNREFUSED",
+ errno: -61,
+ syscall: "connect",
+ address: "10.0.1.5",
+ port: 6379,
+ });
+ expect(context.lastNodeError).not.toHaveProperty("node");
+ });
+
+ it.each([
+ ["timeout", new Error("timeout"), "timeout"],
+ [
+ "cluster-down",
+ new Error("CLUSTERDOWN Hash slot not served"),
+ "cluster-down",
+ ],
+ ["auth", new Error("WRONGPASS invalid username-password pair"), "auth"],
+ [
+ "readonly",
+ new Error("READONLY You can't write against a replica."),
+ "readonly",
+ ],
+ [
+ "dns",
+ Object.assign(new Error("getaddrinfo ENOTFOUND redis"), {
+ code: "ENOTFOUND",
+ }),
+ "dns",
+ ],
+ [
+ "tls",
+ Object.assign(new Error("certificate has expired"), {
+ code: "CERT_HAS_EXPIRED",
+ }),
+ "tls",
+ ],
+ ["unknown", new Error("something else entirely"), "unknown"],
+ ])(
+ "distinguishes %s behind the identical wrapper message",
+ (_label, nodeError, expected) => {
+ expect(
+ buildRedisErrorContext(clusterAllFailedError(nodeError)).failureMode,
+ ).toBe(expected);
+ },
+ );
+
+ it("classifies plain non-cluster errors and omits lastNodeError", () => {
+ const context = buildRedisErrorContext(
+ Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
+ );
+
+ expect(context.failureMode).toBe("connection-reset");
+ expect(context.lastNodeError).toBeUndefined();
+ });
+
+ // The text format in ../logger.ts renders `info.stack` and nothing else.
+ it("keeps the stack at the top level, not nested under error", () => {
+ const context = buildRedisErrorContext(
+ clusterAllFailedError(new Error("timeout")),
+ );
+
+ expect(context.stack).toContain("Failed to refresh slots cache.");
+ expect(context.error).not.toHaveProperty("stack");
+ });
+
+ it("does not throw on non-Error values", () => {
+ expect(buildRedisErrorContext("boom")).toMatchObject({
+ failureMode: "unknown",
+ error: { message: "boom" },
+ });
+ });
+});
+
+describe("formatRedisErrorMessage", () => {
+ it("puts the failure mode and node cause into the log line", () => {
+ const context = buildRedisErrorContext(
+ clusterAllFailedError(
+ Object.assign(new Error("connect ECONNREFUSED 10.0.1.5:6379"), {
+ code: "ECONNREFUSED",
+ }),
+ ),
+ "10.0.1.5:6379",
+ );
+
+ expect(formatRedisErrorMessage("Redis cluster error", context)).toBe(
+ "Redis cluster error [connection-refused]: Failed to refresh slots cache. " +
+ "(last node error: connect ECONNREFUSED 10.0.1.5:6379 [ECONNREFUSED] from 10.0.1.5:6379)",
+ );
+ });
+
+ it("omits the cause suffix when there is no node error", () => {
+ const context = buildRedisErrorContext(new Error("Connection is closed."));
+
+ expect(formatRedisErrorMessage("Redis error", context)).toBe(
+ "Redis error [connection-closed]: Connection is closed.",
+ );
+ });
+});
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index 36ceb183a2..db89915f96 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -3,6 +3,20 @@ import type { QueueBaseOptions } from "bullmq";
import fs from "fs";
import { env } from "../../env";
import { logger } from "../logger";
+import {
+ buildRedisErrorContext,
+ formatRedisErrorMessage,
+ getLastNodeError,
+} from "./redisErrorContext";
+
+const logRedisError = (
+ prefix: string,
+ error: unknown,
+ nodeAddress?: string,
+) => {
+ const context = buildRedisErrorContext(error, nodeAddress);
+ logger.error(formatRedisErrorMessage(prefix, context), context);
+};
const defaultRedisOptions: Partial<RedisOptions> = {
enableReadyCheck: true,
@@ -146,8 +160,20 @@ const createRedisClusterInstance = (
const cluster = new Cluster(nodes, clusterOptions);
+ // The `node error` event is the only place ioredis reports which node failed.
+ let lastNodeFailure: { error: unknown; address: string } | undefined;
+ cluster.on("node error", (error: unknown, address: string) => {
+ lastNodeFailure = { error, address };
+ });
+
cluster.on("error", (error) => {
- logger.error("Redis cluster error", error);
+ const lastNodeError = getLastNodeError(error);
+ const nodeAddress =
+ lastNodeError !== undefined && lastNodeFailure?.error === lastNodeError
+ ? lastNodeFailure.address
+ : undefined;
+
+ logRedisError("Redis cluster error", error, nodeAddress);
});
return cluster;
@@ -200,7 +226,7 @@ const createRedisSentinelInstance = (
});
instance.on("error", (error) => {
- logger.error("Redis sentinel error", error);
+ logRedisError("Redis sentinel error", error);
});
return instance;
@@ -248,7 +274,7 @@ export const createNewRedisInstance = (
: null;
instance?.on("error", (error) => {
- logger.error("Redis error", error);
+ logRedisError("Redis error", error);
});
return instance;
diff --git a/packages/shared/src/server/redis/redisErrorContext.ts b/packages/shared/src/server/redis/redisErrorContext.ts
new file mode 100644
index 0000000000..f075355a85
--- /dev/null
+++ b/packages/shared/src/server/redis/redisErrorContext.ts
@@ -0,0 +1,175 @@
+/**
+ * ioredis reports every failed cluster slot-map refresh as
+ * `ClusterAllFailedError: Failed to refresh slots cache.` — one identical
+ * message whatever went wrong. The cause is the error from the last node it
+ * tried, on `lastNodeError`.
+ */
+
+export type RedisFailureMode =
+ | "timeout"
+ | "connection-refused"
+ | "connection-reset"
+ | "connection-closed"
+ | "host-unreachable"
+ | "dns"
+ | "tls"
+ | "auth"
+ | "cluster-down"
+ | "readonly"
+ | "unknown";
+
+type RedisErrorDetails = {
+ name?: string;
+ message?: string;
+ code?: string;
+ errno?: number;
+ syscall?: string;
+ address?: string;
+ port?: number;
+};
+
+export type RedisErrorContext = {
+ failureMode: RedisFailureMode;
+ /** Must stay top-level: the text format in `../logger.ts` renders `info.stack`. */
+ stack?: string;
+ error: RedisErrorDetails;
+ /** `node` comes from the `node error` event; the error itself carries no address. */
+ lastNodeError?: RedisErrorDetails & { node?: string };
+};
+
+const readString = (source: object, key: string): string | undefined => {
+ const value = (source as Record<string, unknown>)[key];
+ return typeof value === "string" ? value : undefined;
+};
+
+const readNumber = (source: object, key: string): number | undefined => {
+ const value = (source as Record<string, unknown>)[key];
+ return typeof value === "number" ? value : undefined;
+};
+
+const describeError = (error: unknown): RedisErrorDetails => {
+ if (typeof error !== "object" || error === null) {
+ return { message: String(error) };
+ }
+
+ return {
+ name: error instanceof Error ? error.name : readString(error, "name"),
+ message:
+ error instanceof Error ? error.message : readString(error, "message"),
+ code: readString(error, "code"),
+ errno: readNumber(error, "errno"),
+ syscall: readString(error, "syscall"),
+ address: readString(error, "address"),
+ port: readNumber(error, "port"),
+ };
+};
+
+export const getLastNodeError = (error: unknown): unknown => {
+ if (typeof error !== "object" || error === null) return undefined;
+ return (error as { lastNodeError?: unknown }).lastNodeError ?? undefined;
+};
+
+const CODE_FAILURE_MODES: Record<string, RedisFailureMode> = {
+ ECONNREFUSED: "connection-refused",
+ ECONNRESET: "connection-reset",
+ EPIPE: "connection-reset",
+ ETIMEDOUT: "timeout",
+ ESOCKETTIMEDOUT: "timeout",
+ EHOSTUNREACH: "host-unreachable",
+ ENETUNREACH: "host-unreachable",
+ ENOTFOUND: "dns",
+ EAI_AGAIN: "dns",
+ CERT_HAS_EXPIRED: "tls",
+ DEPTH_ZERO_SELF_SIGNED_CERT: "tls",
+ SELF_SIGNED_CERT_IN_CHAIN: "tls",
+ UNABLE_TO_VERIFY_LEAF_SIGNATURE: "tls",
+ ERR_TLS_CERT_ALTNAME_INVALID: "tls",
+};
+
+export const classifyRedisFailure = (error: unknown): RedisFailureMode => {
+ const { code, message } = describeError(error);
+
+ if (code) {
+ const byCode =
+ CODE_FAILURE_MODES[code] ?? (code.startsWith("ERR_TLS_") ? "tls" : null);
+ if (byCode) return byCode;
+ }
+
+ if (!message) return "unknown";
+
+ // Redis replies carry their condition as an uppercase prefix on the message.
+ if (message.includes("CLUSTERDOWN")) return "cluster-down";
+ if (message.includes("READONLY")) return "readonly";
+ if (
+ message.includes("NOAUTH") ||
+ message.includes("WRONGPASS") ||
+ message.includes("NOPERM") ||
+ message.includes("invalid password") ||
+ message.includes("without any password configured")
+ ) {
+ return "auth";
+ }
+
+ const lowerCaseMessage = message.toLowerCase();
+ // ioredis times out `CLUSTER SLOTS` with a bare `new Error("timeout")`.
+ if (
+ lowerCaseMessage.includes("timeout") ||
+ lowerCaseMessage.includes("timed out")
+ ) {
+ return "timeout";
+ }
+ if (
+ lowerCaseMessage.includes("certificate") ||
+ lowerCaseMessage.includes("ssl") ||
+ lowerCaseMessage.includes("tls")
+ ) {
+ return "tls";
+ }
+ if (
+ lowerCaseMessage.includes("connection is closed") ||
+ lowerCaseMessage.includes("connection is already closed") ||
+ lowerCaseMessage.includes("stream isn't writeable") ||
+ lowerCaseMessage.includes("is disconnected") ||
+ lowerCaseMessage.includes("is disconnecting")
+ ) {
+ return "connection-closed";
+ }
+
+ return "unknown";
+};
+
+/** Pass `nodeAddress` only when it belongs to this error's `lastNodeError`. */
+export const buildRedisErrorContext = (
+ error: unknown,
+ nodeAddress?: string,
+): RedisErrorContext => {
+ const lastNodeError = getLastNodeError(error);
+
+ return {
+ failureMode: classifyRedisFailure(lastNodeError ?? error),
+ stack: error instanceof Error ? error.stack : undefined,
+ error: describeError(error),
+ ...(lastNodeError !== undefined
+ ? {
+ lastNodeError: {
+ ...describeError(lastNodeError),
+ ...(nodeAddress ? { node: nodeAddress } : {}),
+ },
+ }
+ : {}),
+ };
+};
+
+export const formatRedisErrorMessage = (
+ prefix: string,
+ context: RedisErrorContext,
+): string => {
+ const cause = context.lastNodeError;
+ const causeSuffix = cause
+ ? ` (last node error: ${cause.message ?? cause.name ?? "unknown"}${
+ cause.code ? ` [${cause.code}]` : ""
+ }${cause.node ? ` from ${cause.node}` : ""})`
+ : "";
+
+ return `${prefix} [${context.failureMode}]: ${context.error.message ?? "unknown error"}${causeSuffix}`;
+};