🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
chore(shared): log the node-level cause behind Redis connection errors (#16200)
Niklas Semmlerb6162c697bac
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.tsindex 2c2f0c5698..b79ab4ae73 100644--- a/packages/shared/src/server/redis/redis.test.ts+++ b/packages/shared/src/server/redis/redis.test.ts@@ -11,6 +11,10 @@ vi.mock("../../env", () => ({import { env } from "../../env";import { safeMultiGet, scanKeys } from "./redis";+import {+ buildRedisErrorContext,+ formatRedisErrorMessage,+} from "./redisErrorContext";type ScanCall = [string, "MATCH", string, "COUNT", number];@@ -121,3 +125,139 @@ describe("safeMultiGet", () => {await expect(safeMultiGet(client, [])).resolves.toEqual([]);});});++const clusterAllFailedError = (lastNodeError: unknown) =>+ Object.assign(new Error("Failed to refresh slots cache."), {+ name: "ClusterAllFailedError",+ lastNodeError,+ });++describe("buildRedisErrorContext", () => {+ it("lifts lastNodeError out of a ClusterAllFailedError", () => {+ const context = buildRedisErrorContext(+ clusterAllFailedError(new Error("timeout")),+ "10.0.1.5:6379",+ );++ expect(context.failureMode).toBe("timeout");+ expect(context.error.message).toBe("Failed to refresh slots cache.");+ expect(context.lastNodeError).toMatchObject({+ message: "timeout",+ node: "10.0.1.5:6379",+ });+ });++ it("keeps the socket-level fields of the node error", () => {+ const nodeError = Object.assign(+ new Error("connect ECONNREFUSED 10.0.1.5:6379"),+ {+ code: "ECONNREFUSED",+ errno: -61,+ syscall: "connect",+ address: "10.0.1.5",+ port: 6379,+ },+ );++ const context = buildRedisErrorContext(clusterAllFailedError(nodeError));++ expect(context.failureMode).toBe("connection-refused");+ expect(context.lastNodeError).toMatchObject({+ code: "ECONNREFUSED",+ errno: -61,+ syscall: "connect",+ address: "10.0.1.5",+ port: 6379,+ });+ expect(context.lastNodeError).not.toHaveProperty("node");+ });++ it.each([+ ["timeout", new Error("timeout"), "timeout"],+ [+ "cluster-down",+ new Error("CLUSTERDOWN Hash slot not served"),+ "cluster-down",+ ],+ ["auth", new Error("WRONGPASS invalid username-password pair"), "auth"],+ [+ "readonly",+ new Error("READONLY You can't write against a replica."),+ "readonly",+ ],+ [+ "dns",+ Object.assign(new Error("getaddrinfo ENOTFOUND redis"), {+ code: "ENOTFOUND",+ }),+ "dns",+ ],+ [+ "tls",+ Object.assign(new Error("certificate has expired"), {+ code: "CERT_HAS_EXPIRED",+ }),+ "tls",+ ],+ ["unknown", new Error("something else entirely"), "unknown"],+ ])(+ "distinguishes %s behind the identical wrapper message",+ (_label, nodeError, expected) => {+ expect(+ buildRedisErrorContext(clusterAllFailedError(nodeError)).failureMode,+ ).toBe(expected);+ },+ );++ it("classifies plain non-cluster errors and omits lastNodeError", () => {+ const context = buildRedisErrorContext(+ Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),+ );++ expect(context.failureMode).toBe("connection-reset");+ expect(context.lastNodeError).toBeUndefined();+ });++ // The text format in ../logger.ts renders `info.stack` and nothing else.+ it("keeps the stack at the top level, not nested under error", () => {+ const context = buildRedisErrorContext(+ clusterAllFailedError(new Error("timeout")),+ );++ expect(context.stack).toContain("Failed to refresh slots cache.");+ expect(context.error).not.toHaveProperty("stack");+ });++ it("does not throw on non-Error values", () => {+ expect(buildRedisErrorContext("boom")).toMatchObject({+ failureMode: "unknown",+ error: { message: "boom" },+ });+ });+});++describe("formatRedisErrorMessage", () => {+ it("puts the failure mode and node cause into the log line", () => {+ const context = buildRedisErrorContext(+ clusterAllFailedError(+ Object.assign(new Error("connect ECONNREFUSED 10.0.1.5:6379"), {+ code: "ECONNREFUSED",+ }),+ ),+ "10.0.1.5:6379",+ );++ expect(formatRedisErrorMessage("Redis cluster error", context)).toBe(+ "Redis cluster error [connection-refused]: Failed to refresh slots cache. " ++ "(last node error: connect ECONNREFUSED 10.0.1.5:6379 [ECONNREFUSED] from 10.0.1.5:6379)",+ );+ });++ it("omits the cause suffix when there is no node error", () => {+ const context = buildRedisErrorContext(new Error("Connection is closed."));++ expect(formatRedisErrorMessage("Redis error", context)).toBe(+ "Redis error [connection-closed]: Connection is closed.",+ );+ });+});diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.tsindex 36ceb183a2..db89915f96 100644--- a/packages/shared/src/server/redis/redis.ts+++ b/packages/shared/src/server/redis/redis.ts@@ -3,6 +3,20 @@ import type { QueueBaseOptions } from "bullmq";import fs from "fs";import { env } from "../../env";import { logger } from "../logger";+import {+ buildRedisErrorContext,+ formatRedisErrorMessage,+ getLastNodeError,+} from "./redisErrorContext";++const logRedisError = (+ prefix: string,+ error: unknown,+ nodeAddress?: string,+) => {+ const context = buildRedisErrorContext(error, nodeAddress);+ logger.error(formatRedisErrorMessage(prefix, context), context);+};const defaultRedisOptions: Partial<RedisOptions> = {enableReadyCheck: true,@@ -146,8 +160,20 @@ const createRedisClusterInstance = (const cluster = new Cluster(nodes, clusterOptions);+ // The `node error` event is the only place ioredis reports which node failed.+ let lastNodeFailure: { error: unknown; address: string } | undefined;+ cluster.on("node error", (error: unknown, address: string) => {+ lastNodeFailure = { error, address };+ });+cluster.on("error", (error) => {- logger.error("Redis cluster error", error);+ const lastNodeError = getLastNodeError(error);+ const nodeAddress =+ lastNodeError !== undefined && lastNodeFailure?.error === lastNodeError+ ? lastNodeFailure.address+ : undefined;++ logRedisError("Redis cluster error", error, nodeAddress);});return cluster;@@ -200,7 +226,7 @@ const createRedisSentinelInstance = (});instance.on("error", (error) => {- logger.error("Redis sentinel error", error);+ logRedisError("Redis sentinel error", error);});return instance;@@ -248,7 +274,7 @@ export const createNewRedisInstance = (: null;instance?.on("error", (error) => {- logger.error("Redis error", error);+ logRedisError("Redis error", error);});return instance;diff --git a/packages/shared/src/server/redis/redisErrorContext.ts b/packages/shared/src/server/redis/redisErrorContext.tsnew file mode 100644index 0000000000..f075355a85--- /dev/null+++ b/packages/shared/src/server/redis/redisErrorContext.ts@@ -0,0 +1,175 @@+/**+ * ioredis reports every failed cluster slot-map refresh as+ * `ClusterAllFailedError: Failed to refresh slots cache.` — one identical+ * message whatever went wrong. The cause is the error from the last node it+ * tried, on `lastNodeError`.+ */++export type RedisFailureMode =+ | "timeout"+ | "connection-refused"+ | "connection-reset"+ | "connection-closed"+ | "host-unreachable"+ | "dns"+ | "tls"+ | "auth"+ | "cluster-down"+ | "readonly"+ | "unknown";++type RedisErrorDetails = {+ name?: string;+ message?: string;+ code?: string;+ errno?: number;+ syscall?: string;+ address?: string;+ port?: number;+};++export type RedisErrorContext = {+ failureMode: RedisFailureMode;+ /** Must stay top-level: the text format in `../logger.ts` renders `info.stack`. */+ stack?: string;+ error: RedisErrorDetails;+ /** `node` comes from the `node error` event; the error itself carries no address. */+ lastNodeError?: RedisErrorDetails & { node?: string };+};++const readString = (source: object, key: string): string | undefined => {+ const value = (source as Record<string, unknown>)[key];+ return typeof value === "string" ? value : undefined;+};++const readNumber = (source: object, key: string): number | undefined => {+ const value = (source as Record<string, unknown>)[key];+ return typeof value === "number" ? value : undefined;+};++const describeError = (error: unknown): RedisErrorDetails => {+ if (typeof error !== "object" || error === null) {+ return { message: String(error) };+ }++ return {+ name: error instanceof Error ? error.name : readString(error, "name"),+ message:+ error instanceof Error ? error.message : readString(error, "message"),+ code: readString(error, "code"),+ errno: readNumber(error, "errno"),+ syscall: readString(error, "syscall"),+ address: readString(error, "address"),+ port: readNumber(error, "port"),+ };+};++export const getLastNodeError = (error: unknown): unknown => {+ if (typeof error !== "object" || error === null) return undefined;+ return (error as { lastNodeError?: unknown }).lastNodeError ?? undefined;+};++const CODE_FAILURE_MODES: Record<string, RedisFailureMode> = {+ ECONNREFUSED: "connection-refused",+ ECONNRESET: "connection-reset",+ EPIPE: "connection-reset",+ ETIMEDOUT: "timeout",+ ESOCKETTIMEDOUT: "timeout",+ EHOSTUNREACH: "host-unreachable",+ ENETUNREACH: "host-unreachable",+ ENOTFOUND: "dns",+ EAI_AGAIN: "dns",+ CERT_HAS_EXPIRED: "tls",+ DEPTH_ZERO_SELF_SIGNED_CERT: "tls",+ SELF_SIGNED_CERT_IN_CHAIN: "tls",+ UNABLE_TO_VERIFY_LEAF_SIGNATURE: "tls",+ ERR_TLS_CERT_ALTNAME_INVALID: "tls",+};++export const classifyRedisFailure = (error: unknown): RedisFailureMode => {+ const { code, message } = describeError(error);++ if (code) {+ const byCode =+ CODE_FAILURE_MODES[code] ?? (code.startsWith("ERR_TLS_") ? "tls" : null);+ if (byCode) return byCode;+ }++ if (!message) return "unknown";++ // Redis replies carry their condition as an uppercase prefix on the message.+ if (message.includes("CLUSTERDOWN")) return "cluster-down";+ if (message.includes("READONLY")) return "readonly";+ if (+ message.includes("NOAUTH") ||+ message.includes("WRONGPASS") ||+ message.includes("NOPERM") ||+ message.includes("invalid password") ||+ message.includes("without any password configured")+ ) {+ return "auth";+ }++ const lowerCaseMessage = message.toLowerCase();+ // ioredis times out `CLUSTER SLOTS` with a bare `new Error("timeout")`.+ if (+ lowerCaseMessage.includes("timeout") ||+ lowerCaseMessage.includes("timed out")+ ) {+ return "timeout";+ }+ if (+ lowerCaseMessage.includes("certificate") ||+ lowerCaseMessage.includes("ssl") ||+ lowerCaseMessage.includes("tls")+ ) {+ return "tls";+ }+ if (+ lowerCaseMessage.includes("connection is closed") ||+ lowerCaseMessage.includes("connection is already closed") ||+ lowerCaseMessage.includes("stream isn't writeable") ||+ lowerCaseMessage.includes("is disconnected") ||+ lowerCaseMessage.includes("is disconnecting")+ ) {+ return "connection-closed";+ }++ return "unknown";+};++/** Pass `nodeAddress` only when it belongs to this error's `lastNodeError`. */+export const buildRedisErrorContext = (+ error: unknown,+ nodeAddress?: string,+): RedisErrorContext => {+ const lastNodeError = getLastNodeError(error);++ return {+ failureMode: classifyRedisFailure(lastNodeError ?? error),+ stack: error instanceof Error ? error.stack : undefined,+ error: describeError(error),+ ...(lastNodeError !== undefined+ ? {+ lastNodeError: {+ ...describeError(lastNodeError),+ ...(nodeAddress ? { node: nodeAddress } : {}),+ },+ }+ : {}),+ };+};++export const formatRedisErrorMessage = (+ prefix: string,+ context: RedisErrorContext,+): string => {+ const cause = context.lastNodeError;+ const causeSuffix = cause+ ? ` (last node error: ${cause.message ?? cause.name ?? "unknown"}${+ cause.code ? ` [${cause.code}]` : ""+ }${cause.node ? ` from ${cause.node}` : ""})`+ : "";++ return `${prefix} [${context.failureMode}]: ${context.error.message ?? "unknown error"}${causeSuffix}`;+};