🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): add jitter to redis reconnect backoff (#15925)
Diwakar Ray Yadav1735adbb2e91
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.tsindex b79ab4ae73..b05287302d 100644--- a/packages/shared/src/server/redis/redis.test.ts+++ b/packages/shared/src/server/redis/redis.test.ts@@ -10,7 +10,8 @@ vi.mock("../../env", () => ({}));import { env } from "../../env";-import { safeMultiGet, scanKeys } from "./redis";+import { safeMultiGet, scanKeys, redisQueueRetryOptions } from "./redis";+import { logger } from "../logger";import {buildRedisErrorContext,formatRedisErrorMessage,@@ -261,3 +262,69 @@ describe("formatRedisErrorMessage", () => {);});});+describe("redisQueueRetryOptions", () => {+ const retryStrategy = redisQueueRetryOptions.retryStrategy as (+ times: number,+ ) => number;++ describe("retryStrategy", () => {+ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("keeps early retries at least 1s apart", () => {+ vi.spyOn(Math, "random").mockReturnValue(0.5);+ const delay = retryStrategy(1);+ expect(delay).toBeGreaterThanOrEqual(1000);+ expect(delay).toBeLessThan(1501);+ });++ it("caps the base delay at 20s before jitter", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+ expect(retryStrategy(50)).toBe(20000);+ });++ it("applies jitter so concurrent connections do not retry in lockstep", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+ const base = retryStrategy(8);+ vi.spyOn(Math, "random").mockReturnValue(1);+ const jittered = retryStrategy(8);+ expect(jittered).toBeGreaterThan(base);+ expect(jittered).toBeLessThanOrEqual(base * 1.5);+ });+ });++ describe("reconnectOnError", () => {+ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("logs ordinary connection errors and does not reconnect", () => {+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);+ const result = redisQueueRetryOptions.reconnectOnError!(+ new Error("getaddrinfo ENOTFOUND langfuse-redis"),+ );+ expect(result).toBe(false);+ expect(warn).toHaveBeenCalledTimes(1);+ });++ it("returns 2 for READONLY errors so the command is retried after reconnect", () => {+ vi.spyOn(logger, "warn").mockImplementation(() => logger);+ const result = redisQueueRetryOptions.reconnectOnError!(+ new Error("READONLY You can't write against a read only replica."),+ );+ expect(result).toBe(2);+ });++ it("treats MOVED redirects as debug noise and does not reconnect", () => {+ const debug = vi.spyOn(logger, "debug").mockImplementation(() => logger);+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);+ const result = redisQueueRetryOptions.reconnectOnError!(+ new Error("MOVED 3999 127.0.0.1:6381"),+ );+ expect(result).toBe(false);+ expect(debug).toHaveBeenCalledTimes(1);+ expect(warn).not.toHaveBeenCalled();+ });+ });+});diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.tsindex db89915f96..230078dc33 100644--- a/packages/shared/src/server/redis/redis.ts+++ b/packages/shared/src/server/redis/redis.ts@@ -40,8 +40,11 @@ export const redisQueueRetryOptions: Partial<RedisOptions> = {// A few retries are expected and no cause for action.logger.warn(`Connection to redis lost. Retry attempt: ${times}`);}- // Retries forever. Waits at least 1s and at most 20s between retries.- return Math.max(Math.min(Math.exp(times), 20000), 1000);+ // Retries forever. Exponential base delay clamped to 1s–20s, plus up to+ // 50% random jitter (so at most 30s), so the per-queue connections do not+ // reconnect in lockstep when Redis becomes unreachable.+ const delay = Math.max(Math.min(Math.exp(times), 20000), 1000);+ return delay + Math.random() * delay * 0.5;},reconnectOnError: (err) => {// MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.