🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(shared): unify jittered backoff across Redis reconnect strategies (#17618)
Steffen Reuther5321f3de81ca
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.tsindex b05287302d..c4bb2dd2d6 100644--- a/packages/shared/src/server/redis/redis.test.ts+++ b/packages/shared/src/server/redis/redis.test.ts@@ -10,7 +10,12 @@ vi.mock("../../env", () => ({}));import { env } from "../../env";-import { safeMultiGet, scanKeys, redisQueueRetryOptions } from "./redis";+import {+ safeMultiGet,+ scanKeys,+ redisQueueRetryOptions,+ redisClusterRetryStrategy,+} from "./redis";import { logger } from "../logger";import {buildRedisErrorContext,@@ -272,25 +277,33 @@ describe("redisQueueRetryOptions", () => {vi.restoreAllMocks();});- it("keeps early retries at least 1s apart", () => {- vi.spyOn(Math, "random").mockReturnValue(0.5);- const delay = retryStrategy(1);- expect(delay).toBeGreaterThanOrEqual(1000);- expect(delay).toBeLessThan(1501);+ it("stays within 1s–20s for every attempt count", () => {+ for (const random of [0, 0.5, 1]) {+ vi.spyOn(Math, "random").mockReturnValue(random);+ // 5000 is past the attempt count where Math.exp overflows to Infinity.+ // The cap has to win there: ioredis stops reconnecting for good if the+ // strategy hands it anything other than a number of milliseconds.+ for (const times of [1, 5, 8, 10, 50, 5000]) {+ const delay = retryStrategy(times);+ expect(delay).toBeGreaterThanOrEqual(1000);+ expect(delay).toBeLessThanOrEqual(20000);+ }+ }});- it("caps the base delay at 20s before jitter", () => {- vi.spyOn(Math, "random").mockReturnValue(0);- expect(retryStrategy(50)).toBe(20000);+ it("barely backs off across the first attempts, so a brief Redis restart recovers quickly", () => {+ vi.spyOn(Math, "random").mockReturnValue(0.5);+ // Pins the shape of the curve, not its constants: a strategy that+ // doubled from the floor would already sit at the cap by this attempt.+ expect(retryStrategy(7)).toBeLessThanOrEqual(retryStrategy(1) * 1.5);});it("applies jitter so concurrent connections do not retry in lockstep", () => {vi.spyOn(Math, "random").mockReturnValue(0);- const base = retryStrategy(8);+ const low = retryStrategy(8);vi.spyOn(Math, "random").mockReturnValue(1);- const jittered = retryStrategy(8);- expect(jittered).toBeGreaterThan(base);- expect(jittered).toBeLessThanOrEqual(base * 1.5);+ const high = retryStrategy(8);+ expect(low).toBeLessThan(high);});});@@ -328,3 +341,31 @@ describe("redisQueueRetryOptions", () => {});});});++describe("redisClusterRetryStrategy", () => {+ afterEach(() => {+ vi.restoreAllMocks();+ });++ it("stays within 100ms–5s for every attempt count", () => {+ for (const random of [0, 0.5, 1]) {+ vi.spyOn(Math, "random").mockReturnValue(random);+ // 5000 is past the attempt count where 2 ** times overflows to Infinity.+ // The cap has to win there: ioredis stops reconnecting for good if the+ // strategy hands it anything other than a number of milliseconds.+ for (const times of [1, 2, 5, 10, 50, 5000]) {+ const delay = redisClusterRetryStrategy(times);+ expect(delay).toBeGreaterThanOrEqual(100);+ expect(delay).toBeLessThanOrEqual(5000);+ }+ }+ });++ it("randomizes the delay so clients do not reconnect in lockstep", () => {+ vi.spyOn(Math, "random").mockReturnValue(0);+ const low = redisClusterRetryStrategy(4);+ vi.spyOn(Math, "random").mockReturnValue(1);+ const high = redisClusterRetryStrategy(4);+ expect(low).toBeLessThan(high);+ });+});diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.tsindex 50eeb9f0c2..3ca72d8ea0 100644--- a/packages/shared/src/server/redis/redis.ts+++ b/packages/shared/src/server/redis/redis.ts@@ -35,17 +35,34 @@ const defaultRedisOptions: Partial<RedisOptions> = {const REDIS_SCAN_COUNT = 1000;+// Both reconnect strategies below randomize their delay within [base/2, base]+// so that clients which failed at the same moment do not all reconnect at the+// same moment. Each caller clamps its own base delay, so that clamp is also+// the upper bound of the delay returned here.+const jitteredBackoffMs = (baseDelayMs: number): number =>+ Math.floor(baseDelayMs / 2 + Math.random() * (baseDelayMs / 2));++// Delay before an ioredis Cluster client reconnects after a failed topology+// refresh or after losing its last node. The ioredis default retries every+// 100–300ms without jitter, and one process holds a separate Cluster client+// per queue and per BullMQ worker, so a single failed refresh sends all of+// them back to the configuration endpoint together. Doubles from 200ms to a+// 5s cap and retries forever: a cluster that fails over is reachable again+// within seconds, so the cap stays low.+export const redisClusterRetryStrategy = (times: number): number =>+ jitteredBackoffMs(Math.min(200 * 2 ** Math.max(times - 1, 0), 5000));+export const redisQueueRetryOptions: Partial<RedisOptions> = {retryStrategy: (times: number) => {if (times >= 5) {// A few retries are expected and no cause for action.logger.warn(`Connection to redis lost. Retry attempt: ${times}`);}- // Retries forever. Exponential base delay clamped to 1s–20s, plus up to- // 50% random jitter (so at most 30s), so the per-queue connections do not- // reconnect in lockstep when Redis becomes unreachable.- const delay = Math.max(Math.min(Math.exp(times), 20000), 1000);- return delay + Math.random() * delay * 0.5;+ // Retries forever, between 1s and 20s apart. The base delay holds at 2s+ // for the first several attempts, so a connection dropped by a Redis+ // restart is re-established promptly, then ramps to a 20s cap once the+ // outage outlives those attempts.+ return jitteredBackoffMs(Math.max(Math.min(Math.exp(times), 20000), 2000));},reconnectOnError: (err) => {// MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.@@ -159,6 +176,7 @@ const createRedisClusterInstance = (...tlsOptions,},// Retry configuration for cluster+ clusterRetryStrategy: redisClusterRetryStrategy,retryDelayOnFailover: 100,};