Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): unify jittered backoff across Redis reconnect strategies (#17618)

Steffen Reuther5321f3de81ca
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.ts
index b05287302d..c4bb2dd2d6 100644
--- a/packages/shared/src/server/redis/redis.test.ts
+++ b/packages/shared/src/server/redis/redis.test.ts
@@ -10,7 +10,12 @@ vi.mock("../../env", () => ({
}));
import { env } from "../../env";
-import { safeMultiGet, scanKeys, redisQueueRetryOptions } from "./redis";
+import {
+ safeMultiGet,
+ scanKeys,
+ redisQueueRetryOptions,
+ redisClusterRetryStrategy,
+} from "./redis";
import { logger } from "../logger";
import {
buildRedisErrorContext,
@@ -272,25 +277,33 @@ describe("redisQueueRetryOptions", () => {
vi.restoreAllMocks();
});
- it("keeps early retries at least 1s apart", () => {
- vi.spyOn(Math, "random").mockReturnValue(0.5);
- const delay = retryStrategy(1);
- expect(delay).toBeGreaterThanOrEqual(1000);
- expect(delay).toBeLessThan(1501);
+ it("stays within 1s–20s for every attempt count", () => {
+ for (const random of [0, 0.5, 1]) {
+ vi.spyOn(Math, "random").mockReturnValue(random);
+ // 5000 is past the attempt count where Math.exp overflows to Infinity.
+ // The cap has to win there: ioredis stops reconnecting for good if the
+ // strategy hands it anything other than a number of milliseconds.
+ for (const times of [1, 5, 8, 10, 50, 5000]) {
+ const delay = retryStrategy(times);
+ expect(delay).toBeGreaterThanOrEqual(1000);
+ expect(delay).toBeLessThanOrEqual(20000);
+ }
+ }
});
- it("caps the base delay at 20s before jitter", () => {
- vi.spyOn(Math, "random").mockReturnValue(0);
- expect(retryStrategy(50)).toBe(20000);
+ it("barely backs off across the first attempts, so a brief Redis restart recovers quickly", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0.5);
+ // Pins the shape of the curve, not its constants: a strategy that
+ // doubled from the floor would already sit at the cap by this attempt.
+ expect(retryStrategy(7)).toBeLessThanOrEqual(retryStrategy(1) * 1.5);
});
it("applies jitter so concurrent connections do not retry in lockstep", () => {
vi.spyOn(Math, "random").mockReturnValue(0);
- const base = retryStrategy(8);
+ const low = retryStrategy(8);
vi.spyOn(Math, "random").mockReturnValue(1);
- const jittered = retryStrategy(8);
- expect(jittered).toBeGreaterThan(base);
- expect(jittered).toBeLessThanOrEqual(base * 1.5);
+ const high = retryStrategy(8);
+ expect(low).toBeLessThan(high);
});
});
@@ -328,3 +341,31 @@ describe("redisQueueRetryOptions", () => {
});
});
});
+
+describe("redisClusterRetryStrategy", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("stays within 100ms–5s for every attempt count", () => {
+ for (const random of [0, 0.5, 1]) {
+ vi.spyOn(Math, "random").mockReturnValue(random);
+ // 5000 is past the attempt count where 2 ** times overflows to Infinity.
+ // The cap has to win there: ioredis stops reconnecting for good if the
+ // strategy hands it anything other than a number of milliseconds.
+ for (const times of [1, 2, 5, 10, 50, 5000]) {
+ const delay = redisClusterRetryStrategy(times);
+ expect(delay).toBeGreaterThanOrEqual(100);
+ expect(delay).toBeLessThanOrEqual(5000);
+ }
+ }
+ });
+
+ it("randomizes the delay so clients do not reconnect in lockstep", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ const low = redisClusterRetryStrategy(4);
+ vi.spyOn(Math, "random").mockReturnValue(1);
+ const high = redisClusterRetryStrategy(4);
+ expect(low).toBeLessThan(high);
+ });
+});
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index 50eeb9f0c2..3ca72d8ea0 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -35,17 +35,34 @@ const defaultRedisOptions: Partial<RedisOptions> = {
const REDIS_SCAN_COUNT = 1000;
+// Both reconnect strategies below randomize their delay within [base/2, base]
+// so that clients which failed at the same moment do not all reconnect at the
+// same moment. Each caller clamps its own base delay, so that clamp is also
+// the upper bound of the delay returned here.
+const jitteredBackoffMs = (baseDelayMs: number): number =>
+ Math.floor(baseDelayMs / 2 + Math.random() * (baseDelayMs / 2));
+
+// Delay before an ioredis Cluster client reconnects after a failed topology
+// refresh or after losing its last node. The ioredis default retries every
+// 100–300ms without jitter, and one process holds a separate Cluster client
+// per queue and per BullMQ worker, so a single failed refresh sends all of
+// them back to the configuration endpoint together. Doubles from 200ms to a
+// 5s cap and retries forever: a cluster that fails over is reachable again
+// within seconds, so the cap stays low.
+export const redisClusterRetryStrategy = (times: number): number =>
+ jitteredBackoffMs(Math.min(200 * 2 ** Math.max(times - 1, 0), 5000));
+
export const redisQueueRetryOptions: Partial<RedisOptions> = {
retryStrategy: (times: number) => {
if (times >= 5) {
// A few retries are expected and no cause for action.
logger.warn(`Connection to redis lost. Retry attempt: ${times}`);
}
- // Retries forever. Exponential base delay clamped to 1s–20s, plus up to
- // 50% random jitter (so at most 30s), so the per-queue connections do not
- // reconnect in lockstep when Redis becomes unreachable.
- const delay = Math.max(Math.min(Math.exp(times), 20000), 1000);
- return delay + Math.random() * delay * 0.5;
+ // Retries forever, between 1s and 20s apart. The base delay holds at 2s
+ // for the first several attempts, so a connection dropped by a Redis
+ // restart is re-established promptly, then ramps to a 20s cap once the
+ // outage outlives those attempts.
+ return jitteredBackoffMs(Math.max(Math.min(Math.exp(times), 20000), 2000));
},
reconnectOnError: (err) => {
// MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.
@@ -159,6 +176,7 @@ const createRedisClusterInstance = (
...tlsOptions,
},
// Retry configuration for cluster
+ clusterRetryStrategy: redisClusterRetryStrategy,
retryDelayOnFailover: 100,
};