Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(shared): add jitter to redis reconnect backoff (#15925)

Diwakar Ray Yadav1735adbb2e91
diff --git a/packages/shared/src/server/redis/redis.test.ts b/packages/shared/src/server/redis/redis.test.ts
index b79ab4ae73..b05287302d 100644
--- a/packages/shared/src/server/redis/redis.test.ts
+++ b/packages/shared/src/server/redis/redis.test.ts
@@ -10,7 +10,8 @@ vi.mock("../../env", () => ({
}));
import { env } from "../../env";
-import { safeMultiGet, scanKeys } from "./redis";
+import { safeMultiGet, scanKeys, redisQueueRetryOptions } from "./redis";
+import { logger } from "../logger";
import {
buildRedisErrorContext,
formatRedisErrorMessage,
@@ -261,3 +262,69 @@ describe("formatRedisErrorMessage", () => {
);
});
});
+describe("redisQueueRetryOptions", () => {
+ const retryStrategy = redisQueueRetryOptions.retryStrategy as (
+ times: number,
+ ) => number;
+
+ describe("retryStrategy", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("keeps early retries at least 1s apart", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0.5);
+ const delay = retryStrategy(1);
+ expect(delay).toBeGreaterThanOrEqual(1000);
+ expect(delay).toBeLessThan(1501);
+ });
+
+ it("caps the base delay at 20s before jitter", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ expect(retryStrategy(50)).toBe(20000);
+ });
+
+ it("applies jitter so concurrent connections do not retry in lockstep", () => {
+ vi.spyOn(Math, "random").mockReturnValue(0);
+ const base = retryStrategy(8);
+ vi.spyOn(Math, "random").mockReturnValue(1);
+ const jittered = retryStrategy(8);
+ expect(jittered).toBeGreaterThan(base);
+ expect(jittered).toBeLessThanOrEqual(base * 1.5);
+ });
+ });
+
+ describe("reconnectOnError", () => {
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it("logs ordinary connection errors and does not reconnect", () => {
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ const result = redisQueueRetryOptions.reconnectOnError!(
+ new Error("getaddrinfo ENOTFOUND langfuse-redis"),
+ );
+ expect(result).toBe(false);
+ expect(warn).toHaveBeenCalledTimes(1);
+ });
+
+ it("returns 2 for READONLY errors so the command is retried after reconnect", () => {
+ vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ const result = redisQueueRetryOptions.reconnectOnError!(
+ new Error("READONLY You can't write against a read only replica."),
+ );
+ expect(result).toBe(2);
+ });
+
+ it("treats MOVED redirects as debug noise and does not reconnect", () => {
+ const debug = vi.spyOn(logger, "debug").mockImplementation(() => logger);
+ const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ const result = redisQueueRetryOptions.reconnectOnError!(
+ new Error("MOVED 3999 127.0.0.1:6381"),
+ );
+ expect(result).toBe(false);
+ expect(debug).toHaveBeenCalledTimes(1);
+ expect(warn).not.toHaveBeenCalled();
+ });
+ });
+});
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index db89915f96..230078dc33 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -40,8 +40,11 @@ export const redisQueueRetryOptions: Partial<RedisOptions> = {
// A few retries are expected and no cause for action.
logger.warn(`Connection to redis lost. Retry attempt: ${times}`);
}
- // Retries forever. Waits at least 1s and at most 20s between retries.
- return Math.max(Math.min(Math.exp(times), 20000), 1000);
+ // Retries forever. Exponential base delay clamped to 1s–20s, plus up to
+ // 50% random jitter (so at most 30s), so the per-queue connections do not
+ // reconnect in lockstep when Redis becomes unreachable.
+ const delay = Math.max(Math.min(Math.exp(times), 20000), 1000);
+ return delay + Math.random() * delay * 0.5;
},
reconnectOnError: (err) => {
// MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.