🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import { afterEach, describe, expect, it, vi } from "vitest";
import type { Redis } from "ioredis";
vi.mock("../../env", () => ({
env: {
REDIS_CLUSTER_ENABLED: "false",
REDIS_ENABLE_AUTO_PIPELINING: "false",
REDIS_SENTINEL_ENABLED: "false",
},
}));
import { env } from "../../env";
import {
safeMultiGet,
scanKeys,
redisQueueRetryOptions,
redisClusterRetryStrategy,
} from "./redis";
import { logger } from "../logger";
import {
buildRedisErrorContext,
formatRedisErrorMessage,
} from "./redisErrorContext";
type ScanCall = [string, "MATCH", string, "COUNT", number];
const createRedisStub = (
scanResults: Array<[string, string[]]>,
options: { keyPrefix?: string } = {},
) => {
let callIndex = 0;
const scan = vi.fn(
async (..._args: ScanCall): Promise<[string, string[]]> =>
scanResults[callIndex++] ?? ["0", []],
);
return {
client: { options, scan } as unknown as Redis,
scan,
};
};
describe("scanKeys", () => {
it("scans every cursor page and returns unique keys", async () => {
const { client, scan } = createRedisStub([
["42", ["cache:first", "cache:second"]],
["0", ["cache:second", "cache:third"]],
]);
await expect(scanKeys(client, "cache:*")).resolves.toEqual([
"cache:first",
"cache:second",
"cache:third",
]);
expect(scan).toHaveBeenNthCalledWith(
1,
"0",
"MATCH",
"cache:*",
"COUNT",
1000,
);
expect(scan).toHaveBeenNthCalledWith(
2,
"42",
"MATCH",
"cache:*",
"COUNT",
1000,
);
});
it("scans physical prefixed keys but returns logical keys", async () => {
const { client, scan } = createRedisStub(
[["0", ["tenant:api-key:first", "tenant:api-key:second"]]],
{ keyPrefix: "tenant:" },
);
await expect(scanKeys(client, "api-key:*")).resolves.toEqual([
"api-key:first",
"api-key:second",
]);
expect(scan).toHaveBeenCalledWith(
"0",
"MATCH",
"tenant:api-key:*",
"COUNT",
1000,
);
});
});
describe("safeMultiGet", () => {
afterEach(() => {
env.REDIS_CLUSTER_ENABLED = "false";
});
it("uses mget when cluster mode is disabled", async () => {
env.REDIS_CLUSTER_ENABLED = "false";
const mget = vi.fn(async () => ["a", null, "c"]);
const get = vi.fn();
const client = { mget, get } as unknown as Redis;
await expect(safeMultiGet(client, ["k1", "k2", "k3"])).resolves.toEqual([
"a",
null,
"c",
]);
expect(mget).toHaveBeenCalledWith(["k1", "k2", "k3"]);
expect(get).not.toHaveBeenCalled();
});
it("uses per-key get when cluster mode is enabled", async () => {
env.REDIS_CLUSTER_ENABLED = "true";
const mget = vi.fn();
const get = vi.fn(async (key: string) => `value:${key}`);
const client = { mget, get } as unknown as Redis;
await expect(safeMultiGet(client, ["k1", "k2"])).resolves.toEqual([
"value:k1",
"value:k2",
]);
expect(get).toHaveBeenCalledTimes(2);
expect(mget).not.toHaveBeenCalled();
});
it("returns an empty array for empty input", async () => {
const client = { mget: vi.fn(), get: vi.fn() } as unknown as Redis;
await expect(safeMultiGet(client, [])).resolves.toEqual([]);
});
});
const clusterAllFailedError = (lastNodeError: unknown) =>
Object.assign(new Error("Failed to refresh slots cache."), {
name: "ClusterAllFailedError",
lastNodeError,
});
describe("buildRedisErrorContext", () => {
it("lifts lastNodeError out of a ClusterAllFailedError", () => {
const context = buildRedisErrorContext(
clusterAllFailedError(new Error("timeout")),
"10.0.1.5:6379",
);
expect(context.failureMode).toBe("timeout");
expect(context.error.message).toBe("Failed to refresh slots cache.");
expect(context.lastNodeError).toMatchObject({
message: "timeout",
node: "10.0.1.5:6379",
});
});
it("keeps the socket-level fields of the node error", () => {
const nodeError = Object.assign(
new Error("connect ECONNREFUSED 10.0.1.5:6379"),
{
code: "ECONNREFUSED",
errno: -61,
syscall: "connect",
address: "10.0.1.5",
port: 6379,
},
);
const context = buildRedisErrorContext(clusterAllFailedError(nodeError));
expect(context.failureMode).toBe("connection-refused");
expect(context.lastNodeError).toMatchObject({
code: "ECONNREFUSED",
errno: -61,
syscall: "connect",
address: "10.0.1.5",
port: 6379,
});
expect(context.lastNodeError).not.toHaveProperty("node");
});
it.each([
["timeout", new Error("timeout"), "timeout"],
[
"cluster-down",
new Error("CLUSTERDOWN Hash slot not served"),
"cluster-down",
],
["auth", new Error("WRONGPASS invalid username-password pair"), "auth"],
[
"readonly",
new Error("READONLY You can't write against a replica."),
"readonly",
],
[
"dns",
Object.assign(new Error("getaddrinfo ENOTFOUND redis"), {
code: "ENOTFOUND",
}),
"dns",
],
[
"tls",
Object.assign(new Error("certificate has expired"), {
code: "CERT_HAS_EXPIRED",
}),
"tls",
],
["unknown", new Error("something else entirely"), "unknown"],
])(
"distinguishes %s behind the identical wrapper message",
(_label, nodeError, expected) => {
expect(
buildRedisErrorContext(clusterAllFailedError(nodeError)).failureMode,
).toBe(expected);
},
);
it("classifies plain non-cluster errors and omits lastNodeError", () => {
const context = buildRedisErrorContext(
Object.assign(new Error("read ECONNRESET"), { code: "ECONNRESET" }),
);
expect(context.failureMode).toBe("connection-reset");
expect(context.lastNodeError).toBeUndefined();
});
// The text format in ../logger.ts renders `info.stack` and nothing else.
it("keeps the stack at the top level, not nested under error", () => {
const context = buildRedisErrorContext(
clusterAllFailedError(new Error("timeout")),
);
expect(context.stack).toContain("Failed to refresh slots cache.");
expect(context.error).not.toHaveProperty("stack");
});
it("does not throw on non-Error values", () => {
expect(buildRedisErrorContext("boom")).toMatchObject({
failureMode: "unknown",
error: { message: "boom" },
});
});
});
describe("formatRedisErrorMessage", () => {
it("puts the failure mode and node cause into the log line", () => {
const context = buildRedisErrorContext(
clusterAllFailedError(
Object.assign(new Error("connect ECONNREFUSED 10.0.1.5:6379"), {
code: "ECONNREFUSED",
}),
),
"10.0.1.5:6379",
);
expect(formatRedisErrorMessage("Redis cluster error", context)).toBe(
"Redis cluster error [connection-refused]: Failed to refresh slots cache. " +
"(last node error: connect ECONNREFUSED 10.0.1.5:6379 [ECONNREFUSED] from 10.0.1.5:6379)",
);
});
it("omits the cause suffix when there is no node error", () => {
const context = buildRedisErrorContext(new Error("Connection is closed."));
expect(formatRedisErrorMessage("Redis error", context)).toBe(
"Redis error [connection-closed]: Connection is closed.",
);
});
});
describe("redisQueueRetryOptions", () => {
const retryStrategy = redisQueueRetryOptions.retryStrategy as (
times: number,
) => number;
describe("retryStrategy", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("stays within 1s–20s for every attempt count", () => {
for (const random of [0, 0.5, 1]) {
vi.spyOn(Math, "random").mockReturnValue(random);
// 5000 is past the attempt count where Math.exp overflows to Infinity.
// The cap has to win there: ioredis stops reconnecting for good if the
// strategy hands it anything other than a number of milliseconds.
for (const times of [1, 5, 8, 10, 50, 5000]) {
const delay = retryStrategy(times);
expect(delay).toBeGreaterThanOrEqual(1000);
expect(delay).toBeLessThanOrEqual(20000);
}
}
});
it("barely backs off across the first attempts, so a brief Redis restart recovers quickly", () => {
vi.spyOn(Math, "random").mockReturnValue(0.5);
// Pins the shape of the curve, not its constants: a strategy that
// doubled from the floor would already sit at the cap by this attempt.
expect(retryStrategy(7)).toBeLessThanOrEqual(retryStrategy(1) * 1.5);
});
it("applies jitter so concurrent connections do not retry in lockstep", () => {
vi.spyOn(Math, "random").mockReturnValue(0);
const low = retryStrategy(8);
vi.spyOn(Math, "random").mockReturnValue(1);
const high = retryStrategy(8);
expect(low).toBeLessThan(high);
});
});
describe("reconnectOnError", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("logs ordinary connection errors and does not reconnect", () => {
const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
const result = redisQueueRetryOptions.reconnectOnError!(
new Error("getaddrinfo ENOTFOUND langfuse-redis"),
);
expect(result).toBe(false);
expect(warn).toHaveBeenCalledTimes(1);
});
it("returns 2 for READONLY errors so the command is retried after reconnect", () => {
vi.spyOn(logger, "warn").mockImplementation(() => logger);
const result = redisQueueRetryOptions.reconnectOnError!(
new Error("READONLY You can't write against a read only replica."),
);
expect(result).toBe(2);
});
it("treats MOVED redirects as debug noise and does not reconnect", () => {
const debug = vi.spyOn(logger, "debug").mockImplementation(() => logger);
const warn = vi.spyOn(logger, "warn").mockImplementation(() => logger);
const result = redisQueueRetryOptions.reconnectOnError!(
new Error("MOVED 3999 127.0.0.1:6381"),
);
expect(result).toBe(false);
expect(debug).toHaveBeenCalledTimes(1);
expect(warn).not.toHaveBeenCalled();
});
});
});
describe("redisClusterRetryStrategy", () => {
afterEach(() => {
vi.restoreAllMocks();
});
it("stays within 100ms–5s for every attempt count", () => {
for (const random of [0, 0.5, 1]) {
vi.spyOn(Math, "random").mockReturnValue(random);
// 5000 is past the attempt count where 2 ** times overflows to Infinity.
// The cap has to win there: ioredis stops reconnecting for good if the
// strategy hands it anything other than a number of milliseconds.
for (const times of [1, 2, 5, 10, 50, 5000]) {
const delay = redisClusterRetryStrategy(times);
expect(delay).toBeGreaterThanOrEqual(100);
expect(delay).toBeLessThanOrEqual(5000);
}
}
});
it("randomizes the delay so clients do not reconnect in lockstep", () => {
vi.spyOn(Math, "random").mockReturnValue(0);
const low = redisClusterRetryStrategy(4);
vi.spyOn(Math, "random").mockReturnValue(1);
const high = redisClusterRetryStrategy(4);
expect(low).toBeLessThan(high);
});
});