🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import Redis, { RedisOptions, Cluster, ClusterOptions } from "ioredis";
import type { QueueBaseOptions, QueueOptions } from "bullmq";
import fs from "fs";
import { env } from "../../env";
import { logger } from "../logger";
import {
buildRedisErrorContext,
formatRedisErrorMessage,
getLastNodeError,
} from "./redisErrorContext";
const logRedisError = (
prefix: string,
error: unknown,
nodeAddress?: string,
) => {
const context = buildRedisErrorContext(error, nodeAddress);
logger.error(formatRedisErrorMessage(prefix, context), context);
};
const defaultRedisOptions: Partial<RedisOptions> = {
enableReadyCheck: true,
maxRetriesPerRequest: null,
enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
keepAlive: 10000, // 10s — prevents middleboxes from killing idle connections
// Forces reconnect if no data received, prevents hung moveToCompleted() from
// blocking concurrency slots forever. ioredis arms the watchdog for any
// defined value (0 would time out instantly), so disabling requires omitting
// the option entirely.
...(env.REDIS_SOCKET_TIMEOUT_MS > 0
? { socketTimeout: env.REDIS_SOCKET_TIMEOUT_MS }
: {}),
};
const REDIS_SCAN_COUNT = 1000;
// Both reconnect strategies below randomize their delay within [base/2, base]
// so that clients which failed at the same moment do not all reconnect at the
// same moment. Each caller clamps its own base delay, so that clamp is also
// the upper bound of the delay returned here.
const jitteredBackoffMs = (baseDelayMs: number): number =>
Math.floor(baseDelayMs / 2 + Math.random() * (baseDelayMs / 2));
// Delay before an ioredis Cluster client reconnects after a failed topology
// refresh or after losing its last node. The ioredis default retries every
// 100–300ms without jitter, and one process holds a separate Cluster client
// per queue and per BullMQ worker, so a single failed refresh sends all of
// them back to the configuration endpoint together. Doubles from 200ms to a
// 5s cap and retries forever: a cluster that fails over is reachable again
// within seconds, so the cap stays low.
export const redisClusterRetryStrategy = (times: number): number =>
jitteredBackoffMs(Math.min(200 * 2 ** Math.max(times - 1, 0), 5000));
export const redisQueueRetryOptions: Partial<RedisOptions> = {
retryStrategy: (times: number) => {
if (times >= 5) {
// A few retries are expected and no cause for action.
logger.warn(`Connection to redis lost. Retry attempt: ${times}`);
}
// Retries forever, between 1s and 20s apart. The base delay holds at 2s
// for the first several attempts, so a connection dropped by a Redis
// restart is re-established promptly, then ramps to a 20s cap once the
// outage outlives those attempts.
return jitteredBackoffMs(Math.max(Math.min(Math.exp(times), 20000), 2000));
},
reconnectOnError: (err) => {
// MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.
if (err.message.includes("MOVED")) {
logger.debug(`Redis cluster redirect: ${err.message}`);
return false;
}
// Reconnects on READONLY errors and auto-retries the command.
logger.warn(`Redis connection error: ${err.message}`);
return err.message.includes("READONLY") ? 2 : false;
},
};
// "settings" comes from QueueOptions (not QueueBaseOptions); the same
// AdvancedRepeatOptions shape is accepted by WorkerOptions["settings"].
type BullMQOptionsWithRedis = Pick<
QueueBaseOptions,
"connection" | "prefix" | "skipVersionCheck"
> &
Pick<QueueOptions, "settings">;
/**
* Parse Redis node definitions from environment variable
* Format: "host1:port1,host2:port2,host3:port3"
*/
const parseRedisNodes = (
nodesString: string,
): Array<{ host: string; port: number }> => {
return nodesString.split(",").map((node) => {
const [host, port] = node.trim().split(":");
if (!host || !port) {
throw new Error(
`Invalid Redis node format: ${node}. Expected format: host:port`,
);
}
return { host, port: parseInt(port, 10) };
});
};
const parseClusterNodes = parseRedisNodes;
const parseSentinelNodes = parseRedisNodes;
/**
* Build TLS options for Redis connections from environment variables
* Returns an object with tls configuration if TLS is enabled, otherwise empty object
*/
const buildTlsOptions = (): Record<string, unknown> => {
if (env.REDIS_TLS_ENABLED !== "true") {
return {};
}
return {
tls: {
ca: env.REDIS_TLS_CA_PATH
? fs.readFileSync(env.REDIS_TLS_CA_PATH)
: undefined,
cert: env.REDIS_TLS_CERT_PATH
? fs.readFileSync(env.REDIS_TLS_CERT_PATH)
: undefined,
key: env.REDIS_TLS_KEY_PATH
? fs.readFileSync(env.REDIS_TLS_KEY_PATH)
: undefined,
...(env.REDIS_TLS_REJECT_UNAUTHORIZED
? {
rejectUnauthorized: env.REDIS_TLS_REJECT_UNAUTHORIZED !== "false",
}
: {}),
...(env.REDIS_TLS_SERVERNAME
? { servername: env.REDIS_TLS_SERVERNAME }
: {}),
...(env.REDIS_TLS_CHECK_SERVER_IDENTITY === "false"
? { checkServerIdentity: () => undefined }
: {}),
...(env.REDIS_TLS_SECURE_PROTOCOL
? { secureProtocol: env.REDIS_TLS_SECURE_PROTOCOL }
: {}),
...(env.REDIS_TLS_CIPHERS ? { ciphers: env.REDIS_TLS_CIPHERS } : {}),
...(env.REDIS_TLS_HONOR_CIPHER_ORDER
? {
honorCipherOrder: env.REDIS_TLS_HONOR_CIPHER_ORDER === "true",
}
: {}),
...(env.REDIS_TLS_KEY_PASSPHRASE
? { passphrase: env.REDIS_TLS_KEY_PASSPHRASE }
: {}),
},
};
};
const createRedisClusterInstance = (
additionalOptions: Partial<RedisOptions> = {},
): Cluster | null => {
if (!env.REDIS_CLUSTER_NODES) {
logger.error(
"REDIS_CLUSTER_NODES is required when REDIS_CLUSTER_ENABLED is true",
);
return null;
}
const nodes = parseClusterNodes(env.REDIS_CLUSTER_NODES);
const tlsOptions = buildTlsOptions();
const clusterOptions: ClusterOptions = {
// Return incoming addresses as-is - required for AWS ElastiCache Certificate resolution
dnsLookup: (address, callback) => {
callback(null, address);
},
slotsRefreshTimeout: env.REDIS_CLUSTER_SLOTS_REFRESH_TIMEOUT,
redisOptions: {
username: env.REDIS_USERNAME || undefined,
password: env.REDIS_AUTH || undefined,
...defaultRedisOptions,
...additionalOptions,
...tlsOptions,
},
// Retry configuration for cluster
clusterRetryStrategy: redisClusterRetryStrategy,
retryDelayOnFailover: 100,
};
const cluster = new Cluster(nodes, clusterOptions);
// The `node error` event is the only place ioredis reports which node failed.
let lastNodeFailure: { error: unknown; address: string } | undefined;
cluster.on("node error", (error: unknown, address: string) => {
lastNodeFailure = { error, address };
});
cluster.on("error", (error) => {
const lastNodeError = getLastNodeError(error);
const nodeAddress =
lastNodeError !== undefined && lastNodeFailure?.error === lastNodeError
? lastNodeFailure.address
: undefined;
logRedisError("Redis cluster error", error, nodeAddress);
});
return cluster;
};
const createRedisSentinelInstance = (
additionalOptions: Partial<RedisOptions> = {},
): Redis | null => {
if (!env.REDIS_SENTINEL_MASTER_NAME) {
logger.error(
"REDIS_SENTINEL_MASTER_NAME is required when REDIS_SENTINEL_ENABLED is true",
);
return null;
}
if (!env.REDIS_SENTINEL_NODES) {
logger.error(
"REDIS_SENTINEL_NODES is required when REDIS_SENTINEL_ENABLED is true",
);
return null;
}
const sentinels = parseSentinelNodes(env.REDIS_SENTINEL_NODES);
const tlsOptions = buildTlsOptions();
const sentinelTlsRequested = env.REDIS_SENTINEL_TLS_ENABLED === "true";
const redisTlsEnabled = env.REDIS_TLS_ENABLED === "true";
if (sentinelTlsRequested && !redisTlsEnabled) {
logger.warn(
"REDIS_SENTINEL_TLS_ENABLED is true but REDIS_TLS_ENABLED is false; sentinel TLS will not be applied",
);
}
const instance = new Redis({
sentinels,
name: env.REDIS_SENTINEL_MASTER_NAME,
username: env.REDIS_USERNAME || undefined,
password: env.REDIS_AUTH || undefined,
sentinelUsername: env.REDIS_SENTINEL_USERNAME || undefined,
sentinelPassword: env.REDIS_SENTINEL_PASSWORD || undefined,
...(sentinelTlsRequested && redisTlsEnabled && tlsOptions.tls
? {
enableTLSForSentinelMode: true,
sentinelTLS: tlsOptions.tls,
}
: {}),
...defaultRedisOptions,
...additionalOptions,
...tlsOptions,
});
instance.on("error", (error) => {
logRedisError("Redis sentinel error", error);
});
return instance;
};
/**
* Every connection handed out by createNewRedisInstance, so that a shutdown can
* release all of them. Each queue owns a dedicated client, and those clients
* retry forever; left connected they keep the event loop alive and the process
* never exits.
*/
const activeRedisInstances = new Set<Redis | Cluster>();
const trackRedisInstance = <T extends Redis | Cluster | null>(
instance: T,
): T => {
if (instance) {
activeRedisInstances.add(instance);
instance.once("end", () => activeRedisInstances.delete(instance));
}
return instance;
};
/**
* Disconnect every client created by createNewRedisInstance, including the
* shared `redis` client, which is created through the same path. Returns how
* many were closed, for shutdown logging.
*/
export const disconnectAllRedisInstances = (): number => {
let closed = 0;
for (const instance of activeRedisInstances) {
try {
instance.disconnect();
closed++;
} catch (error) {
logRedisError("Failed to disconnect Redis instance on shutdown", error);
}
}
activeRedisInstances.clear();
return closed;
};
export const createNewRedisInstance = (
additionalOptions: Partial<RedisOptions> = {},
): Redis | Cluster | null => {
if (
env.REDIS_CLUSTER_ENABLED === "true" &&
env.REDIS_SENTINEL_ENABLED === "true"
) {
logger.error(
"Invalid Redis configuration: REDIS_CLUSTER_ENABLED and REDIS_SENTINEL_ENABLED cannot both be true",
);
return null;
}
if (env.REDIS_CLUSTER_ENABLED === "true") {
return trackRedisInstance(createRedisClusterInstance(additionalOptions));
}
if (env.REDIS_SENTINEL_ENABLED === "true") {
return trackRedisInstance(createRedisSentinelInstance(additionalOptions));
}
const tlsOptions = buildTlsOptions();
const instance = (() => {
if (env.REDIS_CONNECTION_STRING) {
return new Redis(env.REDIS_CONNECTION_STRING, {
...defaultRedisOptions,
...additionalOptions,
...tlsOptions,
});
}
if (env.REDIS_HOST) {
return new Redis({
host: String(env.REDIS_HOST),
port: Number(env.REDIS_PORT),
username: env.REDIS_USERNAME || undefined,
password: env.REDIS_AUTH || undefined,
...defaultRedisOptions,
...additionalOptions,
...tlsOptions,
});
}
return null;
})();
instance?.on("error", (error) => {
logRedisError("Redis error", error);
});
return trackRedisInstance(instance);
};
/**
* Get the queue prefix for BullMQ cluster compatibility
* In cluster mode, uses hash tags to ensure queue keys are on the same node
* In single-node mode, returns the configured prefix or undefined
*/
export const getQueuePrefix = (queueName: string): string | undefined => {
const redisKeyPrefix = env.REDIS_KEY_PREFIX;
if (env.REDIS_CLUSTER_ENABLED === "true") {
// Use hash tags for Redis cluster compatibility
// This ensures all keys for a queue are placed on the same hash slot
// Format: {prefix:queueName} ensures all keys land on same slot
return redisKeyPrefix
? `{${redisKeyPrefix}:${queueName}}`
: `{${queueName}}`;
}
// Non-cluster mode: Return prefix or undefined
return redisKeyPrefix ?? undefined;
};
const getBullMQOptionsForRedisConnection = (
queueName: string,
connection: BullMQOptionsWithRedis["connection"],
): BullMQOptionsWithRedis => ({
connection,
prefix: getQueuePrefix(queueName),
// bullmq hashes with md5 by default, and an OpenSSL FIPS provider refuses
// md5, so any call into the legacy repeat API throws
// ERR_OSSL_EVP_UNSUPPORTED on a FIPS image. Job scheduler registration does
// not hash and is unaffected, but the legacy-schedule cleanup in
// scheduleRecurringJob calls removeRepeatableByKey, which hashes before it
// reaches Redis. sha256 is FIPS-approved and keeps that call working; the
// digest itself is only used to build an identifier the removal script
// never reads.
settings: { repeatKeyHashAlgorithm: "sha256" },
...(env.LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK === "true"
? { skipVersionCheck: true }
: {}),
});
/**
* Creates a new Redis connection and returns the BullMQ queue options that use it.
* Returns null only when the Redis connection cannot be created.
*/
export const createBullMQQueueOptionsWithRedis = (
queueName: string,
): BullMQOptionsWithRedis | null => {
const connection = createNewRedisInstance({
enableOfflineQueue: false,
...redisQueueRetryOptions,
});
return connection
? getBullMQOptionsForRedisConnection(queueName, connection)
: null;
};
/**
* Creates a new Redis connection and returns the BullMQ worker options that use it.
* Returns null only when the Redis connection cannot be created.
*/
export const createBullMQWorkerOptionsWithRedis = (
queueName: string,
): BullMQOptionsWithRedis | null => {
const connection = createNewRedisInstance(redisQueueRetryOptions);
return connection
? getBullMQOptionsForRedisConnection(queueName, connection)
: null;
};
/**
* Execute multiple Redis DEL operations safely in cluster mode
*/
export const safeMultiDel = async (
redis: Redis | Cluster | null,
keys: string[],
): Promise<void> => {
if (!redis || keys.length === 0) return;
if (env.REDIS_CLUSTER_ENABLED === "true") {
// In cluster mode, delete keys in separate commands to avoid CROSSSLOT errors
await Promise.all(keys.map(async (key: string) => redis.del(key)));
} else {
// In single-node mode, can delete all keys at once
await redis.del(keys);
}
};
/**
* Execute multiple Redis GET operations safely in cluster mode.
* MGET requires all keys to hash to the same slot; fall back to per-key GET.
*/
export const safeMultiGet = async (
redis: Redis | Cluster | null,
keys: string[],
): Promise<(string | null)[]> => {
if (!redis || keys.length === 0) return [];
if (env.REDIS_CLUSTER_ENABLED === "true") {
return Promise.all(keys.map(async (key: string) => redis.get(key)));
}
return redis.mget(keys);
};
const scanKeysForNode = async (
client: Redis,
pattern: string,
collector: Set<string>,
keyPrefix: string,
) => {
let cursor = "0";
// ioredis keyPrefix is not applied to SCAN patterns, but it is applied to
// DEL/GET/SET keys. Scan physical keys and return logical keys to callers.
const scanPattern = keyPrefix ? `${keyPrefix}${pattern}` : pattern;
do {
const [nextCursor, keys]: [string, string[]] = await client.scan(
cursor,
"MATCH",
scanPattern,
"COUNT",
REDIS_SCAN_COUNT,
);
keys.forEach((key) =>
collector.add(
keyPrefix && key.startsWith(keyPrefix)
? key.slice(keyPrefix.length)
: key,
),
);
cursor = nextCursor;
} while (cursor !== "0");
};
const getRedisKeyPrefix = (redis: Redis | Cluster): string => {
const keyPrefix =
redis.options?.keyPrefix ??
(redis instanceof Cluster ? redis.options.redisOptions?.keyPrefix : "");
return keyPrefix?.toString() ?? "";
};
export const scanKeys = async (
redis: Redis | Cluster | null,
pattern: string,
): Promise<string[]> => {
if (!redis) return [];
const collectedKeys = new Set<string>();
const keyPrefix = getRedisKeyPrefix(redis);
if (env.REDIS_CLUSTER_ENABLED === "true") {
await Promise.all(
(redis as Cluster)
.nodes("master")
.map((node) =>
scanKeysForNode(node, pattern, collectedKeys, keyPrefix),
),
);
} else {
await scanKeysForNode(redis as Redis, pattern, collectedKeys, keyPrefix);
}
return Array.from(collectedKeys);
};
const createRedisClient = () => {
try {
return createNewRedisInstance({
keyPrefix: env.REDIS_KEY_PREFIX ?? undefined,
});
} catch (e) {
logger.error("Failed to connect to redis", e);
return null;
}
};
declare global {
var redis: undefined | ReturnType<typeof createRedisClient>;
}
export const redis = globalThis.redis ?? createRedisClient();
if (env.NODE_ENV !== "production") globalThis.redis = redis;