Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

16.6 KB
Raw Blame
import Redis, { RedisOptions, Cluster, ClusterOptions } from "ioredis";
import type { QueueBaseOptions, QueueOptions } from "bullmq";
import fs from "fs";
import { env } from "../../env";
import { logger } from "../logger";
import {
  buildRedisErrorContext,
  formatRedisErrorMessage,
  getLastNodeError,
} from "./redisErrorContext";

const logRedisError = (
  prefix: string,
  error: unknown,
  nodeAddress?: string,
) => {
  const context = buildRedisErrorContext(error, nodeAddress);
  logger.error(formatRedisErrorMessage(prefix, context), context);
};

const defaultRedisOptions: Partial<RedisOptions> = {
  enableReadyCheck: true,
  maxRetriesPerRequest: null,
  enableAutoPipelining: env.REDIS_ENABLE_AUTO_PIPELINING === "true",
  keepAlive: 10000, // 10s — prevents middleboxes from killing idle connections
  // Forces reconnect if no data received, prevents hung moveToCompleted() from
  // blocking concurrency slots forever. ioredis arms the watchdog for any
  // defined value (0 would time out instantly), so disabling requires omitting
  // the option entirely.
  ...(env.REDIS_SOCKET_TIMEOUT_MS > 0
    ? { socketTimeout: env.REDIS_SOCKET_TIMEOUT_MS }
    : {}),
};

const REDIS_SCAN_COUNT = 1000;

// Both reconnect strategies below randomize their delay within [base/2, base]
// so that clients which failed at the same moment do not all reconnect at the
// same moment. Each caller clamps its own base delay, so that clamp is also
// the upper bound of the delay returned here.
const jitteredBackoffMs = (baseDelayMs: number): number =>
  Math.floor(baseDelayMs / 2 + Math.random() * (baseDelayMs / 2));

// Delay before an ioredis Cluster client reconnects after a failed topology
// refresh or after losing its last node. The ioredis default retries every
// 100–300ms without jitter, and one process holds a separate Cluster client
// per queue and per BullMQ worker, so a single failed refresh sends all of
// them back to the configuration endpoint together. Doubles from 200ms to a
// 5s cap and retries forever: a cluster that fails over is reachable again
// within seconds, so the cap stays low.
export const redisClusterRetryStrategy = (times: number): number =>
  jitteredBackoffMs(Math.min(200 * 2 ** Math.max(times - 1, 0), 5000));

export const redisQueueRetryOptions: Partial<RedisOptions> = {
  retryStrategy: (times: number) => {
    if (times >= 5) {
      // A few retries are expected and no cause for action.
      logger.warn(`Connection to redis lost. Retry attempt: ${times}`);
    }
    // Retries forever, between 1s and 20s apart. The base delay holds at 2s
    // for the first several attempts, so a connection dropped by a Redis
    // restart is re-established promptly, then ramps to a 20s cap once the
    // outage outlives those attempts.
    return jitteredBackoffMs(Math.max(Math.min(Math.exp(times), 20000), 2000));
  },
  reconnectOnError: (err) => {
    // MOVED/ASK are normal cluster redirections handled by ioredis — not real errors.
    if (err.message.includes("MOVED")) {
      logger.debug(`Redis cluster redirect: ${err.message}`);
      return false;
    }

    // Reconnects on READONLY errors and auto-retries the command.
    logger.warn(`Redis connection error: ${err.message}`);
    return err.message.includes("READONLY") ? 2 : false;
  },
};

// "settings" comes from QueueOptions (not QueueBaseOptions); the same
// AdvancedRepeatOptions shape is accepted by WorkerOptions["settings"].
type BullMQOptionsWithRedis = Pick<
  QueueBaseOptions,
  "connection" | "prefix" | "skipVersionCheck"
> &
  Pick<QueueOptions, "settings">;

/**
 * Parse Redis node definitions from environment variable
 * Format: "host1:port1,host2:port2,host3:port3"
 */
const parseRedisNodes = (
  nodesString: string,
): Array<{ host: string; port: number }> => {
  return nodesString.split(",").map((node) => {
    const [host, port] = node.trim().split(":");
    if (!host || !port) {
      throw new Error(
        `Invalid Redis node format: ${node}. Expected format: host:port`,
      );
    }
    return { host, port: parseInt(port, 10) };
  });
};
const parseClusterNodes = parseRedisNodes;
const parseSentinelNodes = parseRedisNodes;

/**
 * Build TLS options for Redis connections from environment variables
 * Returns an object with tls configuration if TLS is enabled, otherwise empty object
 */
const buildTlsOptions = (): Record<string, unknown> => {
  if (env.REDIS_TLS_ENABLED !== "true") {
    return {};
  }

  return {
    tls: {
      ca: env.REDIS_TLS_CA_PATH
        ? fs.readFileSync(env.REDIS_TLS_CA_PATH)
        : undefined,
      cert: env.REDIS_TLS_CERT_PATH
        ? fs.readFileSync(env.REDIS_TLS_CERT_PATH)
        : undefined,
      key: env.REDIS_TLS_KEY_PATH
        ? fs.readFileSync(env.REDIS_TLS_KEY_PATH)
        : undefined,
      ...(env.REDIS_TLS_REJECT_UNAUTHORIZED
        ? {
            rejectUnauthorized: env.REDIS_TLS_REJECT_UNAUTHORIZED !== "false",
          }
        : {}),
      ...(env.REDIS_TLS_SERVERNAME
        ? { servername: env.REDIS_TLS_SERVERNAME }
        : {}),
      ...(env.REDIS_TLS_CHECK_SERVER_IDENTITY === "false"
        ? { checkServerIdentity: () => undefined }
        : {}),
      ...(env.REDIS_TLS_SECURE_PROTOCOL
        ? { secureProtocol: env.REDIS_TLS_SECURE_PROTOCOL }
        : {}),
      ...(env.REDIS_TLS_CIPHERS ? { ciphers: env.REDIS_TLS_CIPHERS } : {}),
      ...(env.REDIS_TLS_HONOR_CIPHER_ORDER
        ? {
            honorCipherOrder: env.REDIS_TLS_HONOR_CIPHER_ORDER === "true",
          }
        : {}),
      ...(env.REDIS_TLS_KEY_PASSPHRASE
        ? { passphrase: env.REDIS_TLS_KEY_PASSPHRASE }
        : {}),
    },
  };
};

const createRedisClusterInstance = (
  additionalOptions: Partial<RedisOptions> = {},
): Cluster | null => {
  if (!env.REDIS_CLUSTER_NODES) {
    logger.error(
      "REDIS_CLUSTER_NODES is required when REDIS_CLUSTER_ENABLED is true",
    );
    return null;
  }

  const nodes = parseClusterNodes(env.REDIS_CLUSTER_NODES);
  const tlsOptions = buildTlsOptions();

  const clusterOptions: ClusterOptions = {
    // Return incoming addresses as-is - required for AWS ElastiCache Certificate resolution
    dnsLookup: (address, callback) => {
      callback(null, address);
    },
    slotsRefreshTimeout: env.REDIS_CLUSTER_SLOTS_REFRESH_TIMEOUT,
    redisOptions: {
      username: env.REDIS_USERNAME || undefined,
      password: env.REDIS_AUTH || undefined,
      ...defaultRedisOptions,
      ...additionalOptions,
      ...tlsOptions,
    },
    // Retry configuration for cluster
    clusterRetryStrategy: redisClusterRetryStrategy,
    retryDelayOnFailover: 100,
  };

  const cluster = new Cluster(nodes, clusterOptions);

  // The `node error` event is the only place ioredis reports which node failed.
  let lastNodeFailure: { error: unknown; address: string } | undefined;
  cluster.on("node error", (error: unknown, address: string) => {
    lastNodeFailure = { error, address };
  });

  cluster.on("error", (error) => {
    const lastNodeError = getLastNodeError(error);
    const nodeAddress =
      lastNodeError !== undefined && lastNodeFailure?.error === lastNodeError
        ? lastNodeFailure.address
        : undefined;

    logRedisError("Redis cluster error", error, nodeAddress);
  });

  return cluster;
};

const createRedisSentinelInstance = (
  additionalOptions: Partial<RedisOptions> = {},
): Redis | null => {
  if (!env.REDIS_SENTINEL_MASTER_NAME) {
    logger.error(
      "REDIS_SENTINEL_MASTER_NAME is required when REDIS_SENTINEL_ENABLED is true",
    );
    return null;
  }

  if (!env.REDIS_SENTINEL_NODES) {
    logger.error(
      "REDIS_SENTINEL_NODES is required when REDIS_SENTINEL_ENABLED is true",
    );
    return null;
  }

  const sentinels = parseSentinelNodes(env.REDIS_SENTINEL_NODES);
  const tlsOptions = buildTlsOptions();
  const sentinelTlsRequested = env.REDIS_SENTINEL_TLS_ENABLED === "true";
  const redisTlsEnabled = env.REDIS_TLS_ENABLED === "true";

  if (sentinelTlsRequested && !redisTlsEnabled) {
    logger.warn(
      "REDIS_SENTINEL_TLS_ENABLED is true but REDIS_TLS_ENABLED is false; sentinel TLS will not be applied",
    );
  }

  const instance = new Redis({
    sentinels,
    name: env.REDIS_SENTINEL_MASTER_NAME,
    username: env.REDIS_USERNAME || undefined,
    password: env.REDIS_AUTH || undefined,
    sentinelUsername: env.REDIS_SENTINEL_USERNAME || undefined,
    sentinelPassword: env.REDIS_SENTINEL_PASSWORD || undefined,
    ...(sentinelTlsRequested && redisTlsEnabled && tlsOptions.tls
      ? {
          enableTLSForSentinelMode: true,
          sentinelTLS: tlsOptions.tls,
        }
      : {}),
    ...defaultRedisOptions,
    ...additionalOptions,
    ...tlsOptions,
  });

  instance.on("error", (error) => {
    logRedisError("Redis sentinel error", error);
  });

  return instance;
};

/**
 * Every connection handed out by createNewRedisInstance, so that a shutdown can
 * release all of them. Each queue owns a dedicated client, and those clients
 * retry forever; left connected they keep the event loop alive and the process
 * never exits.
 */
const activeRedisInstances = new Set<Redis | Cluster>();

const trackRedisInstance = <T extends Redis | Cluster | null>(
  instance: T,
): T => {
  if (instance) {
    activeRedisInstances.add(instance);
    instance.once("end", () => activeRedisInstances.delete(instance));
  }
  return instance;
};

/**
 * Disconnect every client created by createNewRedisInstance, including the
 * shared `redis` client, which is created through the same path. Returns how
 * many were closed, for shutdown logging.
 */
export const disconnectAllRedisInstances = (): number => {
  let closed = 0;
  for (const instance of activeRedisInstances) {
    try {
      instance.disconnect();
      closed++;
    } catch (error) {
      logRedisError("Failed to disconnect Redis instance on shutdown", error);
    }
  }
  activeRedisInstances.clear();
  return closed;
};

export const createNewRedisInstance = (
  additionalOptions: Partial<RedisOptions> = {},
): Redis | Cluster | null => {
  if (
    env.REDIS_CLUSTER_ENABLED === "true" &&
    env.REDIS_SENTINEL_ENABLED === "true"
  ) {
    logger.error(
      "Invalid Redis configuration: REDIS_CLUSTER_ENABLED and REDIS_SENTINEL_ENABLED cannot both be true",
    );
    return null;
  }

  if (env.REDIS_CLUSTER_ENABLED === "true") {
    return trackRedisInstance(createRedisClusterInstance(additionalOptions));
  }

  if (env.REDIS_SENTINEL_ENABLED === "true") {
    return trackRedisInstance(createRedisSentinelInstance(additionalOptions));
  }

  const tlsOptions = buildTlsOptions();

  const instance = (() => {
    if (env.REDIS_CONNECTION_STRING) {
      return new Redis(env.REDIS_CONNECTION_STRING, {
        ...defaultRedisOptions,
        ...additionalOptions,
        ...tlsOptions,
      });
    }
    if (env.REDIS_HOST) {
      return new Redis({
        host: String(env.REDIS_HOST),
        port: Number(env.REDIS_PORT),
        username: env.REDIS_USERNAME || undefined,
        password: env.REDIS_AUTH || undefined,
        ...defaultRedisOptions,
        ...additionalOptions,
        ...tlsOptions,
      });
    }
    return null;
  })();

  instance?.on("error", (error) => {
    logRedisError("Redis error", error);
  });

  return trackRedisInstance(instance);
};

/**
 * Get the queue prefix for BullMQ cluster compatibility
 * In cluster mode, uses hash tags to ensure queue keys are on the same node
 * In single-node mode, returns the configured prefix or undefined
 */
export const getQueuePrefix = (queueName: string): string | undefined => {
  const redisKeyPrefix = env.REDIS_KEY_PREFIX;

  if (env.REDIS_CLUSTER_ENABLED === "true") {
    // Use hash tags for Redis cluster compatibility
    // This ensures all keys for a queue are placed on the same hash slot
    // Format: {prefix:queueName} ensures all keys land on same slot
    return redisKeyPrefix
      ? `{${redisKeyPrefix}:${queueName}}`
      : `{${queueName}}`;
  }

  // Non-cluster mode: Return prefix or undefined
  return redisKeyPrefix ?? undefined;
};

const getBullMQOptionsForRedisConnection = (
  queueName: string,
  connection: BullMQOptionsWithRedis["connection"],
): BullMQOptionsWithRedis => ({
  connection,
  prefix: getQueuePrefix(queueName),
  // bullmq hashes with md5 by default, and an OpenSSL FIPS provider refuses
  // md5, so any call into the legacy repeat API throws
  // ERR_OSSL_EVP_UNSUPPORTED on a FIPS image. Job scheduler registration does
  // not hash and is unaffected, but the legacy-schedule cleanup in
  // scheduleRecurringJob calls removeRepeatableByKey, which hashes before it
  // reaches Redis. sha256 is FIPS-approved and keeps that call working; the
  // digest itself is only used to build an identifier the removal script
  // never reads.
  settings: { repeatKeyHashAlgorithm: "sha256" },
  ...(env.LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK === "true"
    ? { skipVersionCheck: true }
    : {}),
});

/**
 * Creates a new Redis connection and returns the BullMQ queue options that use it.
 * Returns null only when the Redis connection cannot be created.
 */
export const createBullMQQueueOptionsWithRedis = (
  queueName: string,
): BullMQOptionsWithRedis | null => {
  const connection = createNewRedisInstance({
    enableOfflineQueue: false,
    ...redisQueueRetryOptions,
  });

  return connection
    ? getBullMQOptionsForRedisConnection(queueName, connection)
    : null;
};

/**
 * Creates a new Redis connection and returns the BullMQ worker options that use it.
 * Returns null only when the Redis connection cannot be created.
 */
export const createBullMQWorkerOptionsWithRedis = (
  queueName: string,
): BullMQOptionsWithRedis | null => {
  const connection = createNewRedisInstance(redisQueueRetryOptions);

  return connection
    ? getBullMQOptionsForRedisConnection(queueName, connection)
    : null;
};

/**
 * Execute multiple Redis DEL operations safely in cluster mode
 */
export const safeMultiDel = async (
  redis: Redis | Cluster | null,
  keys: string[],
): Promise<void> => {
  if (!redis || keys.length === 0) return;

  if (env.REDIS_CLUSTER_ENABLED === "true") {
    // In cluster mode, delete keys in separate commands to avoid CROSSSLOT errors
    await Promise.all(keys.map(async (key: string) => redis.del(key)));
  } else {
    // In single-node mode, can delete all keys at once
    await redis.del(keys);
  }
};

/**
 * Execute multiple Redis GET operations safely in cluster mode.
 * MGET requires all keys to hash to the same slot; fall back to per-key GET.
 */
export const safeMultiGet = async (
  redis: Redis | Cluster | null,
  keys: string[],
): Promise<(string | null)[]> => {
  if (!redis || keys.length === 0) return [];

  if (env.REDIS_CLUSTER_ENABLED === "true") {
    return Promise.all(keys.map(async (key: string) => redis.get(key)));
  }

  return redis.mget(keys);
};

const scanKeysForNode = async (
  client: Redis,
  pattern: string,
  collector: Set<string>,
  keyPrefix: string,
) => {
  let cursor = "0";
  // ioredis keyPrefix is not applied to SCAN patterns, but it is applied to
  // DEL/GET/SET keys. Scan physical keys and return logical keys to callers.
  const scanPattern = keyPrefix ? `${keyPrefix}${pattern}` : pattern;

  do {
    const [nextCursor, keys]: [string, string[]] = await client.scan(
      cursor,
      "MATCH",
      scanPattern,
      "COUNT",
      REDIS_SCAN_COUNT,
    );

    keys.forEach((key) =>
      collector.add(
        keyPrefix && key.startsWith(keyPrefix)
          ? key.slice(keyPrefix.length)
          : key,
      ),
    );
    cursor = nextCursor;
  } while (cursor !== "0");
};

const getRedisKeyPrefix = (redis: Redis | Cluster): string => {
  const keyPrefix =
    redis.options?.keyPrefix ??
    (redis instanceof Cluster ? redis.options.redisOptions?.keyPrefix : "");

  return keyPrefix?.toString() ?? "";
};

export const scanKeys = async (
  redis: Redis | Cluster | null,
  pattern: string,
): Promise<string[]> => {
  if (!redis) return [];

  const collectedKeys = new Set<string>();
  const keyPrefix = getRedisKeyPrefix(redis);

  if (env.REDIS_CLUSTER_ENABLED === "true") {
    await Promise.all(
      (redis as Cluster)
        .nodes("master")
        .map((node) =>
          scanKeysForNode(node, pattern, collectedKeys, keyPrefix),
        ),
    );
  } else {
    await scanKeysForNode(redis as Redis, pattern, collectedKeys, keyPrefix);
  }

  return Array.from(collectedKeys);
};

const createRedisClient = () => {
  try {
    return createNewRedisInstance({
      keyPrefix: env.REDIS_KEY_PREFIX ?? undefined,
    });
  } catch (e) {
    logger.error("Failed to connect to redis", e);
    return null;
  }
};

declare global {
  var redis: undefined | ReturnType<typeof createRedisClient>;
}

export const redis = globalThis.redis ?? createRedisClient();

if (env.NODE_ENV !== "production") globalThis.redis = redis;