Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🚢 fix: Reconnect Keyv Redis After Replica Failover (#15502)
Danny Avila519e64769a74
diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.tsindex 16b016f9bd..19f87e6d22 100644--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.ts+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.in_memory_memoization.spec.ts@@ -5,6 +5,7 @@ jest.mock('@keyv/redis', () => ({}));jest.mock('../../redisClients', () => ({+ handleKeyvRedisError: jest.fn(),keyvRedisClient: null,ioredisClient: null,}));diff --git a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.tsindex 9a8b4ff3bf..32a1492555 100644--- a/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts+++ b/packages/api/src/cache/__tests__/cacheFactory/standardCache.namespace_isolation.spec.ts@@ -15,6 +15,7 @@ jest.mock('@keyv/redis', () => ({const mockKeyvRedisClient = { scanIterator: jest.fn() };jest.mock('../../redisClients', () => ({+ handleKeyvRedisError: jest.fn(),keyvRedisClient: mockKeyvRedisClient,ioredisClient: null,}));diff --git a/packages/api/src/cache/__tests__/recovery.spec.ts b/packages/api/src/cache/__tests__/recovery.spec.tsnew file mode 100644index 0000000000..1cbd3eb232--- /dev/null+++ b/packages/api/src/cache/__tests__/recovery.spec.ts@@ -0,0 +1,246 @@+import { createClient } from '@keyv/redis';+import type { RedisClientType } from '@redis/client';+import type { RespServer } from './resp.helper';+import { createReadonlyRecovery, isReadonlyReplicaError } from '~/cache/recovery';+import { closeRedisClients } from './redisClients.helper';+import { startRespServer, waitFor } from './resp.helper';++const READONLY_MESSAGE = "READONLY You can't write against a read only replica.";++const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));++async function readonlyWriteError(client: RedisClientType): Promise<unknown> {+ try {+ await client.set('key', 'value');+ } catch (error) {+ return error;+ }+ throw new Error('Expected the write to be rejected');+}++describe('isReadonlyReplicaError', () => {+ it('recognizes READONLY replies and nothing else', () => {+ expect(isReadonlyReplicaError(new Error(READONLY_MESSAGE))).toBe(true);+ expect(isReadonlyReplicaError(READONLY_MESSAGE)).toBe(true);+ expect(isReadonlyReplicaError(new Error('ECONNRESET'))).toBe(false);+ expect(isReadonlyReplicaError(undefined)).toBe(false);+ });+});++describe('createReadonlyRecovery', () => {+ let server: RespServer;+ let client: RedisClientType;++ const connectClient = async (+ reconnectStrategy: false | ((retries: number) => number),+ ): Promise<void> => {+ client = createClient({ url: server.url, socket: { reconnectStrategy } }) as RedisClientType;+ client.on('error', () => undefined);+ await client.connect();+ };++ beforeEach(async () => {+ server = await startRespServer();+ await connectClient(false);+ });++ afterEach(async () => {+ if (client.isOpen) {+ client.destroy();+ }+ await server.close();+ });++ it('reconnects once per interval and keeps retrying while writes stay READONLY', async () => {+ let clock = 0;+ const recover = createReadonlyRecovery({ client, minIntervalMs: 200, now: () => clock });+ server.readonly = true;+ const error = await readonlyWriteError(client);+ expect(error).toBeInstanceOf(Error);+ expect((error as Error).message).toBe(READONLY_MESSAGE);++ expect(recover(error)).toBe(true);+ for (let i = 0; i < 50; i++) {+ expect(recover(error)).toBe(false);+ }+ await waitFor(() => server.connections === 2 && client.isReady);++ expect(recover(await readonlyWriteError(client))).toBe(false);+ clock = 200;+ expect(recover(await readonlyWriteError(client))).toBe(true);+ await waitFor(() => server.connections === 3 && client.isReady);++ server.readonly = false;+ await expect(client.set('key', 'value')).resolves.toBe('OK');+ await expect(client.get('key')).resolves.toBe('value');+ });++ it('ignores errors that are not READONLY replies', async () => {+ const recover = createReadonlyRecovery({ client, minIntervalMs: 0 });+ expect(recover(new Error('ECONNRESET'))).toBe(false);+ expect(recover(undefined)).toBe(false);+ await sleep(50);+ expect(server.connections).toBe(1);+ });++ it('retries on the next error of any kind after a failed reconnect', async () => {+ const recover = createReadonlyRecovery({ client, minIntervalMs: 0 });+ const { port } = server;+ await server.close();+ await waitFor(() => !client.isOpen);+ expect(recover(new Error('The client is closed'))).toBe(false);++ server = await startRespServer(port);+ await client.connect();+ server.readonly = true;+ expect(recover(await readonlyWriteError(client))).toBe(true);+ await server.close();+ await waitFor(() => !client.isOpen && server.connections === 1);++ expect(recover(new Error('ECONNRESET'))).toBe(true);+ await waitFor(() => !client.isOpen);++ server = await startRespServer(port);+ expect(recover(new Error('The client is closed'))).toBe(true);+ await waitFor(() => server.connections === 1 && client.isReady);+ await expect(client.set('key', 'value')).resolves.toBe('OK');+ });++ it('stays out of the way while node-redis runs its own reconnect loop', async () => {+ client.destroy();+ await connectClient(() => 20);+ const recover = createReadonlyRecovery({ client, minIntervalMs: 0 });+ const { port } = server;+ await server.close();+ await waitFor(() => client.isOpen && !client.isReady);++ expect(recover(new Error(READONLY_MESSAGE))).toBe(false);+ expect(recover(new Error('ECONNRESET'))).toBe(false);++ server = await startRespServer(port);+ await waitFor(() => client.isReady);+ await sleep(50);+ expect(server.connections).toBe(1);+ });++ it('does not tear down a client that reconnected on its own after a failed attempt', async () => {+ const recover = createReadonlyRecovery({ client, minIntervalMs: 0 });+ const { port } = server;+ server.readonly = true;+ const error = await readonlyWriteError(client);+ await server.close();+ expect(recover(error)).toBe(true);+ await waitFor(() => !client.isOpen);++ server = await startRespServer(port);+ await client.connect();+ expect(+ recover(new Error('WRONGTYPE Operation against a key holding the wrong kind of value')),+ ).toBe(false);+ await sleep(50);+ expect(server.connections).toBe(1);+ expect(client.isReady).toBe(true);+ });+});++describe('standalone Keyv Redis client READONLY recovery', () => {+ let originalEnv: NodeJS.ProcessEnv;+ let server: RespServer;+ let clients: typeof import('~/cache/redisClients');+ let cacheFactory: typeof import('~/cache/cacheFactory');+ const keyvClient = (): RedisClientType => clients.keyvRedisClient as RedisClientType;++ beforeAll(async () => {+ originalEnv = { ...process.env };+ server = await startRespServer();+ process.env.USE_REDIS = 'true';+ process.env.USE_REDIS_CLUSTER = 'false';+ process.env.REDIS_URI = server.url;+ process.env.REDIS_PING_INTERVAL = '0';+ process.env.REDIS_KEY_PREFIX = 'readonly-recovery';+ process.env.REDIS_READONLY_RECOVERY_INTERVAL = '100';+ process.env.REDIS_RETRY_MAX_ATTEMPTS = '2';+ process.env.REDIS_RETRY_MAX_DELAY = '50';+ jest.resetModules();+ clients = await import('~/cache/redisClients');+ cacheFactory = await import('~/cache/cacheFactory');+ await clients.keyvRedisClientReady;+ });++ afterAll(async () => {+ await closeRedisClients(clients);+ await server.close();+ process.env = originalEnv;+ });++ it('reconnects when a Keyv write is rejected with READONLY', async () => {+ const connectionsBefore = server.connections;+ const cache = cacheFactory.standardCache('readonly-recovery-test');+ server.readonly = true;++ await cache.set('key', 'value');+ await waitFor(() => server.connections === connectionsBefore + 1 && keyvClient().isReady);++ server.readonly = false;+ await expect(cache.set('key', 'value')).resolves.toBe(true);+ await expect(cache.get('key')).resolves.toBe('value');+ });++ it('reconnects when a Lua script is rejected with READONLY', async () => {+ await sleep(100);+ const connectionsBefore = server.connections;+ server.readonly = true;++ await expect(+ clients.evalKeyvRedisScript('return 1', { keys: ['lock'], arguments: [] }),+ ).rejects.toThrow(READONLY_MESSAGE);+ await waitFor(() => server.connections === connectionsBefore + 1 && keyvClient().isReady);++ server.readonly = false;+ await expect(+ clients.evalKeyvRedisScript('return 1', { keys: ['lock'], arguments: [] }),+ ).resolves.toBe(1);+ });++ it('reconnects when a namespace clear is rejected with READONLY', async () => {+ await sleep(100);+ const connectionsBefore = server.connections;+ const cache = cacheFactory.standardCache('readonly-clear-test');+ await cache.set('key', 'value');+ server.readonly = true;++ await expect(cache.clear()).rejects.toThrow(READONLY_MESSAGE);+ await waitFor(() => server.connections === connectionsBefore + 1 && keyvClient().isReady);+ server.readonly = false;+ });++ it('routes errors handed to handleKeyvRedisError through the same recovery', async () => {+ await sleep(100);+ const connectionsBefore = server.connections;+ expect(clients.handleKeyvRedisError(new Error(READONLY_MESSAGE))).toBe(true);+ await waitFor(() => server.connections === connectionsBefore + 1 && keyvClient().isReady);+ });++ it('recovers through Keyv auto-connect after a failed reconnect without duplicate connections', async () => {+ await sleep(100);+ const cache = cacheFactory.standardCache('readonly-outage-test');+ const { port } = server;+ server.readonly = true;++ await expect(+ clients.evalKeyvRedisScript('return 1', { keys: ['lock'], arguments: [] }),+ ).rejects.toThrow(READONLY_MESSAGE);+ await server.close();+ await waitFor(() => !keyvClient().isOpen, 5000);++ await cache.set('key', 'value');+ await waitFor(() => !keyvClient().isOpen, 5000);++ server = await startRespServer(port);+ await expect(cache.set('key', 'value')).resolves.toBe(true);+ await waitFor(() => keyvClient().isReady);+ expect(server.connections).toBe(1);+ await expect(cache.get('key')).resolves.toBe('value');+ expect(server.connections).toBe(1);+ });+});diff --git a/packages/api/src/cache/__tests__/resp.helper.ts b/packages/api/src/cache/__tests__/resp.helper.tsnew file mode 100644index 0000000000..4da642c173--- /dev/null+++ b/packages/api/src/cache/__tests__/resp.helper.ts@@ -0,0 +1,167 @@+import net from 'node:net';+import type { AddressInfo, Socket } from 'node:net';++/**+ * Minimal RESP2 server standing in for a Redis node whose role can be flipped+ * between master and demoted replica. Real node-redis and ioredis clients speak+ * to it over TCP, which is what a READONLY failover scenario needs: the socket+ * stays open and healthy while every write is rejected.+ */+export type RespServer = {+ url: string;+ port: number;+ /** Total sockets accepted since start; a reconnect shows up as a new one. */+ connections: number;+ /** When true, write commands are rejected with the READONLY reply. */+ readonly: boolean;+ commands: string[][];+ close(): Promise<void>;+};++const READONLY_REPLY = "-READONLY You can't write against a read only replica.\r\n";+const WRITE_COMMANDS = new Set(['SET', 'DEL', 'UNLINK', 'EVAL', 'EVALSHA', 'INCR', 'EXPIRE']);++function encodeBulk(value: string | null): string {+ return value == null ? '$-1\r\n' : `$${Buffer.byteLength(value)}\r\n${value}\r\n`;+}++function globToRegExp(pattern: string): RegExp {+ const escaped = pattern+ .replace(/[.+^${}()|[\]\\]/g, '\\$&')+ .replace(/\*/g, '.*')+ .replace(/\?/g, '.');+ return new RegExp(`^${escaped}$`);+}++function encodeScan(store: Map<string, string>, args: string[]): string {+ const matchIndex = args.findIndex((arg) => arg.toUpperCase() === 'MATCH');+ const matcher = matchIndex === -1 ? /^/ : globToRegExp(args[matchIndex + 1]);+ const keys = [...store.keys()].filter((key) => matcher.test(key));+ return `*2\r\n$1\r\n0\r\n*${keys.length}\r\n${keys.map(encodeBulk).join('')}`;+}++function parseFrames(buffer: Buffer): { frames: string[][]; rest: Buffer } {+ const frames: string[][] = [];+ let offset = 0;+ while (offset < buffer.length) {+ if (buffer[offset] !== 0x2a) {+ break;+ }+ const headerEnd = buffer.indexOf('\r\n', offset);+ if (headerEnd === -1) {+ break;+ }+ const count = Number(buffer.subarray(offset + 1, headerEnd).toString());+ let cursor = headerEnd + 2;+ const args: string[] = [];+ let complete = true;+ for (let i = 0; i < count; i++) {+ const lengthEnd = buffer.indexOf('\r\n', cursor);+ if (lengthEnd === -1 || buffer[cursor] !== 0x24) {+ complete = false;+ break;+ }+ const length = Number(buffer.subarray(cursor + 1, lengthEnd).toString());+ const valueStart = lengthEnd + 2;+ const valueEnd = valueStart + length;+ if (buffer.length < valueEnd + 2) {+ complete = false;+ break;+ }+ args.push(buffer.subarray(valueStart, valueEnd).toString());+ cursor = valueEnd + 2;+ }+ if (!complete) {+ break;+ }+ frames.push(args);+ offset = cursor;+ }+ return { frames, rest: buffer.subarray(offset) };+}++export async function startRespServer(port = 0): Promise<RespServer> {+ const store = new Map<string, string>();+ const sockets = new Set<Socket>();+ const state = { connections: 0, readonly: false, commands: [] as string[][] };++ const reply = (args: string[]): string => {+ const command = args[0]?.toUpperCase() ?? '';+ if (state.readonly && WRITE_COMMANDS.has(command)) {+ return READONLY_REPLY;+ }+ switch (command) {+ case 'PING':+ return '+PONG\r\n';+ case 'INFO':+ return encodeBulk('# Server\r\nredis_version:7.2.4\r\nloading:0\r\n');+ case 'GET':+ return encodeBulk(store.get(args[1]) ?? null);+ case 'SET':+ store.set(args[1], args[2]);+ return '+OK\r\n';+ case 'DEL':+ case 'UNLINK':+ return `:${args.slice(1).filter((key) => store.delete(key)).length}\r\n`;+ case 'EVAL':+ case 'EVALSHA':+ return ':1\r\n';+ case 'SCAN':+ return encodeScan(store, args);+ default:+ return '+OK\r\n';+ }+ };++ const server = net.createServer((socket) => {+ state.connections += 1;+ sockets.add(socket);+ let pending: Buffer = Buffer.alloc(0);+ socket.on('data', (chunk) => {+ const { frames, rest } = parseFrames(Buffer.concat([pending, chunk]));+ pending = rest;+ for (const frame of frames) {+ state.commands.push(frame);+ socket.write(reply(frame));+ }+ });+ socket.on('close', () => sockets.delete(socket));+ socket.on('error', () => undefined);+ });++ await new Promise<void>((resolve) => server.listen(port, '127.0.0.1', resolve));+ const { port: boundPort } = server.address() as AddressInfo;++ return {+ url: `redis://127.0.0.1:${boundPort}`,+ port: boundPort,+ get connections() {+ return state.connections;+ },+ get readonly() {+ return state.readonly;+ },+ set readonly(value: boolean) {+ state.readonly = value;+ },+ commands: state.commands,+ close: () =>+ new Promise<void>((resolve) => {+ for (const socket of sockets) {+ socket.destroy();+ }+ server.close(() => resolve());+ }),+ };+}++/** Polls until `predicate` holds, failing after `timeoutMs`. */+export async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {+ const deadline = Date.now() + timeoutMs;+ while (!predicate()) {+ if (Date.now() > deadline) {+ throw new Error('Timed out waiting for condition');+ }+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+}diff --git a/packages/api/src/cache/cacheConfig.ts b/packages/api/src/cache/cacheConfig.tsindex 303fb968c8..619662c416 100644--- a/packages/api/src/cache/cacheConfig.ts+++ b/packages/api/src/cache/cacheConfig.ts@@ -94,6 +94,8 @@ const cacheConfig: {REDIS_RETRY_MAX_ATTEMPTS: number;/** Connection timeout in ms */REDIS_CONNECT_TIMEOUT: number;+ /** Min spacing in ms between reconnects forced by READONLY replies after a failover */+ REDIS_READONLY_RECOVERY_INTERVAL: number;/** Queue commands when disconnected */REDIS_ENABLE_OFFLINE_QUEUE: boolean;/** flag to modify redis connection by adding dnsLookup this is required when connecting to elasticache for ioredis@@ -171,6 +173,8 @@ const cacheConfig: {REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10),/** Connection timeout in ms */REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000),+ /** Min spacing in ms between reconnects forced by READONLY replies after a failover */+ REDIS_READONLY_RECOVERY_INTERVAL: math(process.env.REDIS_READONLY_RECOVERY_INTERVAL, 5000),/** Queue commands when disconnected */REDIS_ENABLE_OFFLINE_QUEUE: isEnabled(process.env.REDIS_ENABLE_OFFLINE_QUEUE ?? 'true'),/** flag to modify redis connection by adding dnsLookup this is required when connecting to elasticache for ioredisdiff --git a/packages/api/src/cache/cacheFactory.ts b/packages/api/src/cache/cacheFactory.tsindex 4e760058e8..5a45e2578c 100644--- a/packages/api/src/cache/cacheFactory.ts+++ b/packages/api/src/cache/cacheFactory.ts@@ -14,7 +14,7 @@ import session, { MemoryStore } from 'express-session';import { Time, CacheKeys } from 'librechat-data-provider';import { RedisStore as ConnectRedis } from 'connect-redis';import type { SendCommandFn } from 'rate-limit-redis';-import { keyvRedisClient, ioredisClient } from './redisClients';+import { keyvRedisClient, ioredisClient, handleKeyvRedisError } from './redisClients';import { batchDeleteKeys, scanKeys } from './redisUtils';import {instrumentIORedisClient,@@ -35,6 +35,34 @@ import { violationFile } from './keyvFiles';*/const inMemoryCacheMap = new Map<string, Keyv>();+/**+ * Deletes every key under a namespace through the raw client, which is the one+ * write path that bypasses the Keyv error funnel; READONLY rejections are routed+ * to failover recovery before propagating.+ */+async function clearRedisNamespace(namespace: string): Promise<void> {+ if (!keyvRedisClient || !('scanIterator' in keyvRedisClient)) {+ logger.warn(`Cannot clear namespace ${namespace}: Redis scanIterator not available`);+ return;+ }++ const pattern = cacheConfig.REDIS_KEY_PREFIX+ ? `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}${namespace}:*`+ : `${namespace}:*`;++ try {+ const keysToDelete = await scanKeys(keyvRedisClient, pattern);+ if (keysToDelete.length === 0) {+ return;+ }+ await batchDeleteKeys(keyvRedisClient, keysToDelete);+ logger.debug(`Cleared ${keysToDelete.length} keys from namespace ${namespace}`);+ } catch (error) {+ handleKeyvRedisError(error);+ throw error;+ }+}+/*** Creates a cache instance using Redis or a fallback store. Suitable for general caching needs.*@@ -57,33 +85,13 @@ export const standardCache = (namespace: string, ttl?: number, fallbackStore?: ocache.on('error', (err) => {logger.error(`Cache error in namespace ${namespace}:`, err);+ handleKeyvRedisError(err);});// Override clear() to handle namespace-aware deletion// The default Keyv clear() doesn't respect namespace due to the workaround above// Workaround for issue #10487 https://github.com/danny-avila/LibreChat/issues/10487- cache.clear = async () => {- // Type-safe check for Redis client with scanIterator support- if (!keyvRedisClient || !('scanIterator' in keyvRedisClient)) {- logger.warn(`Cannot clear namespace ${namespace}: Redis scanIterator not available`);- return;- }-- // Build pattern: globalPrefix::namespace:* or namespace:*- const pattern = cacheConfig.REDIS_KEY_PREFIX- ? `${cacheConfig.REDIS_KEY_PREFIX}${cacheConfig.GLOBAL_PREFIX_SEPARATOR}${namespace}:*`- : `${namespace}:*`;-- // Use utility functions for efficient scan and parallel deletion- const keysToDelete = await scanKeys(keyvRedisClient, pattern);-- if (keysToDelete.length === 0) {- return;- }-- await batchDeleteKeys(keyvRedisClient, keysToDelete);- logger.debug(`Cleared ${keysToDelete.length} keys from namespace ${namespace}`);- };+ cache.clear = () => clearRedisNamespace(namespace);return instrumentRedisCache(cache, namespace);} catch (err) {diff --git a/packages/api/src/cache/recovery.ts b/packages/api/src/cache/recovery.tsnew file mode 100644index 0000000000..86a4d6a9ce--- /dev/null+++ b/packages/api/src/cache/recovery.ts@@ -0,0 +1,116 @@+import { logger } from '@librechat/data-schemas';++/** Reply prefix Redis returns for writes sent to a demoted replica. */+export const READONLY_ERROR_PREFIX = 'READONLY';++export type ReconnectableClient = {+ readonly isOpen: boolean;+ readonly isReady: boolean;+ destroy(): void;+ connect(): Promise<unknown>;+};++export type ReadonlyRecoveryOptions = {+ client: ReconnectableClient;+ /**+ * Minimum spacing between reconnect attempts. A failover produces READONLY+ * replies by the hundred per second, and a reconnect issued before the+ * topology settles can land on the demoted node again, so attempts are+ * spaced out and retried on the next error instead of given up on.+ */+ minIntervalMs: number;+ /** Monotonic clock in milliseconds; defaults to `performance.now`. */+ now?: () => number;+ /** Client label used in log lines. */+ label?: string;+};++/** Handler returning whether the error started a reconnect attempt. */+export type ReadonlyRecoveryHandler = (error: unknown) => boolean;++function errorMessage(error: unknown): string {+ if (error instanceof Error) {+ return error.message;+ }+ return typeof error === 'string' ? error : '';+}++/** Whether an error is a READONLY reply from a Redis node that was demoted to replica. */+export function isReadonlyReplicaError(error: unknown): boolean {+ return errorMessage(error).includes(READONLY_ERROR_PREFIX);+}++/**+ * Creates the READONLY recovery hook for a standalone node-redis client.+ *+ * node-redis only reconnects when its socket closes, and a demoted replica keeps+ * existing sockets open while rejecting every write, so a READONLY reply never+ * reaches `socket.reconnectStrategy`. The returned handler mirrors what ioredis+ * does through `reconnectOnError`: it tears the socket down and reconnects, which+ * re-resolves the connection to whatever the master address now points at.+ *+ * Tearing the socket down rejects the commands queued or in flight at that+ * moment (reads included); node-redis cannot replay them the way ioredis does.+ * That is a bounded, once-per-attempt cost, traded against every write failing+ * until the process restarts.+ *+ * READONLY replies reject straight to the command's promise and never reach the+ * client's error event, so the handler is meant to be called from every error+ * funnel that sees command failures: the Keyv cache error event, Lua scripts+ * evaluated directly against the client, and namespace clears. It debounces+ * internally and never throws.+ *+ * The hook only acts on a client it can reason about: a READONLY reply on a+ * ready socket, or a client its own failed reconnect left closed. While+ * node-redis runs its own reconnect loop (open but not ready) the hook stays+ * out of the way, since a second `connect()` would start a concurrent loop+ * that leaks sockets into one shared reply decoder.+ */+export function createReadonlyRecovery(options: ReadonlyRecoveryOptions): ReadonlyRecoveryHandler {+ const { client, minIntervalMs, label = '@keyv/redis' } = options;+ const now = options.now ?? (() => performance.now());+ let inFlight = false;+ let retryPending = false;+ let lastAttemptAt = Number.NEGATIVE_INFINITY;++ const reconnect = async (): Promise<void> => {+ logger.warn(`${label} client reconnecting due to READONLY error`);+ if (client.isOpen) {+ client.destroy();+ }+ await client.connect();+ logger.info(`${label} client reconnected after READONLY error`);+ };++ const shouldReconnect = (error: unknown): boolean => {+ if (client.isReady) {+ retryPending = false;+ return isReadonlyReplicaError(error);+ }+ return !client.isOpen && retryPending;+ };++ return function recoverIfReadonly(error: unknown): boolean {+ if (inFlight || !shouldReconnect(error)) {+ return false;+ }+ const attemptAt = now();+ if (attemptAt - lastAttemptAt < minIntervalMs) {+ return false;+ }+ inFlight = true;+ lastAttemptAt = attemptAt;+ void reconnect()+ .then(() => {+ retryPending = false;+ })+ .catch((reconnectError: unknown) => {+ retryPending = true;+ logger.error(`${label} client reconnect after READONLY error failed:`, reconnectError);+ })+ .finally(() => {+ inFlight = false;+ });+ return true;+ };+}diff --git a/packages/api/src/cache/redisClients.ts b/packages/api/src/cache/redisClients.tsindex c6f6dcbef0..076d73d99e 100644--- a/packages/api/src/cache/redisClients.ts+++ b/packages/api/src/cache/redisClients.ts@@ -5,6 +5,8 @@ import { createClient, createCluster } from '@keyv/redis';import type { ScanOptions } from '@redis/client/dist/lib/commands/SCAN';import type { RedisClientType, RedisClusterType } from '@redis/client';import type { Redis, Cluster } from 'ioredis';+import type { ReadonlyRecoveryHandler } from './recovery';+import { createReadonlyRecovery, isReadonlyReplicaError } from './recovery';import { cacheConfig } from './cacheConfig';const urls = cacheConfig.REDIS_URI?.split(',').map((uri) => new URL(uri)) || [];@@ -62,8 +64,7 @@ if (cacheConfig.USE_REDIS) {return delay;},reconnectOnError: (err: Error) => {- const targetError = 'READONLY';- if (err.message.includes(targetError)) {+ if (isReadonlyReplicaError(err)) {logger.warn('ioredis reconnecting due to READONLY error');return 2; // Return retry delay instead of boolean}@@ -151,31 +152,57 @@ if (cacheConfig.USE_REDIS) {}let keyvRedisClient: RedisClientType | RedisClusterType | null = null;+let recoverKeyvRedisClient: ReadonlyRecoveryHandler | undefined;++/**+ * Routes a Keyv Redis client error through READONLY failover recovery.+ * Cluster clients follow MOVED to the owning master on their own, so the hook+ * only exists for the standalone client. Returns whether a reconnect was started.+ */+function handleKeyvRedisError(error: unknown): boolean {+ return recoverKeyvRedisClient?.(error) ?? false;+}type RedisEvalOptions = { keys: string[]; arguments: string[] };+async function runKeyvRedisScript(+ client: RedisClientType | RedisClusterType,+ script: string,+ options: RedisEvalOptions,+): Promise<unknown> {+ if (!('masters' in client) || options.keys.length === 0) {+ return client.eval(script, options);+ }++ const slot = calculateSlot(options.keys[0]);+ if (options.keys.some((key) => calculateSlot(key) !== slot)) {+ throw new Error('Redis catalog script keys must share one cluster slot');+ }+ const master = client.getSlotMaster(slot);+ const nodeClient = await client.nodeClient(master);+ return nodeClient.eval(script, options);+}+/*** Runs a Lua script on the master that owns its keys. Node Redis can execute a* cluster EVAL through an arbitrary node while the slot map is settling, which* leaks a MOVED reply instead of following it. Catalog scripts are deliberately* single-slot, so selecting the owning master also makes that invariant explicit.+ *+ * Script failures reject straight to the caller without passing through any+ * client error event, so READONLY replies are routed to failover recovery here.*/async function evalKeyvRedisScript(script: string, options: RedisEvalOptions): Promise<unknown> {await waitForKeyvRedisClient();if (!keyvRedisClient) {throw new Error('Keyv Redis client is not configured');}- if (!('masters' in keyvRedisClient) || options.keys.length === 0) {- return keyvRedisClient.eval(script, options);- }-- const slot = calculateSlot(options.keys[0]);- if (options.keys.some((key) => calculateSlot(key) !== slot)) {- throw new Error('Redis catalog script keys must share one cluster slot');+ try {+ return await runKeyvRedisScript(keyvRedisClient, script, options);+ } catch (error) {+ handleKeyvRedisError(error);+ throw error;}- const master = keyvRedisClient.getSlotMaster(slot);- const nodeClient = await keyvRedisClient.nodeClient(master);- return nodeClient.eval(script, options);}if (cacheConfig.USE_REDIS) {@@ -238,6 +265,13 @@ if (cacheConfig.USE_REDIS) {keyvRedisClient.setMaxListeners(cacheConfig.REDIS_MAX_LISTENERS);+ if (!isRedisCluster) {+ recoverKeyvRedisClient = createReadonlyRecovery({+ client: keyvRedisClient as RedisClientType,+ minIntervalMs: cacheConfig.REDIS_READONLY_RECOVERY_INTERVAL,+ });+ }+keyvRedisClient.on('error', (err) => {logger.error('@keyv/redis client error:', err);});@@ -272,4 +306,5 @@ export {keyvRedisClientReady,waitForKeyvRedisClient,evalKeyvRedisScript,+ handleKeyvRedisError,};