Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

✨ feat: Enhance Redis Config and Error Handling (#8709)

Danny Avilaec3cbca6e367
diff --git a/api/cache/cacheConfig.js b/api/cache/cacheConfig.js
index 87c403bae4..1ca3e902e7 100644
--- a/api/cache/cacheConfig.js
+++ b/api/cache/cacheConfig.js
@@ -44,6 +44,14 @@ const cacheConfig = {
REDIS_KEY_PREFIX: process.env[REDIS_KEY_PREFIX_VAR] || REDIS_KEY_PREFIX || '',
REDIS_MAX_LISTENERS: math(process.env.REDIS_MAX_LISTENERS, 40),
REDIS_PING_INTERVAL: math(process.env.REDIS_PING_INTERVAL, 0),
+ /** Max delay between reconnection attempts in ms */
+ REDIS_RETRY_MAX_DELAY: math(process.env.REDIS_RETRY_MAX_DELAY, 3000),
+ /** Max number of reconnection attempts (0 = infinite) */
+ REDIS_RETRY_MAX_ATTEMPTS: math(process.env.REDIS_RETRY_MAX_ATTEMPTS, 10),
+ /** Connection timeout in ms */
+ REDIS_CONNECT_TIMEOUT: math(process.env.REDIS_CONNECT_TIMEOUT, 10000),
+ /** Queue commands when disconnected */
+ REDIS_ENABLE_OFFLINE_QUEUE: isEnabled(process.env.REDIS_ENABLE_OFFLINE_QUEUE ?? 'true'),
CI: isEnabled(process.env.CI),
DEBUG_MEMORY_CACHE: isEnabled(process.env.DEBUG_MEMORY_CACHE),
diff --git a/api/cache/cacheFactory.js b/api/cache/cacheFactory.js
index b9739f4d39..bc361d6615 100644
--- a/api/cache/cacheFactory.js
+++ b/api/cache/cacheFactory.js
@@ -1,12 +1,13 @@
const KeyvRedis = require('@keyv/redis').default;
const { Keyv } = require('keyv');
-const { cacheConfig } = require('./cacheConfig');
-const { keyvRedisClient, ioredisClient, GLOBAL_PREFIX_SEPARATOR } = require('./redisClients');
+const { RedisStore } = require('rate-limit-redis');
const { Time } = require('librechat-data-provider');
+const { logger } = require('@librechat/data-schemas');
const { RedisStore: ConnectRedis } = require('connect-redis');
const MemoryStore = require('memorystore')(require('express-session'));
+const { keyvRedisClient, ioredisClient, GLOBAL_PREFIX_SEPARATOR } = require('./redisClients');
+const { cacheConfig } = require('./cacheConfig');
const { violationFile } = require('./keyvFiles');
-const { RedisStore } = require('rate-limit-redis');
/**
* Creates a cache instance using Redis or a fallback store. Suitable for general caching needs.
@@ -20,11 +21,21 @@ const standardCache = (namespace, ttl = undefined, fallbackStore = undefined) =>
cacheConfig.USE_REDIS &&
!cacheConfig.FORCED_IN_MEMORY_CACHE_NAMESPACES?.includes(namespace)
) {
- const keyvRedis = new KeyvRedis(keyvRedisClient);
- const cache = new Keyv(keyvRedis, { namespace, ttl });
- keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX;
- keyvRedis.keyPrefixSeparator = GLOBAL_PREFIX_SEPARATOR;
- return cache;
+ try {
+ const keyvRedis = new KeyvRedis(keyvRedisClient);
+ const cache = new Keyv(keyvRedis, { namespace, ttl });
+ keyvRedis.namespace = cacheConfig.REDIS_KEY_PREFIX;
+ keyvRedis.keyPrefixSeparator = GLOBAL_PREFIX_SEPARATOR;
+
+ cache.on('error', (err) => {
+ logger.error(`Cache error in namespace ${namespace}:`, err);
+ });
+
+ return cache;
+ } catch (err) {
+ logger.error(`Failed to create Redis cache for namespace ${namespace}:`, err);
+ throw err;
+ }
}
if (fallbackStore) return new Keyv({ store: fallbackStore, namespace, ttl });
return new Keyv({ namespace, ttl });
@@ -50,7 +61,13 @@ const violationCache = (namespace, ttl = undefined) => {
const sessionCache = (namespace, ttl = undefined) => {
namespace = namespace.endsWith(':') ? namespace : `${namespace}:`;
if (!cacheConfig.USE_REDIS) return new MemoryStore({ ttl, checkPeriod: Time.ONE_DAY });
- return new ConnectRedis({ client: ioredisClient, ttl, prefix: namespace });
+ const store = new ConnectRedis({ client: ioredisClient, ttl, prefix: namespace });
+ if (ioredisClient) {
+ ioredisClient.on('error', (err) => {
+ logger.error(`Session store Redis error for namespace ${namespace}:`, err);
+ });
+ }
+ return store;
};
/**
@@ -62,8 +79,30 @@ const limiterCache = (prefix) => {
if (!prefix) throw new Error('prefix is required');
if (!cacheConfig.USE_REDIS) return undefined;
prefix = prefix.endsWith(':') ? prefix : `${prefix}:`;
- return new RedisStore({ sendCommand, prefix });
+
+ try {
+ if (!ioredisClient) {
+ logger.warn(`Redis client not available for rate limiter with prefix ${prefix}`);
+ return undefined;
+ }
+
+ return new RedisStore({ sendCommand, prefix });
+ } catch (err) {
+ logger.error(`Failed to create Redis rate limiter for prefix ${prefix}:`, err);
+ return undefined;
+ }
+};
+
+const sendCommand = (...args) => {
+ if (!ioredisClient) {
+ logger.warn('Redis client not available for command execution');
+ return Promise.reject(new Error('Redis client not available'));
+ }
+
+ return ioredisClient.call(...args).catch((err) => {
+ logger.error('Redis command execution failed:', err);
+ throw err;
+ });
};
-const sendCommand = (...args) => ioredisClient?.call(...args);
module.exports = { standardCache, sessionCache, violationCache, limiterCache };
diff --git a/api/cache/cacheFactory.spec.js b/api/cache/cacheFactory.spec.js
index 76d01a915c..ce364a4a32 100644
--- a/api/cache/cacheFactory.spec.js
+++ b/api/cache/cacheFactory.spec.js
@@ -6,13 +6,17 @@ const mockKeyvRedis = {
keyPrefixSeparator: '',
};
-const mockKeyv = jest.fn().mockReturnValue({ mock: 'keyv' });
+const mockKeyv = jest.fn().mockReturnValue({
+ mock: 'keyv',
+ on: jest.fn(),
+});
const mockConnectRedis = jest.fn().mockReturnValue({ mock: 'connectRedis' });
const mockMemoryStore = jest.fn().mockReturnValue({ mock: 'memoryStore' });
const mockRedisStore = jest.fn().mockReturnValue({ mock: 'redisStore' });
const mockIoredisClient = {
call: jest.fn(),
+ on: jest.fn(),
};
const mockKeyvRedisClient = {};
@@ -53,6 +57,14 @@ jest.mock('rate-limit-redis', () => ({
RedisStore: mockRedisStore,
}));
+jest.mock('@librechat/data-schemas', () => ({
+ logger: {
+ error: jest.fn(),
+ warn: jest.fn(),
+ info: jest.fn(),
+ },
+}));
+
// Import after mocking
const { standardCache, sessionCache, violationCache, limiterCache } = require('./cacheFactory');
const { cacheConfig } = require('./cacheConfig');
@@ -142,6 +154,28 @@ describe('cacheFactory', () => {
expect(require('@keyv/redis').default).toHaveBeenCalledWith(mockKeyvRedisClient);
expect(mockKeyv).toHaveBeenCalledWith(mockKeyvRedis, { namespace, ttl });
});
+
+ it('should throw error when Redis cache creation fails', () => {
+ cacheConfig.USE_REDIS = true;
+ const namespace = 'test-namespace';
+ const ttl = 3600;
+ const testError = new Error('Redis connection failed');
+
+ const KeyvRedis = require('@keyv/redis').default;
+ KeyvRedis.mockImplementationOnce(() => {
+ throw testError;
+ });
+
+ expect(() => standardCache(namespace, ttl)).toThrow('Redis connection failed');
+
+ const { logger } = require('@librechat/data-schemas');
+ expect(logger.error).toHaveBeenCalledWith(
+ `Failed to create Redis cache for namespace ${namespace}:`,
+ testError,
+ );
+
+ expect(mockKeyv).not.toHaveBeenCalled();
+ });
});
describe('violationCache', () => {
@@ -233,6 +267,86 @@ describe('cacheFactory', () => {
checkPeriod: Time.ONE_DAY,
});
});
+
+ it('should throw error when ConnectRedis constructor fails', () => {
+ cacheConfig.USE_REDIS = true;
+ const namespace = 'sessions';
+ const ttl = 86400;
+
+ // Mock ConnectRedis to throw an error during construction
+ const redisError = new Error('Redis connection failed');
+ mockConnectRedis.mockImplementationOnce(() => {
+ throw redisError;
+ });
+
+ // The error should propagate up, not be caught
+ expect(() => sessionCache(namespace, ttl)).toThrow('Redis connection failed');
+
+ // Verify that MemoryStore was NOT used as fallback
+ expect(mockMemoryStore).not.toHaveBeenCalled();
+ });
+
+ it('should register error handler but let errors propagate to Express', () => {
+ cacheConfig.USE_REDIS = true;
+ const namespace = 'sessions';
+
+ // Create a mock session store with middleware methods
+ const mockSessionStore = {
+ get: jest.fn(),
+ set: jest.fn(),
+ destroy: jest.fn(),
+ };
+ mockConnectRedis.mockReturnValue(mockSessionStore);
+
+ const store = sessionCache(namespace);
+
+ // Verify error handler was registered
+ expect(mockIoredisClient.on).toHaveBeenCalledWith('error', expect.any(Function));
+
+ // Get the error handler
+ const errorHandler = mockIoredisClient.on.mock.calls.find((call) => call[0] === 'error')[1];
+
+ // Simulate an error from Redis during a session operation
+ const redisError = new Error('Socket closed unexpectedly');
+
+ // The error handler should log but not swallow the error
+ const { logger } = require('@librechat/data-schemas');
+ errorHandler(redisError);
+
+ expect(logger.error).toHaveBeenCalledWith(
+ `Session store Redis error for namespace ${namespace}::`,
+ redisError,
+ );
+
+ // Now simulate what happens when session middleware tries to use the store
+ const callback = jest.fn();
+ mockSessionStore.get.mockImplementation((sid, cb) => {
+ cb(new Error('Redis connection lost'));
+ });
+
+ // Call the store's get method (as Express session would)
+ store.get('test-session-id', callback);
+
+ // The error should be passed to the callback, not swallowed
+ expect(callback).toHaveBeenCalledWith(new Error('Redis connection lost'));
+ });
+
+ it('should handle null ioredisClient gracefully', () => {
+ cacheConfig.USE_REDIS = true;
+ const namespace = 'sessions';
+
+ // Temporarily set ioredisClient to null (simulating connection not established)
+ const originalClient = require('./redisClients').ioredisClient;
+ require('./redisClients').ioredisClient = null;
+
+ // ConnectRedis might accept null client but would fail on first use
+ // The important thing is it doesn't throw uncaught exceptions during construction
+ const store = sessionCache(namespace);
+ expect(store).toBeDefined();
+
+ // Restore original client
+ require('./redisClients').ioredisClient = originalClient;
+ });
});
describe('limiterCache', () => {
@@ -274,8 +388,10 @@ describe('cacheFactory', () => {
});
});
- it('should pass sendCommand function that calls ioredisClient.call', () => {
+ it('should pass sendCommand function that calls ioredisClient.call', async () => {
cacheConfig.USE_REDIS = true;
+ mockIoredisClient.call.mockResolvedValue('test-value');
+
limiterCache('rate-limit');
const sendCommandCall = mockRedisStore.mock.calls[0][0];
@@ -283,8 +399,28 @@ describe('cacheFactory', () => {
// Test that sendCommand properly delegates to ioredisClient.call
const args = ['GET', 'test-key'];
- sendCommand(...args);
+ const result = await sendCommand(...args);
+
+ expect(mockIoredisClient.call).toHaveBeenCalledWith(...args);
+ expect(result).toBe('test-value');
+ });
+
+ it('should handle sendCommand errors properly', async () => {
+ cacheConfig.USE_REDIS = true;
+
+ // Mock the call method to reject with an error
+ const testError = new Error('Redis error');
+ mockIoredisClient.call.mockRejectedValue(testError);
+
+ limiterCache('rate-limit');
+
+ const sendCommandCall = mockRedisStore.mock.calls[0][0];
+ const sendCommand = sendCommandCall.sendCommand;
+
+ // Test that sendCommand properly handles errors
+ const args = ['GET', 'test-key'];
+ await expect(sendCommand(...args)).rejects.toThrow('Redis error');
expect(mockIoredisClient.call).toHaveBeenCalledWith(...args);
});
diff --git a/api/cache/redisClients.js b/api/cache/redisClients.js
index 46c2813e95..17889d24f3 100644
--- a/api/cache/redisClients.js
+++ b/api/cache/redisClients.js
@@ -13,23 +13,82 @@ const ca = cacheConfig.REDIS_CA;
/** @type {import('ioredis').Redis | import('ioredis').Cluster | null} */
let ioredisClient = null;
if (cacheConfig.USE_REDIS) {
+ /** @type {import('ioredis').RedisOptions | import('ioredis').ClusterOptions} */
const redisOptions = {
username: username,
password: password,
tls: ca ? { ca } : undefined,
keyPrefix: `${cacheConfig.REDIS_KEY_PREFIX}${GLOBAL_PREFIX_SEPARATOR}`,
maxListeners: cacheConfig.REDIS_MAX_LISTENERS,
+ retryStrategy: (times) => {
+ if (
+ cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
+ times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
+ ) {
+ logger.error(
+ `ioredis giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
+ );
+ return null;
+ }
+ const delay = Math.min(times * 50, cacheConfig.REDIS_RETRY_MAX_DELAY);
+ logger.info(`ioredis reconnecting... attempt ${times}, delay ${delay}ms`);
+ return delay;
+ },
+ reconnectOnError: (err) => {
+ const targetError = 'READONLY';
+ if (err.message.includes(targetError)) {
+ logger.warn('ioredis reconnecting due to READONLY error');
+ return true;
+ }
+ return false;
+ },
+ enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
+ connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
+ maxRetriesPerRequest: 3,
};
ioredisClient =
urls.length === 1
? new IoRedis(cacheConfig.REDIS_URI, redisOptions)
- : new IoRedis.Cluster(cacheConfig.REDIS_URI, { redisOptions });
+ : new IoRedis.Cluster(cacheConfig.REDIS_URI, {
+ redisOptions,
+ clusterRetryStrategy: (times) => {
+ if (
+ cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
+ times > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
+ ) {
+ logger.error(
+ `ioredis cluster giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
+ );
+ return null;
+ }
+ const delay = Math.min(times * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
+ logger.info(`ioredis cluster reconnecting... attempt ${times}, delay ${delay}ms`);
+ return delay;
+ },
+ enableOfflineQueue: cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
+ });
ioredisClient.on('error', (err) => {
logger.error('ioredis client error:', err);
});
+ ioredisClient.on('connect', () => {
+ logger.info('ioredis client connected');
+ });
+
+ ioredisClient.on('ready', () => {
+ logger.info('ioredis client ready');
+ });
+
+ ioredisClient.on('reconnecting', (delay) => {
+ logger.info(`ioredis client reconnecting in ${delay}ms`);
+ });
+
+ ioredisClient.on('close', () => {
+ logger.warn('ioredis client connection closed');
+ });
+
/** Ping Interval to keep the Redis server connection alive (if enabled) */
let pingInterval = null;
const clearPingInterval = () => {
@@ -42,7 +101,9 @@ if (cacheConfig.USE_REDIS) {
if (cacheConfig.REDIS_PING_INTERVAL > 0) {
pingInterval = setInterval(() => {
if (ioredisClient && ioredisClient.status === 'ready') {
- ioredisClient.ping();
+ ioredisClient.ping().catch((err) => {
+ logger.error('ioredis ping failed:', err);
+ });
}
}, cacheConfig.REDIS_PING_INTERVAL * 1000);
ioredisClient.on('close', clearPingInterval);
@@ -56,8 +117,32 @@ if (cacheConfig.USE_REDIS) {
/**
* ** WARNING ** Keyv Redis client does not support Prefix like ioredis above.
* The prefix feature will be handled by the Keyv-Redis store in cacheFactory.js
+ * @type {import('@keyv/redis').RedisClientOptions | import('@keyv/redis').RedisClusterOptions}
*/
- const redisOptions = { username, password, socket: { tls: ca != null, ca } };
+ const redisOptions = {
+ username,
+ password,
+ socket: {
+ tls: ca != null,
+ ca,
+ connectTimeout: cacheConfig.REDIS_CONNECT_TIMEOUT,
+ reconnectStrategy: (retries) => {
+ if (
+ cacheConfig.REDIS_RETRY_MAX_ATTEMPTS > 0 &&
+ retries > cacheConfig.REDIS_RETRY_MAX_ATTEMPTS
+ ) {
+ logger.error(
+ `@keyv/redis client giving up after ${cacheConfig.REDIS_RETRY_MAX_ATTEMPTS} reconnection attempts`,
+ );
+ return new Error('Max reconnection attempts reached');
+ }
+ const delay = Math.min(retries * 100, cacheConfig.REDIS_RETRY_MAX_DELAY);
+ logger.info(`@keyv/redis reconnecting... attempt ${retries}, delay ${delay}ms`);
+ return delay;
+ },
+ },
+ disableOfflineQueue: !cacheConfig.REDIS_ENABLE_OFFLINE_QUEUE,
+ };
keyvRedisClient =
urls.length === 1
@@ -73,6 +158,27 @@ if (cacheConfig.USE_REDIS) {
logger.error('@keyv/redis client error:', err);
});
+ keyvRedisClient.on('connect', () => {
+ logger.info('@keyv/redis client connected');
+ });
+
+ keyvRedisClient.on('ready', () => {
+ logger.info('@keyv/redis client ready');
+ });
+
+ keyvRedisClient.on('reconnecting', () => {
+ logger.info('@keyv/redis client reconnecting...');
+ });
+
+ keyvRedisClient.on('disconnect', () => {
+ logger.warn('@keyv/redis client disconnected');
+ });
+
+ keyvRedisClient.connect().catch((err) => {
+ logger.error('@keyv/redis initial connection failed:', err);
+ throw err;
+ });
+
/** Ping Interval to keep the Redis server connection alive (if enabled) */
let pingInterval = null;
const clearPingInterval = () => {
@@ -85,7 +191,9 @@ if (cacheConfig.USE_REDIS) {
if (cacheConfig.REDIS_PING_INTERVAL > 0) {
pingInterval = setInterval(() => {
if (keyvRedisClient && keyvRedisClient.isReady) {
- keyvRedisClient.ping();
+ keyvRedisClient.ping().catch((err) => {
+ logger.error('@keyv/redis ping failed:', err);
+ });
}
}, cacheConfig.REDIS_PING_INTERVAL * 1000);
keyvRedisClient.on('disconnect', clearPingInterval);
diff --git a/redis-config/README.md b/redis-config/README.md
index 024d0b1683..607075569b 100644
--- a/redis-config/README.md
+++ b/redis-config/README.md
@@ -174,6 +174,15 @@ REDIS_KEY_PREFIX=librechat
# Connection limits
REDIS_MAX_LISTENERS=40
+
+# Ping interval to keep connection alive (seconds, 0 to disable)
+REDIS_PING_INTERVAL=0
+
+# Reconnection configuration
+REDIS_RETRY_MAX_DELAY=3000 # Max delay between reconnection attempts (ms)
+REDIS_RETRY_MAX_ATTEMPTS=10 # Max reconnection attempts (0 = infinite)
+REDIS_CONNECT_TIMEOUT=10000 # Connection timeout (ms)
+REDIS_ENABLE_OFFLINE_QUEUE=true # Queue commands when disconnected
```
## TLS/SSL Redis Setup