Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🩺 fix: Report Not Ready on /readyz While Redis Is Disconnected (#17232)

Marco Beretta41d53a2989ad
diff --git a/api/server/index.js b/api/server/index.js
index 4acad6ff88..8f0aef132d 100644
--- a/api/server/index.js
+++ b/api/server/index.js
@@ -54,6 +54,10 @@ const {
getRemainingShutdownMs,
registerBackgroundTaskShutdown,
configureServerTimeouts,
+ createReadinessHandler,
+ keyvRedisClient,
+ ioredisClient,
+ cacheConfig,
setupGracefulShutdown,
updateInterfacePermissions,
configureMessageFilterRegexValidator,
@@ -350,12 +354,14 @@ const startServer = async () => {
app.get('/health', (_req, res) => res.status(200).send('OK'));
app.get('/livez', (_req, res) => res.status(200).send('OK'));
- app.get('/readyz', (_req, res) => {
- if (!serverReady) {
- return res.status(503).send('NOT_READY');
- }
- return res.status(200).send('OK');
- });
+ app.get(
+ '/readyz',
+ createReadinessHandler(() => serverReady, {
+ useRedis: cacheConfig.USE_REDIS,
+ ioredis: ioredisClient,
+ keyv: keyvRedisClient,
+ }),
+ );
/* Middleware */
app.use(requestContextMiddleware);
diff --git a/e2e/specs/mock/scenarios/readiness-probes.spec.ts b/e2e/specs/mock/scenarios/readiness-probes.spec.ts
new file mode 100644
index 0000000000..e2b2896923
--- /dev/null
+++ b/e2e/specs/mock/scenarios/readiness-probes.spec.ts
@@ -0,0 +1,181 @@
+import { resolve } from 'node:path';
+import { expect, test } from '@playwright/test';
+import { run, repoRoot, inOneProject } from './lint.helpers';
+
+/**
+ * `/readyz` tells an orchestrator whether to route traffic to a pod. With
+ * USE_REDIS on, a pod whose Redis connection dropped after startup cannot sign
+ * anyone in, so it must report not ready until the connection returns, while
+ * `/livez` stays green so a Redis blip never triggers a liveness restart loop.
+ *
+ * The outage half runs in a subprocess against a throwaway RESP server it owns:
+ * the lab Redis is shared by every worktree and must never be stopped.
+ */
+
+const OUTAGE_PROBE = `
+const net = require('node:net');
+const http = require('node:http');
+
+function parseFrames(buffer) {
+ const frames = [];
+ let offset = 0;
+ while (offset < buffer.length && buffer[offset] === 0x2a) {
+ const headerEnd = buffer.indexOf('\\r\\n', offset);
+ if (headerEnd === -1) break;
+ const count = Number(buffer.subarray(offset + 1, headerEnd).toString());
+ let cursor = headerEnd + 2;
+ const args = [];
+ for (let i = 0; i < count; i++) {
+ const lengthEnd = buffer.indexOf('\\r\\n', cursor);
+ if (lengthEnd === -1) return { frames, rest: buffer.subarray(offset) };
+ const length = Number(buffer.subarray(cursor + 1, lengthEnd).toString());
+ const end = lengthEnd + 2 + length;
+ if (buffer.length < end + 2) return { frames, rest: buffer.subarray(offset) };
+ args.push(buffer.subarray(lengthEnd + 2, end).toString());
+ cursor = end + 2;
+ }
+ frames.push(args);
+ offset = cursor;
+ }
+ return { frames, rest: buffer.subarray(offset) };
+}
+
+function reply([command = '']) {
+ switch (command.toUpperCase()) {
+ case 'PING':
+ return '+PONG\\r\\n';
+ case 'INFO': {
+ const info = '# Server\\r\\nredis_version:7.2.4\\r\\nloading:0\\r\\n';
+ return '$' + Buffer.byteLength(info) + '\\r\\n' + info + '\\r\\n';
+ }
+ default:
+ return '+OK\\r\\n';
+ }
+}
+
+function startRedis(port) {
+ const sockets = new Set();
+ const server = net.createServer((socket) => {
+ sockets.add(socket);
+ let pending = Buffer.alloc(0);
+ socket.on('data', (chunk) => {
+ const { frames, rest } = parseFrames(Buffer.concat([pending, chunk]));
+ pending = rest;
+ frames.forEach((frame) => socket.write(reply(frame)));
+ });
+ socket.on('close', () => sockets.delete(socket));
+ socket.on('error', () => undefined);
+ });
+ return new Promise((done) =>
+ server.listen(port, '127.0.0.1', () =>
+ done({
+ port: server.address().port,
+ stop: () =>
+ new Promise((closed) => {
+ sockets.forEach((socket) => socket.destroy());
+ server.close(() => closed());
+ }),
+ }),
+ ),
+ );
+}
+
+const get = (port, path) =>
+ new Promise((done, fail) =>
+ http.get({ host: '127.0.0.1', port, path }, (res) => {
+ let body = '';
+ res.on('data', (chunk) => (body += chunk));
+ res.on('end', () => done(res.statusCode + ' ' + body));
+ }).on('error', fail),
+ );
+
+async function until(check, expected, timeoutMs = 15000) {
+ const deadline = Date.now() + timeoutMs;
+ let last;
+ while (Date.now() < deadline) {
+ last = await check();
+ if (last === expected) return last;
+ await new Promise((wait) => setTimeout(wait, 50));
+ }
+ return last;
+}
+
+(async () => {
+ let redis = await startRedis(0);
+ process.env.USE_REDIS = 'true';
+ process.env.REDIS_URI = 'redis://127.0.0.1:' + redis.port;
+ process.env.REDIS_PING_INTERVAL = '0';
+ process.env.REDIS_RETRY_MAX_DELAY = '100';
+
+ const express = require('express');
+ const { logger } = require('@librechat/data-schemas');
+ logger.silent = true;
+ const {
+ cacheConfig,
+ ioredisClient,
+ keyvRedisClient,
+ createReadinessHandler,
+ } = require('@librechat/api');
+
+ const app = express();
+ app.get('/livez', (_req, res) => res.status(200).send('OK'));
+ app.get(
+ '/readyz',
+ createReadinessHandler(() => true, {
+ useRedis: cacheConfig.USE_REDIS,
+ ioredis: ioredisClient,
+ keyv: keyvRedisClient,
+ }),
+ );
+ const listener = app.listen(0, '127.0.0.1');
+ await new Promise((ready) => listener.once('listening', ready));
+ const appPort = listener.address().port;
+ const readyz = () => get(appPort, '/readyz');
+
+ const observed = {};
+ observed.connected = await until(readyz, '200 OK');
+ await redis.stop();
+ observed.down = await until(readyz, '503 NOT_READY');
+ observed.liveWhileDown = await get(appPort, '/livez');
+ redis = await startRedis(redis.port);
+ observed.recovered = await until(readyz, '200 OK');
+
+ process.stdout.write('RESULT' + JSON.stringify(observed));
+ process.exit(0);
+})().catch((error) => {
+ process.stdout.write('ERROR' + (error && error.stack ? error.stack : String(error)));
+ process.exit(1);
+});
+`;
+
+test.describe('readiness probes', () => {
+ test('a started pod reports ready and live while its dependencies are up @scenario:readyz-reports-ready-when-started-and-redis-connected', async ({
+ request,
+ }) => {
+ const ready = await request.get('/readyz');
+ expect(ready.status()).toBe(200);
+ expect(await ready.text()).toBe('OK');
+
+ const live = await request.get('/livez');
+ expect(live.status()).toBe(200);
+ });
+
+ test('a pod whose Redis drops reports not ready but stays live, then recovers @scenario:readyz-reports-not-ready-while-redis-is-disconnected', () => {
+ inOneProject();
+ test.setTimeout(120_000);
+ const result = run('node', ['-e', OUTAGE_PROBE], {
+ cwd: resolve(repoRoot, 'packages/api'),
+ });
+ expect(result.status, result.output).toBe(0);
+
+ const marker = result.stdout.lastIndexOf('RESULT');
+ expect(marker, result.output).toBeGreaterThan(-1);
+ const observed = JSON.parse(result.stdout.slice(marker + 'RESULT'.length));
+ expect(observed).toEqual({
+ connected: '200 OK',
+ down: '503 NOT_READY',
+ liveWhileDown: '200 OK',
+ recovered: '200 OK',
+ });
+ });
+});
diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.ts
index 99589970aa..03b9edb3de 100644
--- a/packages/api/src/app/index.ts
+++ b/packages/api/src/app/index.ts
@@ -8,6 +8,7 @@ export * from './checks';
export * from './resolve';
export * from './shutdown';
export * from './server';
+export * from './readiness';
export * from './origin';
export * from './agents';
export { resolveBuildInfo } from './build';
diff --git a/packages/api/src/app/readiness.spec.ts b/packages/api/src/app/readiness.spec.ts
new file mode 100644
index 0000000000..dbed6cc2d8
--- /dev/null
+++ b/packages/api/src/app/readiness.spec.ts
@@ -0,0 +1,131 @@
+import IoRedis from 'ioredis';
+import { createClient } from '@keyv/redis';
+import type { RedisClientType } from '@redis/client';
+import type { Request, Response } from 'express';
+import type { RespServer } from '~/cache/__tests__/resp.helper';
+import type { RedisReadinessClients } from './readiness';
+import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';
+import { createReadinessHandler, isRedisReady } from './readiness';
+
+const clients = (overrides: Partial<RedisReadinessClients> = {}): RedisReadinessClients => ({
+ useRedis: true,
+ ioredis: { status: 'ready' },
+ keyv: { isReady: true },
+ ...overrides,
+});
+
+describe('isRedisReady', () => {
+ it('is ready when Redis is disabled, even without clients', () => {
+ expect(isRedisReady({ useRedis: false, ioredis: null, keyv: null })).toBe(true);
+ });
+
+ it('is ready when both clients are connected', () => {
+ expect(isRedisReady(clients())).toBe(true);
+ });
+
+ it.each(['connecting', 'reconnecting', 'close', 'end', 'wait'] as const)(
+ 'is not ready when ioredis is %s',
+ (status) => {
+ expect(isRedisReady(clients({ ioredis: { status } }))).toBe(false);
+ },
+ );
+
+ it('is not ready when the Keyv client is disconnected', () => {
+ expect(isRedisReady(clients({ keyv: { isReady: false } }))).toBe(false);
+ });
+
+ it('is not ready when a configured client is missing', () => {
+ expect(isRedisReady(clients({ ioredis: null }))).toBe(false);
+ expect(isRedisReady(clients({ keyv: null }))).toBe(false);
+ });
+
+ it('requires every master of a Keyv cluster to be ready', () => {
+ const ready = { isReady: true };
+ const down = { isReady: false };
+ const cluster = (isOpen: boolean, masters: { client?: { isReady: boolean } }[]) =>
+ isRedisReady(clients({ keyv: { isOpen, masters } }));
+
+ expect(cluster(true, [{ client: ready }, { client: ready }])).toBe(true);
+ expect(cluster(true, [{ client: ready }, { client: down }])).toBe(false);
+ expect(cluster(true, [{ client: ready }, {}])).toBe(false);
+ expect(cluster(true, [])).toBe(false);
+ expect(cluster(false, [{ client: ready }])).toBe(false);
+ });
+});
+
+describe('isRedisReady with live clients', () => {
+ let server: RespServer;
+ let ioredis: IoRedis;
+ let keyv: RedisClientType;
+
+ afterEach(async () => {
+ ioredis?.disconnect();
+ if (keyv?.isOpen) {
+ keyv.destroy();
+ }
+ await server?.close();
+ });
+
+ it('turns not ready when Redis goes away and ready again once it returns', async () => {
+ server = await startRespServer();
+ const { port, url } = server;
+ ioredis = new IoRedis(url, { retryStrategy: () => 20, maxRetriesPerRequest: 0 });
+ ioredis.on('error', () => undefined);
+ keyv = createClient({ url, socket: { reconnectStrategy: () => 20 } }) as RedisClientType;
+ keyv.on('error', () => undefined);
+ await keyv.connect();
+ await waitFor(() => ioredis.status === 'ready');
+
+ const live = (): RedisReadinessClients => ({ useRedis: true, ioredis, keyv });
+ expect(isRedisReady(live())).toBe(true);
+
+ await server.close();
+ await waitFor(() => !isRedisReady(live()));
+ expect(ioredis.status).not.toBe('ready');
+
+ server = await startRespServer(port);
+ await waitFor(() => isRedisReady(live()), 5000);
+ });
+});
+
+describe('createReadinessHandler', () => {
+ const respond = (handler: ReturnType<typeof createReadinessHandler>) => {
+ const res = { status: jest.fn().mockReturnThis(), send: jest.fn() };
+ handler({} as Request, res as unknown as Response);
+ return { code: res.status.mock.calls[0][0], body: res.send.mock.calls[0][0] };
+ };
+ const probe = (serverReady: boolean, redis: RedisReadinessClients) =>
+ respond(createReadinessHandler(() => serverReady, redis));
+
+ it('returns 503 before startup completes', () => {
+ expect(probe(false, clients())).toEqual({ code: 503, body: 'NOT_READY' });
+ });
+
+ it('returns 503 after startup when Redis drops', () => {
+ expect(probe(true, clients({ ioredis: { status: 'reconnecting' } }))).toEqual({
+ code: 503,
+ body: 'NOT_READY',
+ });
+ });
+
+ it('returns 200 when started and Redis is connected', () => {
+ expect(probe(true, clients())).toEqual({ code: 200, body: 'OK' });
+ });
+
+ it('returns 200 when started with Redis disabled', () => {
+ expect(probe(true, { useRedis: false, ioredis: null, keyv: null })).toEqual({
+ code: 200,
+ body: 'OK',
+ });
+ });
+
+ it('re-evaluates readiness on every request', () => {
+ const keyv = { isReady: true };
+ const handler = createReadinessHandler(() => true, clients({ keyv }));
+ expect(respond(handler).code).toBe(200);
+ keyv.isReady = false;
+ expect(respond(handler).code).toBe(503);
+ keyv.isReady = true;
+ expect(respond(handler).code).toBe(200);
+ });
+});
diff --git a/packages/api/src/app/readiness.ts b/packages/api/src/app/readiness.ts
new file mode 100644
index 0000000000..29a0ed4f5a
--- /dev/null
+++ b/packages/api/src/app/readiness.ts
@@ -0,0 +1,65 @@
+import type { RedisClientType } from '@redis/client';
+import type { Request, Response } from 'express';
+import type { Redis, Cluster } from 'ioredis';
+
+type NodeRedisReadiness = Pick<RedisClientType, 'isReady'>;
+
+/** The subset of a Node Redis cluster that reports per-master connectivity. */
+type NodeRedisClusterReadiness = {
+ isOpen: boolean;
+ masters: ReadonlyArray<{ client?: NodeRedisReadiness }>;
+};
+
+export type RedisReadinessClients = {
+ useRedis: boolean;
+ ioredis: Pick<Redis | Cluster, 'status'> | null;
+ keyv: NodeRedisReadiness | NodeRedisClusterReadiness | null;
+};
+
+/**
+ * A Node Redis cluster stays `isOpen` while an individual master reconnects, and every
+ * key in that master's slots fails meanwhile. Masters connect eagerly (the client does
+ * not set `minimizeConnections`), so each one must have a ready client.
+ */
+function isKeyvReady(keyv: NodeRedisReadiness | NodeRedisClusterReadiness): boolean {
+ if ('isReady' in keyv) {
+ return keyv.isReady;
+ }
+ return (
+ keyv.isOpen &&
+ keyv.masters.length > 0 &&
+ keyv.masters.every((master) => master.client?.isReady === true)
+ );
+}
+
+/**
+ * Whether the shared Redis clients can serve commands. Always true when Redis is disabled.
+ * A configured client that is missing, reconnecting, closed or ended counts as not ready,
+ * because requests that touch the cache (sessions, OAuth state, rate limits) fail until it
+ * recovers.
+ */
+export function isRedisReady({ useRedis, ioredis, keyv }: RedisReadinessClients): boolean {
+ if (!useRedis) {
+ return true;
+ }
+ if (ioredis?.status !== 'ready' || keyv == null) {
+ return false;
+ }
+ return isKeyvReady(keyv);
+}
+
+/**
+ * `/readyz` handler: 503 until startup completes, and again whenever Redis is configured
+ * but disconnected, so readiness-gated orchestration stops routing to a pod that cannot
+ * serve authenticated traffic. `/livez` stays unconditional so a Redis blip does not
+ * trigger a liveness restart loop.
+ */
+export const createReadinessHandler =
+ (isServerReady: () => boolean, redis: RedisReadinessClients) =>
+ (_req: Request, res: Response): void => {
+ if (!isServerReady() || !isRedisReady(redis)) {
+ res.status(503).send('NOT_READY');
+ return;
+ }
+ res.status(200).send('OK');
+ };