Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🩺 fix: Report Not Ready on /readyz While Redis Is Disconnected (#17232)
Marco Beretta41d53a2989ad
diff --git a/api/server/index.js b/api/server/index.jsindex 4acad6ff88..8f0aef132d 100644--- a/api/server/index.js+++ b/api/server/index.js@@ -54,6 +54,10 @@ const {getRemainingShutdownMs,registerBackgroundTaskShutdown,configureServerTimeouts,+ createReadinessHandler,+ keyvRedisClient,+ ioredisClient,+ cacheConfig,setupGracefulShutdown,updateInterfacePermissions,configureMessageFilterRegexValidator,@@ -350,12 +354,14 @@ const startServer = async () => {app.get('/health', (_req, res) => res.status(200).send('OK'));app.get('/livez', (_req, res) => res.status(200).send('OK'));- app.get('/readyz', (_req, res) => {- if (!serverReady) {- return res.status(503).send('NOT_READY');- }- return res.status(200).send('OK');- });+ app.get(+ '/readyz',+ createReadinessHandler(() => serverReady, {+ useRedis: cacheConfig.USE_REDIS,+ ioredis: ioredisClient,+ keyv: keyvRedisClient,+ }),+ );/* Middleware */app.use(requestContextMiddleware);diff --git a/e2e/specs/mock/scenarios/readiness-probes.spec.ts b/e2e/specs/mock/scenarios/readiness-probes.spec.tsnew file mode 100644index 0000000000..e2b2896923--- /dev/null+++ b/e2e/specs/mock/scenarios/readiness-probes.spec.ts@@ -0,0 +1,181 @@+import { resolve } from 'node:path';+import { expect, test } from '@playwright/test';+import { run, repoRoot, inOneProject } from './lint.helpers';++/**+ * `/readyz` tells an orchestrator whether to route traffic to a pod. With+ * USE_REDIS on, a pod whose Redis connection dropped after startup cannot sign+ * anyone in, so it must report not ready until the connection returns, while+ * `/livez` stays green so a Redis blip never triggers a liveness restart loop.+ *+ * The outage half runs in a subprocess against a throwaway RESP server it owns:+ * the lab Redis is shared by every worktree and must never be stopped.+ */++const OUTAGE_PROBE = `+const net = require('node:net');+const http = require('node:http');++function parseFrames(buffer) {+ const frames = [];+ let offset = 0;+ while (offset < buffer.length && buffer[offset] === 0x2a) {+ const headerEnd = buffer.indexOf('\\r\\n', offset);+ if (headerEnd === -1) break;+ const count = Number(buffer.subarray(offset + 1, headerEnd).toString());+ let cursor = headerEnd + 2;+ const args = [];+ for (let i = 0; i < count; i++) {+ const lengthEnd = buffer.indexOf('\\r\\n', cursor);+ if (lengthEnd === -1) return { frames, rest: buffer.subarray(offset) };+ const length = Number(buffer.subarray(cursor + 1, lengthEnd).toString());+ const end = lengthEnd + 2 + length;+ if (buffer.length < end + 2) return { frames, rest: buffer.subarray(offset) };+ args.push(buffer.subarray(lengthEnd + 2, end).toString());+ cursor = end + 2;+ }+ frames.push(args);+ offset = cursor;+ }+ return { frames, rest: buffer.subarray(offset) };+}++function reply([command = '']) {+ switch (command.toUpperCase()) {+ case 'PING':+ return '+PONG\\r\\n';+ case 'INFO': {+ const info = '# Server\\r\\nredis_version:7.2.4\\r\\nloading:0\\r\\n';+ return '$' + Buffer.byteLength(info) + '\\r\\n' + info + '\\r\\n';+ }+ default:+ return '+OK\\r\\n';+ }+}++function startRedis(port) {+ const sockets = new Set();+ const server = net.createServer((socket) => {+ sockets.add(socket);+ let pending = Buffer.alloc(0);+ socket.on('data', (chunk) => {+ const { frames, rest } = parseFrames(Buffer.concat([pending, chunk]));+ pending = rest;+ frames.forEach((frame) => socket.write(reply(frame)));+ });+ socket.on('close', () => sockets.delete(socket));+ socket.on('error', () => undefined);+ });+ return new Promise((done) =>+ server.listen(port, '127.0.0.1', () =>+ done({+ port: server.address().port,+ stop: () =>+ new Promise((closed) => {+ sockets.forEach((socket) => socket.destroy());+ server.close(() => closed());+ }),+ }),+ ),+ );+}++const get = (port, path) =>+ new Promise((done, fail) =>+ http.get({ host: '127.0.0.1', port, path }, (res) => {+ let body = '';+ res.on('data', (chunk) => (body += chunk));+ res.on('end', () => done(res.statusCode + ' ' + body));+ }).on('error', fail),+ );++async function until(check, expected, timeoutMs = 15000) {+ const deadline = Date.now() + timeoutMs;+ let last;+ while (Date.now() < deadline) {+ last = await check();+ if (last === expected) return last;+ await new Promise((wait) => setTimeout(wait, 50));+ }+ return last;+}++(async () => {+ let redis = await startRedis(0);+ process.env.USE_REDIS = 'true';+ process.env.REDIS_URI = 'redis://127.0.0.1:' + redis.port;+ process.env.REDIS_PING_INTERVAL = '0';+ process.env.REDIS_RETRY_MAX_DELAY = '100';++ const express = require('express');+ const { logger } = require('@librechat/data-schemas');+ logger.silent = true;+ const {+ cacheConfig,+ ioredisClient,+ keyvRedisClient,+ createReadinessHandler,+ } = require('@librechat/api');++ const app = express();+ app.get('/livez', (_req, res) => res.status(200).send('OK'));+ app.get(+ '/readyz',+ createReadinessHandler(() => true, {+ useRedis: cacheConfig.USE_REDIS,+ ioredis: ioredisClient,+ keyv: keyvRedisClient,+ }),+ );+ const listener = app.listen(0, '127.0.0.1');+ await new Promise((ready) => listener.once('listening', ready));+ const appPort = listener.address().port;+ const readyz = () => get(appPort, '/readyz');++ const observed = {};+ observed.connected = await until(readyz, '200 OK');+ await redis.stop();+ observed.down = await until(readyz, '503 NOT_READY');+ observed.liveWhileDown = await get(appPort, '/livez');+ redis = await startRedis(redis.port);+ observed.recovered = await until(readyz, '200 OK');++ process.stdout.write('RESULT' + JSON.stringify(observed));+ process.exit(0);+})().catch((error) => {+ process.stdout.write('ERROR' + (error && error.stack ? error.stack : String(error)));+ process.exit(1);+});+`;++test.describe('readiness probes', () => {+ test('a started pod reports ready and live while its dependencies are up @scenario:readyz-reports-ready-when-started-and-redis-connected', async ({+ request,+ }) => {+ const ready = await request.get('/readyz');+ expect(ready.status()).toBe(200);+ expect(await ready.text()).toBe('OK');++ const live = await request.get('/livez');+ expect(live.status()).toBe(200);+ });++ test('a pod whose Redis drops reports not ready but stays live, then recovers @scenario:readyz-reports-not-ready-while-redis-is-disconnected', () => {+ inOneProject();+ test.setTimeout(120_000);+ const result = run('node', ['-e', OUTAGE_PROBE], {+ cwd: resolve(repoRoot, 'packages/api'),+ });+ expect(result.status, result.output).toBe(0);++ const marker = result.stdout.lastIndexOf('RESULT');+ expect(marker, result.output).toBeGreaterThan(-1);+ const observed = JSON.parse(result.stdout.slice(marker + 'RESULT'.length));+ expect(observed).toEqual({+ connected: '200 OK',+ down: '503 NOT_READY',+ liveWhileDown: '200 OK',+ recovered: '200 OK',+ });+ });+});diff --git a/packages/api/src/app/index.ts b/packages/api/src/app/index.tsindex 99589970aa..03b9edb3de 100644--- a/packages/api/src/app/index.ts+++ b/packages/api/src/app/index.ts@@ -8,6 +8,7 @@ export * from './checks';export * from './resolve';export * from './shutdown';export * from './server';+export * from './readiness';export * from './origin';export * from './agents';export { resolveBuildInfo } from './build';diff --git a/packages/api/src/app/readiness.spec.ts b/packages/api/src/app/readiness.spec.tsnew file mode 100644index 0000000000..dbed6cc2d8--- /dev/null+++ b/packages/api/src/app/readiness.spec.ts@@ -0,0 +1,131 @@+import IoRedis from 'ioredis';+import { createClient } from '@keyv/redis';+import type { RedisClientType } from '@redis/client';+import type { Request, Response } from 'express';+import type { RespServer } from '~/cache/__tests__/resp.helper';+import type { RedisReadinessClients } from './readiness';+import { startRespServer, waitFor } from '~/cache/__tests__/resp.helper';+import { createReadinessHandler, isRedisReady } from './readiness';++const clients = (overrides: Partial<RedisReadinessClients> = {}): RedisReadinessClients => ({+ useRedis: true,+ ioredis: { status: 'ready' },+ keyv: { isReady: true },+ ...overrides,+});++describe('isRedisReady', () => {+ it('is ready when Redis is disabled, even without clients', () => {+ expect(isRedisReady({ useRedis: false, ioredis: null, keyv: null })).toBe(true);+ });++ it('is ready when both clients are connected', () => {+ expect(isRedisReady(clients())).toBe(true);+ });++ it.each(['connecting', 'reconnecting', 'close', 'end', 'wait'] as const)(+ 'is not ready when ioredis is %s',+ (status) => {+ expect(isRedisReady(clients({ ioredis: { status } }))).toBe(false);+ },+ );++ it('is not ready when the Keyv client is disconnected', () => {+ expect(isRedisReady(clients({ keyv: { isReady: false } }))).toBe(false);+ });++ it('is not ready when a configured client is missing', () => {+ expect(isRedisReady(clients({ ioredis: null }))).toBe(false);+ expect(isRedisReady(clients({ keyv: null }))).toBe(false);+ });++ it('requires every master of a Keyv cluster to be ready', () => {+ const ready = { isReady: true };+ const down = { isReady: false };+ const cluster = (isOpen: boolean, masters: { client?: { isReady: boolean } }[]) =>+ isRedisReady(clients({ keyv: { isOpen, masters } }));++ expect(cluster(true, [{ client: ready }, { client: ready }])).toBe(true);+ expect(cluster(true, [{ client: ready }, { client: down }])).toBe(false);+ expect(cluster(true, [{ client: ready }, {}])).toBe(false);+ expect(cluster(true, [])).toBe(false);+ expect(cluster(false, [{ client: ready }])).toBe(false);+ });+});++describe('isRedisReady with live clients', () => {+ let server: RespServer;+ let ioredis: IoRedis;+ let keyv: RedisClientType;++ afterEach(async () => {+ ioredis?.disconnect();+ if (keyv?.isOpen) {+ keyv.destroy();+ }+ await server?.close();+ });++ it('turns not ready when Redis goes away and ready again once it returns', async () => {+ server = await startRespServer();+ const { port, url } = server;+ ioredis = new IoRedis(url, { retryStrategy: () => 20, maxRetriesPerRequest: 0 });+ ioredis.on('error', () => undefined);+ keyv = createClient({ url, socket: { reconnectStrategy: () => 20 } }) as RedisClientType;+ keyv.on('error', () => undefined);+ await keyv.connect();+ await waitFor(() => ioredis.status === 'ready');++ const live = (): RedisReadinessClients => ({ useRedis: true, ioredis, keyv });+ expect(isRedisReady(live())).toBe(true);++ await server.close();+ await waitFor(() => !isRedisReady(live()));+ expect(ioredis.status).not.toBe('ready');++ server = await startRespServer(port);+ await waitFor(() => isRedisReady(live()), 5000);+ });+});++describe('createReadinessHandler', () => {+ const respond = (handler: ReturnType<typeof createReadinessHandler>) => {+ const res = { status: jest.fn().mockReturnThis(), send: jest.fn() };+ handler({} as Request, res as unknown as Response);+ return { code: res.status.mock.calls[0][0], body: res.send.mock.calls[0][0] };+ };+ const probe = (serverReady: boolean, redis: RedisReadinessClients) =>+ respond(createReadinessHandler(() => serverReady, redis));++ it('returns 503 before startup completes', () => {+ expect(probe(false, clients())).toEqual({ code: 503, body: 'NOT_READY' });+ });++ it('returns 503 after startup when Redis drops', () => {+ expect(probe(true, clients({ ioredis: { status: 'reconnecting' } }))).toEqual({+ code: 503,+ body: 'NOT_READY',+ });+ });++ it('returns 200 when started and Redis is connected', () => {+ expect(probe(true, clients())).toEqual({ code: 200, body: 'OK' });+ });++ it('returns 200 when started with Redis disabled', () => {+ expect(probe(true, { useRedis: false, ioredis: null, keyv: null })).toEqual({+ code: 200,+ body: 'OK',+ });+ });++ it('re-evaluates readiness on every request', () => {+ const keyv = { isReady: true };+ const handler = createReadinessHandler(() => true, clients({ keyv }));+ expect(respond(handler).code).toBe(200);+ keyv.isReady = false;+ expect(respond(handler).code).toBe(503);+ keyv.isReady = true;+ expect(respond(handler).code).toBe(200);+ });+});diff --git a/packages/api/src/app/readiness.ts b/packages/api/src/app/readiness.tsnew file mode 100644index 0000000000..29a0ed4f5a--- /dev/null+++ b/packages/api/src/app/readiness.ts@@ -0,0 +1,65 @@+import type { RedisClientType } from '@redis/client';+import type { Request, Response } from 'express';+import type { Redis, Cluster } from 'ioredis';++type NodeRedisReadiness = Pick<RedisClientType, 'isReady'>;++/** The subset of a Node Redis cluster that reports per-master connectivity. */+type NodeRedisClusterReadiness = {+ isOpen: boolean;+ masters: ReadonlyArray<{ client?: NodeRedisReadiness }>;+};++export type RedisReadinessClients = {+ useRedis: boolean;+ ioredis: Pick<Redis | Cluster, 'status'> | null;+ keyv: NodeRedisReadiness | NodeRedisClusterReadiness | null;+};++/**+ * A Node Redis cluster stays `isOpen` while an individual master reconnects, and every+ * key in that master's slots fails meanwhile. Masters connect eagerly (the client does+ * not set `minimizeConnections`), so each one must have a ready client.+ */+function isKeyvReady(keyv: NodeRedisReadiness | NodeRedisClusterReadiness): boolean {+ if ('isReady' in keyv) {+ return keyv.isReady;+ }+ return (+ keyv.isOpen &&+ keyv.masters.length > 0 &&+ keyv.masters.every((master) => master.client?.isReady === true)+ );+}++/**+ * Whether the shared Redis clients can serve commands. Always true when Redis is disabled.+ * A configured client that is missing, reconnecting, closed or ended counts as not ready,+ * because requests that touch the cache (sessions, OAuth state, rate limits) fail until it+ * recovers.+ */+export function isRedisReady({ useRedis, ioredis, keyv }: RedisReadinessClients): boolean {+ if (!useRedis) {+ return true;+ }+ if (ioredis?.status !== 'ready' || keyv == null) {+ return false;+ }+ return isKeyvReady(keyv);+}++/**+ * `/readyz` handler: 503 until startup completes, and again whenever Redis is configured+ * but disconnected, so readiness-gated orchestration stops routing to a pod that cannot+ * serve authenticated traffic. `/livez` stays unconditional so a Redis blip does not+ * trigger a liveness restart loop.+ */+export const createReadinessHandler =+ (isServerReady: () => boolean, redis: RedisReadinessClients) =>+ (_req: Request, res: Response): void => {+ if (!isServerReady() || !isRedisReady(redis)) {+ res.status(503).send('NOT_READY');+ return;+ }+ res.status(200).send('OK');+ };