Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(server): add /livez, /readyz, /v1/providers + ?ready=true for meta-gateway interop (#594)

GAURANG PATEL43a73579ece5
diff --git a/server/src/__tests__/routes/status.test.ts b/server/src/__tests__/routes/status.test.ts
new file mode 100644
index 00000000..bdc97c0a
--- /dev/null
+++ b/server/src/__tests__/routes/status.test.ts
@@ -0,0 +1,106 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+
+async function request(
+ app: Express,
+ method: string,
+ path: string,
+ headers: Record<string, string> = {},
+) {
+ const server = app.listen(0);
+ const addr = server.address() as { port: number };
+ const url = `http://127.0.0.1:${addr.port}${path}`;
+ const res = await fetch(url, { method, headers });
+ const data = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body: data };
+}
+
+function insertHealthyKey(platform: string) {
+ const secret = encrypt(`${platform}-test-key`);
+ getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)
+ `).run(platform, secret.encrypted, secret.iv, secret.authTag);
+}
+
+// These tests share one in-memory DB and run top-to-bottom: the "no upstreams"
+// assertions must precede the key insert.
+describe('status & interop endpoints (#433)', () => {
+ let app: Express;
+ let unifiedKey = '';
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ unifiedKey = getUnifiedApiKey();
+ });
+
+ it('GET /livez returns 200 with status, version, and uptime', async () => {
+ const { status, body } = await request(app, 'GET', '/livez');
+ expect(status).toBe(200);
+ expect(body.status).toBe('ok');
+ expect(typeof body.version).toBe('string');
+ expect(typeof body.uptime_s).toBe('number');
+ });
+
+ it('GET /readyz returns 503 no_upstreams_configured when nothing is set up', async () => {
+ const { status, body } = await request(app, 'GET', '/readyz');
+ expect(status).toBe(503);
+ expect(body.reason).toBe('no_upstreams_configured');
+ });
+
+ it('GET /v1/providers rejects a missing/invalid key with 401', async () => {
+ const { status, body } = await request(app, 'GET', '/v1/providers');
+ expect(status).toBe(401);
+ expect(body.error.type).toBe('authentication_error');
+ });
+
+ it('GET /v1/providers returns empty providers and zero counts before any key', async () => {
+ const { status, body } = await request(app, 'GET', '/v1/providers', {
+ Authorization: `Bearer ${unifiedKey}`,
+ });
+ expect(status).toBe(200);
+ expect(body.providers).toEqual([]);
+ expect(body.counts).toEqual({ healthy: 0, rate_limited: 0, invalid: 0, unknown: 0 });
+ });
+
+ it('GET /readyz returns 200 once a healthy upstream exists', async () => {
+ insertHealthyKey('groq');
+ const { status, body } = await request(app, 'GET', '/readyz');
+ expect(status).toBe(200);
+ expect(body.status).toBe('ok');
+ expect(body.ready_upstreams).toBeGreaterThanOrEqual(1);
+ });
+
+ it('GET /v1/providers reports the healthy upstream', async () => {
+ const { status, body } = await request(app, 'GET', '/v1/providers', {
+ 'x-api-key': unifiedKey,
+ });
+ expect(status).toBe(200);
+ const groq = body.providers.find((p: { platform: string }) => p.platform === 'groq');
+ expect(groq).toBeDefined();
+ expect(groq.status).toBe('healthy');
+ expect(groq.keys).toBe(1);
+ expect(body.counts.healthy).toBeGreaterThanOrEqual(1);
+ });
+
+ it('GET /v1/models?ready=true filters to servable models (auth required)', async () => {
+ const unauth = await request(app, 'GET', '/v1/models?ready=true');
+ expect(unauth.status).toBe(401);
+
+ const full = await request(app, 'GET', '/v1/models', { 'x-api-key': unifiedKey });
+ const ready = await request(app, 'GET', '/v1/models?ready=true', { 'x-api-key': unifiedKey });
+ expect(ready.status).toBe(200);
+ expect(ready.body.object).toBe('list');
+ // Every non-virtual model in the filtered list is actually available, and the
+ // filter never returns more than the unfiltered catalog.
+ const realModels = ready.body.data.filter((m: { id: string }) => m.id !== 'auto' && m.id !== 'fusion');
+ for (const m of realModels) expect(m.available).toBe(true);
+ expect(ready.body.data.length).toBeLessThanOrEqual(full.body.data.length);
+ });
+});
diff --git a/server/src/app.ts b/server/src/app.ts
index a93e4e2e..c48cbaf3 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -22,6 +22,7 @@ import { compressionRouter } from './routes/compression.js';
import { authRouter } from './routes/auth.js';
import { docsRouter } from './routes/docs.js';
import { mcpRouter } from './routes/mcp.js';
+import { statusRouter, providersRouter } from './routes/status.js';
import { geminiRouter } from './routes/gemini.js';
import { ollamaRouter } from './routes/ollama.js';
import { urlTokenRouter } from './routes/url-tokens.js';
@@ -120,6 +121,12 @@ export function createApp(config?: Config) {
// owns those two paths; everything else falls through to the routers below.
app.use('/v1', docsRouter);
+ // Read-only per-upstream status for meta-gateways (GET /v1/providers, #433).
+ // Unified-key auth is enforced inside the handler; mounted before the rate
+ // limiter (like docsRouter) so status polling doesn't draw down a caller's
+ // request budget.
+ app.use('/v1', providersRouter);
+
// Separately revocable URL tokens for clients that cannot set headers.
app.use('/v1/t/:token', createProxyRateLimiter(cfg.proxyRateLimitRpm));
app.use('/v1/t/:token', urlTokenRouter);
@@ -155,6 +162,12 @@ export function createApp(config?: Config) {
res.json({ status: 'ok', timestamp: new Date().toISOString() });
});
+ // Liveness / readiness probes for orchestrators (GET /livez, /readyz, #433).
+ // Unauthenticated so a load balancer can probe them; registered before the
+ // static / SPA-fallback block below so these root paths resolve to JSON
+ // instead of index.html.
+ app.use(statusRouter);
+
// Error handler (for API routes)
app.use(errorHandler);
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 1eeb3bf7..983eb181 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -208,9 +208,11 @@ proxyRouter.get('/models', (req: Request, res: Response) => {
// By default we return the WHOLE catalog (one row per model id), each tagged
// with whether it is currently usable, so a client can see everything and know
- // what's connected vs. disabled/keyless (#242). `?available=true` (alias
- // `?connected=true`) narrows the list to only models that can serve a request
- // right now — the previous default behavior. `available` is computed as
+ // what's connected vs. disabled/keyless (#242). `?available=true` (aliases
+ // `?connected=true`, `?ready=true`) narrows the list to only models that can
+ // serve a request right now — the previous default behavior. The `ready`
+ // alias is the machine-readable filter a meta-gateway uses (#433) to ask
+ // "which models can this instance actually serve now". `available` is computed as
// "enabled AND an enabled key can serve it"; dedup prefers an available
// instance of a model id over a disabled/keyless one.
// Shared catalog listing (one source of truth for the OpenAI and Anthropic
@@ -221,7 +223,7 @@ proxyRouter.get('/models', (req: Request, res: Response) => {
// conservative default and truncate long inputs before they reach us (#282).
const { models: allListed, autoContextWindow } = buildModelListing();
- const q = String(req.query.available ?? req.query.connected ?? '').toLowerCase();
+ const q = String(req.query.available ?? req.query.connected ?? req.query.ready ?? '').toLowerCase();
const onlyAvailable = q === '1' || q === 'true' || q === 'yes';
const listed = onlyAvailable ? allListed.filter(m => m.available === 1) : allListed;
diff --git a/server/src/routes/status.ts b/server/src/routes/status.ts
new file mode 100644
index 00000000..4c216585
--- /dev/null
+++ b/server/src/routes/status.ts
@@ -0,0 +1,208 @@
+import { Router } from 'express';
+import type { Request, Response } from 'express';
+import type { Platform } from '@freellmapi/shared/types.js';
+import { getDb, getUnifiedApiKey } from '../db/index.js';
+import { isEncryptionKeyInitialized } from '../lib/crypto.js';
+import { extractApiToken, timingSafeStringEqual } from './proxy.js';
+import { getProvider } from '../providers/index.js';
+import { getSoonestCooldownExpiry } from '../services/ratelimit.js';
+import { getQuotaStateForKeys } from '../services/provider-quota.js';
+import { openapiSpec } from '../docs/openapi.js';
+
+// Machine-readable operational endpoints for meta-gateways / orchestrators that
+// run this proxy as one upstream lane (LibreChat, LiteLLM Proxy, OmniRoute — see
+// #433). `/livez` and `/readyz` are unauthenticated so a load balancer can probe
+// them; `/v1/providers` sits behind the same unified API key as the rest of /v1.
+
+export const statusRouter = Router();
+export const providersRouter = Router();
+
+// Cheap "is the SQLite connection answering" probe. getDb() throws before initDb
+// has run, and prepare/get throws if the handle is closed — either way the
+// server is not serviceable, which is exactly the /livez and /readyz 503 signal.
+function isDbReachable(): boolean {
+ try {
+ getDb().prepare('SELECT 1').get();
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+// GET /livez — liveness. 200 when the process can serve at all: the DB answers
+// and the encryption key is loaded (without it, stored provider keys can't be
+// decrypted, so no request can be routed). 503 otherwise.
+statusRouter.get('/livez', (_req: Request, res: Response) => {
+ const db = isDbReachable();
+ const encryptionKey = isEncryptionKeyInitialized();
+ const uptime_s = Math.floor(process.uptime());
+ const version = openapiSpec.info.version;
+
+ if (!db || !encryptionKey) {
+ res.status(503).json({
+ status: 'unavailable',
+ version,
+ uptime_s,
+ checks: { db, encryption_key: encryptionKey },
+ });
+ return;
+ }
+
+ res.json({ status: 'ok', version, uptime_s });
+});
+
+// GET /readyz — readiness. 200 when at least one upstream is serviceable right
+// now (an enabled key whose status is healthy or not-yet-probed). 503 with a
+// machine-readable reason otherwise, so a meta-gateway can skip this instance
+// instead of eating a failover on every call.
+statusRouter.get('/readyz', (_req: Request, res: Response) => {
+ if (!isDbReachable()) {
+ res.status(503).json({ status: 'unavailable', reason: 'db_unreachable' });
+ return;
+ }
+
+ const db = getDb();
+ const agg = db.prepare(`
+ SELECT
+ COUNT(*) AS enabled_keys,
+ COUNT(DISTINCT CASE WHEN status IN ('healthy', 'unknown') THEN platform END) AS ready_upstreams
+ FROM api_keys
+ WHERE enabled = 1
+ `).get() as { enabled_keys: number; ready_upstreams: number };
+
+ const readyUpstreams = agg.ready_upstreams ?? 0;
+ if (readyUpstreams > 0) {
+ res.json({ status: 'ok', ready_upstreams: readyUpstreams });
+ return;
+ }
+
+ // Distinguish "nothing configured" from "everything cooling down" from
+ // "everything unhealthy" so the caller can act on the specific reason.
+ let reason: string;
+ if ((agg.enabled_keys ?? 0) === 0) {
+ reason = 'no_upstreams_configured';
+ } else if (getSoonestCooldownExpiry() != null) {
+ reason = 'all_upstreams_rate_limited';
+ } else {
+ reason = 'all_upstreams_unhealthy';
+ }
+
+ res.status(503).json({ status: 'unavailable', reason });
+});
+
+interface PlatformKeyRow {
+ platform: string;
+ enabled_keys: number;
+ healthy_keys: number;
+ unknown_keys: number;
+ invalid_keys: number;
+ error_keys: number;
+}
+
+// GET /v1/providers — aggregate per-upstream state a smart router needs to
+// decide "route here or skip", with no key material and no PII. Behind the
+// unified API key (like the rest of /v1); mounted before the proxy rate limiter
+// so status polling doesn't draw down a caller's request budget.
+providersRouter.get('/providers', (req: Request, res: Response) => {
+ const token = extractApiToken(req);
+ const unifiedKey = getUnifiedApiKey();
+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {
+ res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });
+ return;
+ }
+
+ const db = getDb();
+
+ // Per-platform key-status rollup (only enabled keys count toward serviceability).
+ const platformRows = db.prepare(`
+ SELECT
+ platform,
+ SUM(CASE WHEN enabled = 1 THEN 1 ELSE 0 END) AS enabled_keys,
+ SUM(CASE WHEN enabled = 1 AND status = 'healthy' THEN 1 ELSE 0 END) AS healthy_keys,
+ SUM(CASE WHEN enabled = 1 AND status = 'unknown' THEN 1 ELSE 0 END) AS unknown_keys,
+ SUM(CASE WHEN enabled = 1 AND status = 'invalid' THEN 1 ELSE 0 END) AS invalid_keys,
+ SUM(CASE WHEN enabled = 1 AND status = 'error' THEN 1 ELSE 0 END) AS error_keys
+ FROM api_keys
+ GROUP BY platform
+ `).all() as PlatformKeyRow[];
+
+ // Active cooldowns (rate limiting) rolled up to the platform, with the soonest
+ // resume time. rate_limit_cooldowns is per (platform, model, key); the MIN is
+ // the earliest moment any of that platform's routes frees up.
+ const now = Date.now();
+ const cooldownRows = db.prepare(`
+ SELECT platform, MIN(expires_at_ms) AS resume_ms
+ FROM rate_limit_cooldowns
+ WHERE expires_at_ms > ?
+ GROUP BY platform
+ `).all(now) as Array<{ platform: string; resume_ms: number }>;
+ const resumeByPlatform = new Map(cooldownRows.map(r => [r.platform, r.resume_ms]));
+
+ // Most recent probe error per platform (enabled keys, newest first).
+ const errorRows = db.prepare(`
+ SELECT platform, last_health_error
+ FROM api_keys
+ WHERE enabled = 1 AND last_health_error IS NOT NULL
+ ORDER BY last_checked_at DESC
+ `).all() as Array<{ platform: string; last_health_error: string }>;
+ const lastErrorByPlatform = new Map<string, string>();
+ for (const row of errorRows) {
+ if (!lastErrorByPlatform.has(row.platform)) {
+ lastErrorByPlatform.set(row.platform, row.last_health_error);
+ }
+ }
+
+ // Best-effort request-quota headroom from observed quota state. Only surfaced
+ // when a provider actually reported a `requests` limit; absent otherwise.
+ const requestsRemainingByPlatform = new Map<string, number>();
+ for (const q of getQuotaStateForKeys()) {
+ if (q.metric !== 'requests' || typeof q.limit !== 'number' || q.limit <= 0) continue;
+ if (typeof q.remaining !== 'number') continue;
+ const pct = Math.max(0, Math.min(100, Math.round((q.remaining / q.limit) * 100)));
+ const prev = requestsRemainingByPlatform.get(q.platform);
+ // Keep the tightest (lowest) headroom seen for the platform.
+ if (prev === undefined || pct < prev) requestsRemainingByPlatform.set(q.platform, pct);
+ }
+
+ const counts = { healthy: 0, rate_limited: 0, invalid: 0, unknown: 0 };
+
+ const providers = platformRows
+ .filter(p => p.enabled_keys > 0)
+ .map(p => {
+ const onCooldown = resumeByPlatform.has(p.platform);
+ let status: 'healthy' | 'rate_limited' | 'invalid' | 'unknown';
+ if (p.healthy_keys > 0 && !onCooldown) {
+ status = 'healthy';
+ } else if (onCooldown) {
+ status = 'rate_limited';
+ } else if (p.unknown_keys > 0) {
+ status = 'unknown';
+ } else if (p.invalid_keys > 0 || p.error_keys > 0) {
+ status = 'invalid';
+ } else {
+ status = 'unknown';
+ }
+ counts[status] += 1;
+
+ const provider = getProvider(p.platform as Platform);
+ const entry: Record<string, unknown> = {
+ platform: p.platform,
+ name: provider?.name ?? p.platform,
+ status,
+ keys: p.enabled_keys,
+ };
+ if (status === 'rate_limited') {
+ const resumeMs = resumeByPlatform.get(p.platform);
+ if (resumeMs != null) entry.resume_at = new Date(resumeMs).toISOString();
+ }
+ if (status === 'invalid') {
+ const lastError = lastErrorByPlatform.get(p.platform);
+ if (lastError) entry.last_error = lastError;
+ }
+ const requestsRemaining = requestsRemainingByPlatform.get(p.platform);
+ if (requestsRemaining != null) entry.requests_remaining_pct = requestsRemaining;
+ return entry;
+ });
+
+ res.json({ providers, counts });
+});