7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(server): add /livez, /readyz, /v1/providers + ?ready=true for meta-gateway interop (#594)
GAURANG PATEL43a73579ece5
diff --git a/server/src/__tests__/routes/status.test.ts b/server/src/__tests__/routes/status.test.tsnew file mode 100644index 00000000..bdc97c0a--- /dev/null+++ b/server/src/__tests__/routes/status.test.ts@@ -0,0 +1,106 @@+import { describe, it, expect, beforeAll } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';++async function request(+ app: Express,+ method: string,+ path: string,+ headers: Record<string, string> = {},+) {+ const server = app.listen(0);+ const addr = server.address() as { port: number };+ const url = `http://127.0.0.1:${addr.port}${path}`;+ const res = await fetch(url, { method, headers });+ const data = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: data };+}++function insertHealthyKey(platform: string) {+ const secret = encrypt(`${platform}-test-key`);+ getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'test', ?, ?, ?, 'healthy', 1)+ `).run(platform, secret.encrypted, secret.iv, secret.authTag);+}++// These tests share one in-memory DB and run top-to-bottom: the "no upstreams"+// assertions must precede the key insert.+describe('status & interop endpoints (#433)', () => {+ let app: Express;+ let unifiedKey = '';++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ unifiedKey = getUnifiedApiKey();+ });++ it('GET /livez returns 200 with status, version, and uptime', async () => {+ const { status, body } = await request(app, 'GET', '/livez');+ expect(status).toBe(200);+ expect(body.status).toBe('ok');+ expect(typeof body.version).toBe('string');+ expect(typeof body.uptime_s).toBe('number');+ });++ it('GET /readyz returns 503 no_upstreams_configured when nothing is set up', async () => {+ const { status, body } = await request(app, 'GET', '/readyz');+ expect(status).toBe(503);+ expect(body.reason).toBe('no_upstreams_configured');+ });++ it('GET /v1/providers rejects a missing/invalid key with 401', async () => {+ const { status, body } = await request(app, 'GET', '/v1/providers');+ expect(status).toBe(401);+ expect(body.error.type).toBe('authentication_error');+ });++ it('GET /v1/providers returns empty providers and zero counts before any key', async () => {+ const { status, body } = await request(app, 'GET', '/v1/providers', {+ Authorization: `Bearer ${unifiedKey}`,+ });+ expect(status).toBe(200);+ expect(body.providers).toEqual([]);+ expect(body.counts).toEqual({ healthy: 0, rate_limited: 0, invalid: 0, unknown: 0 });+ });++ it('GET /readyz returns 200 once a healthy upstream exists', async () => {+ insertHealthyKey('groq');+ const { status, body } = await request(app, 'GET', '/readyz');+ expect(status).toBe(200);+ expect(body.status).toBe('ok');+ expect(body.ready_upstreams).toBeGreaterThanOrEqual(1);+ });++ it('GET /v1/providers reports the healthy upstream', async () => {+ const { status, body } = await request(app, 'GET', '/v1/providers', {+ 'x-api-key': unifiedKey,+ });+ expect(status).toBe(200);+ const groq = body.providers.find((p: { platform: string }) => p.platform === 'groq');+ expect(groq).toBeDefined();+ expect(groq.status).toBe('healthy');+ expect(groq.keys).toBe(1);+ expect(body.counts.healthy).toBeGreaterThanOrEqual(1);+ });++ it('GET /v1/models?ready=true filters to servable models (auth required)', async () => {+ const unauth = await request(app, 'GET', '/v1/models?ready=true');+ expect(unauth.status).toBe(401);++ const full = await request(app, 'GET', '/v1/models', { 'x-api-key': unifiedKey });+ const ready = await request(app, 'GET', '/v1/models?ready=true', { 'x-api-key': unifiedKey });+ expect(ready.status).toBe(200);+ expect(ready.body.object).toBe('list');+ // Every non-virtual model in the filtered list is actually available, and the+ // filter never returns more than the unfiltered catalog.+ const realModels = ready.body.data.filter((m: { id: string }) => m.id !== 'auto' && m.id !== 'fusion');+ for (const m of realModels) expect(m.available).toBe(true);+ expect(ready.body.data.length).toBeLessThanOrEqual(full.body.data.length);+ });+});diff --git a/server/src/app.ts b/server/src/app.tsindex a93e4e2e..c48cbaf3 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -22,6 +22,7 @@ import { compressionRouter } from './routes/compression.js';import { authRouter } from './routes/auth.js';import { docsRouter } from './routes/docs.js';import { mcpRouter } from './routes/mcp.js';+import { statusRouter, providersRouter } from './routes/status.js';import { geminiRouter } from './routes/gemini.js';import { ollamaRouter } from './routes/ollama.js';import { urlTokenRouter } from './routes/url-tokens.js';@@ -120,6 +121,12 @@ export function createApp(config?: Config) {// owns those two paths; everything else falls through to the routers below.app.use('/v1', docsRouter);+ // Read-only per-upstream status for meta-gateways (GET /v1/providers, #433).+ // Unified-key auth is enforced inside the handler; mounted before the rate+ // limiter (like docsRouter) so status polling doesn't draw down a caller's+ // request budget.+ app.use('/v1', providersRouter);+// Separately revocable URL tokens for clients that cannot set headers.app.use('/v1/t/:token', createProxyRateLimiter(cfg.proxyRateLimitRpm));app.use('/v1/t/:token', urlTokenRouter);@@ -155,6 +162,12 @@ export function createApp(config?: Config) {res.json({ status: 'ok', timestamp: new Date().toISOString() });});+ // Liveness / readiness probes for orchestrators (GET /livez, /readyz, #433).+ // Unauthenticated so a load balancer can probe them; registered before the+ // static / SPA-fallback block below so these root paths resolve to JSON+ // instead of index.html.+ app.use(statusRouter);+// Error handler (for API routes)app.use(errorHandler);diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 1eeb3bf7..983eb181 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -208,9 +208,11 @@ proxyRouter.get('/models', (req: Request, res: Response) => {// By default we return the WHOLE catalog (one row per model id), each tagged// with whether it is currently usable, so a client can see everything and know- // what's connected vs. disabled/keyless (#242). `?available=true` (alias- // `?connected=true`) narrows the list to only models that can serve a request- // right now — the previous default behavior. `available` is computed as+ // what's connected vs. disabled/keyless (#242). `?available=true` (aliases+ // `?connected=true`, `?ready=true`) narrows the list to only models that can+ // serve a request right now — the previous default behavior. The `ready`+ // alias is the machine-readable filter a meta-gateway uses (#433) to ask+ // "which models can this instance actually serve now". `available` is computed as// "enabled AND an enabled key can serve it"; dedup prefers an available// instance of a model id over a disabled/keyless one.// Shared catalog listing (one source of truth for the OpenAI and Anthropic@@ -221,7 +223,7 @@ proxyRouter.get('/models', (req: Request, res: Response) => {// conservative default and truncate long inputs before they reach us (#282).const { models: allListed, autoContextWindow } = buildModelListing();- const q = String(req.query.available ?? req.query.connected ?? '').toLowerCase();+ const q = String(req.query.available ?? req.query.connected ?? req.query.ready ?? '').toLowerCase();const onlyAvailable = q === '1' || q === 'true' || q === 'yes';const listed = onlyAvailable ? allListed.filter(m => m.available === 1) : allListed;diff --git a/server/src/routes/status.ts b/server/src/routes/status.tsnew file mode 100644index 00000000..4c216585--- /dev/null+++ b/server/src/routes/status.ts@@ -0,0 +1,208 @@+import { Router } from 'express';+import type { Request, Response } from 'express';+import type { Platform } from '@freellmapi/shared/types.js';+import { getDb, getUnifiedApiKey } from '../db/index.js';+import { isEncryptionKeyInitialized } from '../lib/crypto.js';+import { extractApiToken, timingSafeStringEqual } from './proxy.js';+import { getProvider } from '../providers/index.js';+import { getSoonestCooldownExpiry } from '../services/ratelimit.js';+import { getQuotaStateForKeys } from '../services/provider-quota.js';+import { openapiSpec } from '../docs/openapi.js';++// Machine-readable operational endpoints for meta-gateways / orchestrators that+// run this proxy as one upstream lane (LibreChat, LiteLLM Proxy, OmniRoute — see+// #433). `/livez` and `/readyz` are unauthenticated so a load balancer can probe+// them; `/v1/providers` sits behind the same unified API key as the rest of /v1.++export const statusRouter = Router();+export const providersRouter = Router();++// Cheap "is the SQLite connection answering" probe. getDb() throws before initDb+// has run, and prepare/get throws if the handle is closed — either way the+// server is not serviceable, which is exactly the /livez and /readyz 503 signal.+function isDbReachable(): boolean {+ try {+ getDb().prepare('SELECT 1').get();+ return true;+ } catch {+ return false;+ }+}++// GET /livez — liveness. 200 when the process can serve at all: the DB answers+// and the encryption key is loaded (without it, stored provider keys can't be+// decrypted, so no request can be routed). 503 otherwise.+statusRouter.get('/livez', (_req: Request, res: Response) => {+ const db = isDbReachable();+ const encryptionKey = isEncryptionKeyInitialized();+ const uptime_s = Math.floor(process.uptime());+ const version = openapiSpec.info.version;++ if (!db || !encryptionKey) {+ res.status(503).json({+ status: 'unavailable',+ version,+ uptime_s,+ checks: { db, encryption_key: encryptionKey },+ });+ return;+ }++ res.json({ status: 'ok', version, uptime_s });+});++// GET /readyz — readiness. 200 when at least one upstream is serviceable right+// now (an enabled key whose status is healthy or not-yet-probed). 503 with a+// machine-readable reason otherwise, so a meta-gateway can skip this instance+// instead of eating a failover on every call.+statusRouter.get('/readyz', (_req: Request, res: Response) => {+ if (!isDbReachable()) {+ res.status(503).json({ status: 'unavailable', reason: 'db_unreachable' });+ return;+ }++ const db = getDb();+ const agg = db.prepare(`+ SELECT+ COUNT(*) AS enabled_keys,+ COUNT(DISTINCT CASE WHEN status IN ('healthy', 'unknown') THEN platform END) AS ready_upstreams+ FROM api_keys+ WHERE enabled = 1+ `).get() as { enabled_keys: number; ready_upstreams: number };++ const readyUpstreams = agg.ready_upstreams ?? 0;+ if (readyUpstreams > 0) {+ res.json({ status: 'ok', ready_upstreams: readyUpstreams });+ return;+ }++ // Distinguish "nothing configured" from "everything cooling down" from+ // "everything unhealthy" so the caller can act on the specific reason.+ let reason: string;+ if ((agg.enabled_keys ?? 0) === 0) {+ reason = 'no_upstreams_configured';+ } else if (getSoonestCooldownExpiry() != null) {+ reason = 'all_upstreams_rate_limited';+ } else {+ reason = 'all_upstreams_unhealthy';+ }++ res.status(503).json({ status: 'unavailable', reason });+});++interface PlatformKeyRow {+ platform: string;+ enabled_keys: number;+ healthy_keys: number;+ unknown_keys: number;+ invalid_keys: number;+ error_keys: number;+}++// GET /v1/providers — aggregate per-upstream state a smart router needs to+// decide "route here or skip", with no key material and no PII. Behind the+// unified API key (like the rest of /v1); mounted before the proxy rate limiter+// so status polling doesn't draw down a caller's request budget.+providersRouter.get('/providers', (req: Request, res: Response) => {+ const token = extractApiToken(req);+ const unifiedKey = getUnifiedApiKey();+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {+ res.status(401).json({ error: { message: 'Invalid API key', type: 'authentication_error' } });+ return;+ }++ const db = getDb();++ // Per-platform key-status rollup (only enabled keys count toward serviceability).+ const platformRows = db.prepare(`+ SELECT+ platform,+ SUM(CASE WHEN enabled = 1 THEN 1 ELSE 0 END) AS enabled_keys,+ SUM(CASE WHEN enabled = 1 AND status = 'healthy' THEN 1 ELSE 0 END) AS healthy_keys,+ SUM(CASE WHEN enabled = 1 AND status = 'unknown' THEN 1 ELSE 0 END) AS unknown_keys,+ SUM(CASE WHEN enabled = 1 AND status = 'invalid' THEN 1 ELSE 0 END) AS invalid_keys,+ SUM(CASE WHEN enabled = 1 AND status = 'error' THEN 1 ELSE 0 END) AS error_keys+ FROM api_keys+ GROUP BY platform+ `).all() as PlatformKeyRow[];++ // Active cooldowns (rate limiting) rolled up to the platform, with the soonest+ // resume time. rate_limit_cooldowns is per (platform, model, key); the MIN is+ // the earliest moment any of that platform's routes frees up.+ const now = Date.now();+ const cooldownRows = db.prepare(`+ SELECT platform, MIN(expires_at_ms) AS resume_ms+ FROM rate_limit_cooldowns+ WHERE expires_at_ms > ?+ GROUP BY platform+ `).all(now) as Array<{ platform: string; resume_ms: number }>;+ const resumeByPlatform = new Map(cooldownRows.map(r => [r.platform, r.resume_ms]));++ // Most recent probe error per platform (enabled keys, newest first).+ const errorRows = db.prepare(`+ SELECT platform, last_health_error+ FROM api_keys+ WHERE enabled = 1 AND last_health_error IS NOT NULL+ ORDER BY last_checked_at DESC+ `).all() as Array<{ platform: string; last_health_error: string }>;+ const lastErrorByPlatform = new Map<string, string>();+ for (const row of errorRows) {+ if (!lastErrorByPlatform.has(row.platform)) {+ lastErrorByPlatform.set(row.platform, row.last_health_error);+ }+ }++ // Best-effort request-quota headroom from observed quota state. Only surfaced+ // when a provider actually reported a `requests` limit; absent otherwise.+ const requestsRemainingByPlatform = new Map<string, number>();+ for (const q of getQuotaStateForKeys()) {+ if (q.metric !== 'requests' || typeof q.limit !== 'number' || q.limit <= 0) continue;+ if (typeof q.remaining !== 'number') continue;+ const pct = Math.max(0, Math.min(100, Math.round((q.remaining / q.limit) * 100)));+ const prev = requestsRemainingByPlatform.get(q.platform);+ // Keep the tightest (lowest) headroom seen for the platform.+ if (prev === undefined || pct < prev) requestsRemainingByPlatform.set(q.platform, pct);+ }++ const counts = { healthy: 0, rate_limited: 0, invalid: 0, unknown: 0 };++ const providers = platformRows+ .filter(p => p.enabled_keys > 0)+ .map(p => {+ const onCooldown = resumeByPlatform.has(p.platform);+ let status: 'healthy' | 'rate_limited' | 'invalid' | 'unknown';+ if (p.healthy_keys > 0 && !onCooldown) {+ status = 'healthy';+ } else if (onCooldown) {+ status = 'rate_limited';+ } else if (p.unknown_keys > 0) {+ status = 'unknown';+ } else if (p.invalid_keys > 0 || p.error_keys > 0) {+ status = 'invalid';+ } else {+ status = 'unknown';+ }+ counts[status] += 1;++ const provider = getProvider(p.platform as Platform);+ const entry: Record<string, unknown> = {+ platform: p.platform,+ name: provider?.name ?? p.platform,+ status,+ keys: p.enabled_keys,+ };+ if (status === 'rate_limited') {+ const resumeMs = resumeByPlatform.get(p.platform);+ if (resumeMs != null) entry.resume_at = new Date(resumeMs).toISOString();+ }+ if (status === 'invalid') {+ const lastError = lastErrorByPlatform.get(p.platform);+ if (lastError) entry.last_error = lastError;+ }+ const requestsRemaining = requestsRemainingByPlatform.get(p.platform);+ if (requestsRemaining != null) entry.requests_remaining_pct = requestsRemaining;+ return entry;+ });++ res.json({ providers, counts });+});