7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(server): degraded-mode state machine (#904) (#906)
suantea (asuan_suan)f412e9723979
diff --git a/server/src/__tests__/services/degradation.test.ts b/server/src/__tests__/services/degradation.test.tsnew file mode 100644index 00000000..098cff3a--- /dev/null+++ b/server/src/__tests__/services/degradation.test.ts@@ -0,0 +1,166 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++const { initDb, getDb } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const {+ computeHealthSnapshot,+ updateDegradationState,+ getDegradationStatus,+ isDegraded,+ resetDegradationState,+} = await import('../../services/degradation.js');++// #904 degraded-mode state machine: when the healthy-provider ratio stays+// below the threshold for a sustained period, the gateway flips to degraded+// mode (router stops exploring, health endpoint reports the state); recovery+// needs the ratio back above threshold for a longer grace period.++let nextId = 40000;++function seedKey(platform: string, opts: { status?: string; enabled?: number } = {}): number {+ const id = ++nextId;+ const enc = encrypt(`deg-${id}`);+ getDb().prepare(`+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)+ `).run(id, platform, `deg-${id}`, enc.encrypted, enc.iv, enc.authTag, opts.enabled ?? 1, opts.status ?? 'healthy');+ return id;+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ getDb().prepare('DELETE FROM api_keys').run();+ resetDegradationState();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+});++describe('degraded-mode state machine', () => {+ it('reports normal with all providers healthy', () => {+ seedKey('groq');+ seedKey('cloudflare');+ seedKey('mistral');+ const s = updateDegradationState(0);+ expect(s.state).toBe('normal');+ expect(s.healthyProviders).toBe(3);+ expect(s.totalProviders).toBe(3);+ expect(s.ratio).toBe(1);+ expect(isDegraded()).toBe(false);+ });++ it('counts unknown-status keys as usable, error keys as not', () => {+ seedKey('groq', { status: 'unknown' });+ seedKey('cloudflare', { status: 'healthy' });+ seedKey('mistral', { status: 'error' });+ const snap = computeHealthSnapshot();+ expect(snap.healthyProviders).toBe(2);+ expect(snap.totalProviders).toBe(3);+ expect(snap.ratio).toBeCloseTo(2 / 3);+ });++ it('ignores disabled keys', () => {+ seedKey('groq', { enabled: 0, status: 'healthy' });+ seedKey('cloudflare', { enabled: 0, status: 'healthy' });+ seedKey('mistral', { enabled: 0, status: 'healthy' });+ const snap = computeHealthSnapshot();+ expect(snap.totalProviders).toBe(0);+ expect(snap.ratio).toBe(1); // no providers to judge → not degraded+ });++ it('enters degraded mode only after the ratio stays low for the grace period', () => {+ seedKey('groq', { status: 'healthy' });+ seedKey('cloudflare', { status: 'error' });+ seedKey('mistral', { status: 'error' });+ seedKey('openai', { status: 'error' });++ // Below threshold (1/4 = 25% < 50%) but grace period not elapsed yet.+ const early = updateDegradationState(0);+ expect(early.state).toBe('normal');+ expect(isDegraded()).toBe(false);++ // Within the entry grace (60s default) still normal.+ const within = updateDegradationState(59_000);+ expect(within.state).toBe('normal');++ // Past the grace → degraded.+ const degraded = updateDegradationState(60_000);+ expect(degraded.state).toBe('degraded');+ expect(degraded.degradedAt).not.toBeNull();+ expect(isDegraded()).toBe(true);+ });++ it('stays normal when the ratio is at or above the threshold', () => {+ seedKey('groq', { status: 'healthy' });+ seedKey('cloudflare', { status: 'healthy' });+ seedKey('mistral', { status: 'error' });+ seedKey('openai', { status: 'error' });+ // 2/4 = 50% — exactly at the default threshold → not below.+ const s = updateDegradationState(0);+ expect(s.state).toBe('normal');+ });++ it('exits degraded mode only after recovery grace elapses', () => {+ seedKey('groq', { status: 'healthy' });+ seedKey('cloudflare', { status: 'error' });+ seedKey('mistral', { status: 'error' });+ seedKey('openai', { status: 'error' });+ updateDegradationState(0);+ updateDegradationState(60_000);+ expect(isDegraded()).toBe(true);++ // Providers recover, but the exit grace (120s default) hasn't elapsed.+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform IN ('cloudflare', 'mistral', 'openai')").run();+ const recovering = updateDegradationState(60_001);+ expect(recovering.state).toBe('degraded'); // still degraded+ expect(isDegraded()).toBe(true);++ // Past the exit grace → back to normal.+ const recovered = updateDegradationState(60_001 + 120_000);+ expect(recovered.state).toBe('normal');+ expect(recovered.degradedAt).toBeNull();+ expect(isDegraded()).toBe(false);+ });++ it('does not flap: a transient single-pass recovery resets the recovery streak', () => {+ seedKey('groq', { status: 'healthy' });+ seedKey('cloudflare', { status: 'error' });+ seedKey('mistral', { status: 'error' });+ seedKey('openai', { status: 'error' });+ updateDegradationState(0);+ updateDegradationState(60_000);+ expect(isDegraded()).toBe(true);++ // One good pass then back down: the recovery streak must reset so the+ // gateway stays degraded instead of exiting early.+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform = 'cloudflare'").run();+ updateDegradationState(60_001); // recovered pass starts streak+ getDb().prepare("UPDATE api_keys SET status = 'error' WHERE platform = 'cloudflare'").run();+ updateDegradationState(60_002); // down again — streak reset+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform IN ('cloudflare', 'mistral', 'openai')").run();+ const afterLongRecovery = updateDegradationState(60_002 + 120_000);+ expect(afterLongRecovery.state).toBe('degraded'); // exit streak restarted at 60_002+ });++ it('does not degrade when there are fewer providers than DEGRADED_MIN_PROVIDERS', () => {+ seedKey('groq', { status: 'healthy' });+ seedKey('cloudflare', { status: 'error' });+ // Only 2 providers < default min of 3 → never degraded.+ const s = updateDegradationState(0);+ updateDegradationState(60_000);+ expect(s.state).toBe('normal');+ expect(isDegraded()).toBe(false);+ });++ it('getDegradationStatus returns the last snapshot without re-querying', () => {+ seedKey('groq', { status: 'healthy' });+ updateDegradationState(0);+ const status = getDegradationStatus();+ expect(status.totalProviders).toBe(1);+ expect(status.state).toBe('normal');+ });+});diff --git a/server/src/routes/health.ts b/server/src/routes/health.tsindex 74ae3141..1791fa25 100644--- a/server/src/routes/health.ts+++ b/server/src/routes/health.ts@@ -2,6 +2,7 @@ import { Router } from 'express';import type { Request, Response } from 'express';import { getDb } from '../db/index.js';import { checkKeyHealth, checkAllKeys } from '../services/health.js';+import { getDegradationStatus } from '../services/degradation.js';import { hasProvider } from '../providers/index.js';import { getQuotaStateForKeys } from '../services/provider-quota.js';@@ -54,6 +55,7 @@ healthRouter.get('/', (_req: Request, res: Response) => {lastHealthError: k.last_health_error,})),quotaStates: getQuotaStateForKeys(),+ degradation: getDegradationStatus(),});});diff --git a/server/src/services/degradation.ts b/server/src/services/degradation.tsnew file mode 100644index 00000000..59f2f3c2--- /dev/null+++ b/server/src/services/degradation.ts@@ -0,0 +1,173 @@+import { getDb } from '../db/index.js';++/**+ * Global degraded-mode state machine (#904).+ *+ * When a large share of enabled providers fails at once, per-request probing+ * and bandit exploration just burn retry budget on dead routes. This module+ * tracks the healthy-provider ratio (driven by the scheduled health pass) and+ * flips the gateway into a degraded state once the ratio stays below a+ * threshold for a sustained period. While degraded, the router skips+ * exploration and sticks to the scored order of remaining healthy providers;+ * the health endpoint reports the state so operators can see it.+ *+ * Recovery is the mirror image: the ratio must stay at or above the threshold+ * for a sustained period before the gateway exits degraded mode, so a single+ * flickering pass doesn't flap the state.+ */++export type DegradationState = 'normal' | 'degraded';++export interface HealthSnapshot {+ /** Enabled providers that still have at least one usable key. */+ healthyProviders: number;+ /** Enabled providers with at least one key, usable or not. */+ totalProviders: number;+ /** healthyProviders / totalProviders, 1 when there are no providers. */+ ratio: number;+}++export interface DegradationStatus extends HealthSnapshot {+ state: DegradationState;+ /** ms since epoch when degraded mode was entered; null while normal. */+ degradedAt: number | null;+}++// Healthy keys are the ones the router can actually use. 'unknown' counts as+// healthy: an unprobed key is treated as usable until a probe says otherwise+// (the router does the same for skip/fallback decisions).+const HEALTHY_STATUSES = new Set(['healthy', 'unknown']);++function positiveEnv(raw: string | undefined, fallback: number): number {+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isFinite(n) && n > 0) return n;+ }+ return fallback;+}++// Fraction of providers that must remain healthy to stay out of degraded mode.+const DEGRADED_HEALTHY_RATIO = positiveEnv(process.env.DEGRADED_HEALTHY_RATIO, 0.5);+// Only meaningful when there are at least this many enabled providers — a+// single-provider deployment shouldn't flap degraded mode every outage.+const DEGRADED_MIN_PROVIDERS = positiveEnv(process.env.DEGRADED_MIN_PROVIDERS, 3);+// How long the ratio must stay below the threshold before entering degraded+// mode, so a transient bad pass doesn't flip the gateway.+const DEGRADED_ENTRY_GRACE_MS = positiveEnv(process.env.DEGRADED_ENTRY_GRACE_MS, 60_000);+// How long the ratio must stay at/above the threshold before exiting degraded+// mode. Longer than the entry grace: exiting prematurely re-enters quickly.+const DEGRADED_EXIT_GRACE_MS = positiveEnv(process.env.DEGRADED_EXIT_GRACE_MS, 120_000);++interface State {+ state: DegradationState;+ degradedAt: number | null;+ /** ms since epoch the ratio first dropped below the threshold (streak start). */+ belowSince: number | null;+ /** ms since epoch the ratio first recovered to the threshold (streak start). */+ recoveredSince: number | null;+}++const state: State = {+ state: 'normal',+ degradedAt: null,+ belowSince: null,+ recoveredSince: null,+};++let lastSnapshot: HealthSnapshot | null = null;++/** Compute the current healthy-provider ratio from the key table. Exported for+ * tests to inject a fake DB; callers normally use updateDegradationState. */+export function computeHealthSnapshot(): HealthSnapshot {+ const db = getDb();+ const rows = db.prepare(`+ SELECT platform, status+ FROM api_keys+ WHERE enabled = 1+ `).all() as { platform: string; status: string }[];++ const usable = new Set<string>();+ const present = new Set<string>();+ for (const row of rows) {+ present.add(row.platform);+ if (HEALTHY_STATUSES.has(row.status)) usable.add(row.platform);+ }++ const totalProviders = present.size;+ const healthyProviders = usable.size;+ const ratio = totalProviders === 0 ? 1 : healthyProviders / totalProviders;+ const snapshot: HealthSnapshot = { healthyProviders, totalProviders, ratio };+ lastSnapshot = snapshot;+ return snapshot;+}++/** Re-evaluate the degraded state from the current key table. Call this after+ * every health pass; also callable on demand (dashboard, router entry). */+export function updateDegradationState(now = Date.now()): DegradationStatus {+ const snapshot = computeHealthSnapshot();++ // Degradation is only meaningful with enough providers to judge.+ if (snapshot.totalProviders < DEGRADED_MIN_PROVIDERS) {+ state.state = 'normal';+ state.degradedAt = null;+ state.belowSince = null;+ state.recoveredSince = null;+ return getDegradationStatus();+ }++ const belowThreshold = snapshot.ratio < DEGRADED_HEALTHY_RATIO;++ if (belowThreshold) {+ state.recoveredSince = null;+ if (state.state === 'normal') {+ state.belowSince ??= now;+ if (now - state.belowSince >= DEGRADED_ENTRY_GRACE_MS) {+ state.state = 'degraded';+ state.degradedAt = now;+ console.warn(+ `[Degradation] Entering degraded mode: ${snapshot.healthyProviders}/${snapshot.totalProviders} providers healthy ` ++ `(${(snapshot.ratio * 100).toFixed(0)}% < ${(DEGRADED_HEALTHY_RATIO * 100).toFixed(0)}%)`,+ );+ }+ }+ } else {+ state.belowSince = null;+ if (state.state === 'degraded') {+ state.recoveredSince ??= now;+ if (now - state.recoveredSince >= DEGRADED_EXIT_GRACE_MS) {+ state.state = 'normal';+ state.degradedAt = null;+ console.log(+ `[Degradation] Exiting degraded mode: ${snapshot.healthyProviders}/${snapshot.totalProviders} providers healthy ` ++ `(${(snapshot.ratio * 100).toFixed(0)}% >= ${(DEGRADED_HEALTHY_RATIO * 100).toFixed(0)}%)`,+ );+ }+ } else {+ state.recoveredSince = null;+ }+ }++ return getDegradationStatus();+}++export function isDegraded(): boolean {+ return state.state === 'degraded';+}++export function getDegradationStatus(): DegradationStatus {+ const snapshot = lastSnapshot ?? computeHealthSnapshot();+ return {+ ...snapshot,+ state: state.state,+ degradedAt: state.degradedAt,+ };+}++/** Reset the machine (tests, and the post-start re-probe path). */+export function resetDegradationState(): void {+ state.state = 'normal';+ state.degradedAt = null;+ state.belowSince = null;+ state.recoveredSince = null;+ lastSnapshot = null;+}diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex dc9927f1..6a171dfb 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -5,6 +5,7 @@ import { decryptProxyUrl } from '../lib/key-proxy.js';import { withKeyProxy } from '../lib/proxy.js';import type { Platform, KeyStatus } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey } from './provider-quota.js';+import { updateDegradationState } from './degradation.js';import type { Scheduler } from '../lib/scheduler.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';@@ -276,6 +277,10 @@ export function checkAllKeys(opts: HealthPassOptions = {}): Promise<HealthPassRecheckAllInFlight = runHealthPass(opts).finally(() => {checkAllInFlight = null;});+ // Keep the degraded-mode state machine in step with the latest verdicts+ // (#904): a fleet-wide outage should flip the gateway into degraded mode+ // shortly after the pass that observed it, not after the next request.+ void checkAllInFlight.then(() => updateDegradationState());return checkAllInFlight;}diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 1063836d..66137eff 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -28,6 +28,7 @@ import { platformDropsResponseFormat } from '../lib/sampling-params.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from './model-groups.js';import { getActiveProfileId } from './profile-models.js';import { customEndpointKeyIds } from './custom-endpoint.js';+import { isDegraded } from './degradation.js';import { modelStatsKey, endpointScopeForBaseUrl } from '../lib/endpoint-scope.js';import { parseModelScope, scopeAllows } from '../lib/model-scope.js';import type { BaseProvider } from '../providers/base.js';@@ -1563,7 +1564,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// that can't serve the request ahead of capable ones would just get it// skipped a moment later (e.g. an image request must never randomly probe a// text-only model).- if (strategy !== 'priority' && getExploreEnabled() && Math.random() < EXPLORE_CHANCE) {+ // While the gateway is in degraded mode (#904) exploration is skipped+ // entirely: probing unmeasured models during a fleet-wide outage just burns+ // retry budget on the same dead providers, and the scored order of known+ // survivors is the only thing worth trying.+ if (strategy !== 'priority' && getExploreEnabled() && !isDegraded() && Math.random() < EXPLORE_CHANCE) {// A model the operator zeroed out via MODEL_ROUTING_OVERRIDES never wins a// bandit draw, so it would stay under EXPLORE_MIN_SAMPLES forever and become// a perpetual probe target — the explicit ban outranks exploration.