Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(server): degraded-mode state machine (#904) (#906)

suantea (asuan_suan)f412e9723979
diff --git a/server/src/__tests__/services/degradation.test.ts b/server/src/__tests__/services/degradation.test.ts
new file mode 100644
index 00000000..098cff3a
--- /dev/null
+++ b/server/src/__tests__/services/degradation.test.ts
@@ -0,0 +1,166 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+const { initDb, getDb } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const {
+ computeHealthSnapshot,
+ updateDegradationState,
+ getDegradationStatus,
+ isDegraded,
+ resetDegradationState,
+} = await import('../../services/degradation.js');
+
+// #904 degraded-mode state machine: when the healthy-provider ratio stays
+// below the threshold for a sustained period, the gateway flips to degraded
+// mode (router stops exploring, health endpoint reports the state); recovery
+// needs the ratio back above threshold for a longer grace period.
+
+let nextId = 40000;
+
+function seedKey(platform: string, opts: { status?: string; enabled?: number } = {}): number {
+ const id = ++nextId;
+ const enc = encrypt(`deg-${id}`);
+ getDb().prepare(`
+ INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, enabled, status)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+ `).run(id, platform, `deg-${id}`, enc.encrypted, enc.iv, enc.authTag, opts.enabled ?? 1, opts.status ?? 'healthy');
+ return id;
+}
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ getDb().prepare('DELETE FROM api_keys').run();
+ resetDegradationState();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+});
+
+describe('degraded-mode state machine', () => {
+ it('reports normal with all providers healthy', () => {
+ seedKey('groq');
+ seedKey('cloudflare');
+ seedKey('mistral');
+ const s = updateDegradationState(0);
+ expect(s.state).toBe('normal');
+ expect(s.healthyProviders).toBe(3);
+ expect(s.totalProviders).toBe(3);
+ expect(s.ratio).toBe(1);
+ expect(isDegraded()).toBe(false);
+ });
+
+ it('counts unknown-status keys as usable, error keys as not', () => {
+ seedKey('groq', { status: 'unknown' });
+ seedKey('cloudflare', { status: 'healthy' });
+ seedKey('mistral', { status: 'error' });
+ const snap = computeHealthSnapshot();
+ expect(snap.healthyProviders).toBe(2);
+ expect(snap.totalProviders).toBe(3);
+ expect(snap.ratio).toBeCloseTo(2 / 3);
+ });
+
+ it('ignores disabled keys', () => {
+ seedKey('groq', { enabled: 0, status: 'healthy' });
+ seedKey('cloudflare', { enabled: 0, status: 'healthy' });
+ seedKey('mistral', { enabled: 0, status: 'healthy' });
+ const snap = computeHealthSnapshot();
+ expect(snap.totalProviders).toBe(0);
+ expect(snap.ratio).toBe(1); // no providers to judge → not degraded
+ });
+
+ it('enters degraded mode only after the ratio stays low for the grace period', () => {
+ seedKey('groq', { status: 'healthy' });
+ seedKey('cloudflare', { status: 'error' });
+ seedKey('mistral', { status: 'error' });
+ seedKey('openai', { status: 'error' });
+
+ // Below threshold (1/4 = 25% < 50%) but grace period not elapsed yet.
+ const early = updateDegradationState(0);
+ expect(early.state).toBe('normal');
+ expect(isDegraded()).toBe(false);
+
+ // Within the entry grace (60s default) still normal.
+ const within = updateDegradationState(59_000);
+ expect(within.state).toBe('normal');
+
+ // Past the grace → degraded.
+ const degraded = updateDegradationState(60_000);
+ expect(degraded.state).toBe('degraded');
+ expect(degraded.degradedAt).not.toBeNull();
+ expect(isDegraded()).toBe(true);
+ });
+
+ it('stays normal when the ratio is at or above the threshold', () => {
+ seedKey('groq', { status: 'healthy' });
+ seedKey('cloudflare', { status: 'healthy' });
+ seedKey('mistral', { status: 'error' });
+ seedKey('openai', { status: 'error' });
+ // 2/4 = 50% — exactly at the default threshold → not below.
+ const s = updateDegradationState(0);
+ expect(s.state).toBe('normal');
+ });
+
+ it('exits degraded mode only after recovery grace elapses', () => {
+ seedKey('groq', { status: 'healthy' });
+ seedKey('cloudflare', { status: 'error' });
+ seedKey('mistral', { status: 'error' });
+ seedKey('openai', { status: 'error' });
+ updateDegradationState(0);
+ updateDegradationState(60_000);
+ expect(isDegraded()).toBe(true);
+
+ // Providers recover, but the exit grace (120s default) hasn't elapsed.
+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform IN ('cloudflare', 'mistral', 'openai')").run();
+ const recovering = updateDegradationState(60_001);
+ expect(recovering.state).toBe('degraded'); // still degraded
+ expect(isDegraded()).toBe(true);
+
+ // Past the exit grace → back to normal.
+ const recovered = updateDegradationState(60_001 + 120_000);
+ expect(recovered.state).toBe('normal');
+ expect(recovered.degradedAt).toBeNull();
+ expect(isDegraded()).toBe(false);
+ });
+
+ it('does not flap: a transient single-pass recovery resets the recovery streak', () => {
+ seedKey('groq', { status: 'healthy' });
+ seedKey('cloudflare', { status: 'error' });
+ seedKey('mistral', { status: 'error' });
+ seedKey('openai', { status: 'error' });
+ updateDegradationState(0);
+ updateDegradationState(60_000);
+ expect(isDegraded()).toBe(true);
+
+ // One good pass then back down: the recovery streak must reset so the
+ // gateway stays degraded instead of exiting early.
+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform = 'cloudflare'").run();
+ updateDegradationState(60_001); // recovered pass starts streak
+ getDb().prepare("UPDATE api_keys SET status = 'error' WHERE platform = 'cloudflare'").run();
+ updateDegradationState(60_002); // down again — streak reset
+ getDb().prepare("UPDATE api_keys SET status = 'healthy' WHERE platform IN ('cloudflare', 'mistral', 'openai')").run();
+ const afterLongRecovery = updateDegradationState(60_002 + 120_000);
+ expect(afterLongRecovery.state).toBe('degraded'); // exit streak restarted at 60_002
+ });
+
+ it('does not degrade when there are fewer providers than DEGRADED_MIN_PROVIDERS', () => {
+ seedKey('groq', { status: 'healthy' });
+ seedKey('cloudflare', { status: 'error' });
+ // Only 2 providers < default min of 3 → never degraded.
+ const s = updateDegradationState(0);
+ updateDegradationState(60_000);
+ expect(s.state).toBe('normal');
+ expect(isDegraded()).toBe(false);
+ });
+
+ it('getDegradationStatus returns the last snapshot without re-querying', () => {
+ seedKey('groq', { status: 'healthy' });
+ updateDegradationState(0);
+ const status = getDegradationStatus();
+ expect(status.totalProviders).toBe(1);
+ expect(status.state).toBe('normal');
+ });
+});
diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts
index 74ae3141..1791fa25 100644
--- a/server/src/routes/health.ts
+++ b/server/src/routes/health.ts
@@ -2,6 +2,7 @@ import { Router } from 'express';
import type { Request, Response } from 'express';
import { getDb } from '../db/index.js';
import { checkKeyHealth, checkAllKeys } from '../services/health.js';
+import { getDegradationStatus } from '../services/degradation.js';
import { hasProvider } from '../providers/index.js';
import { getQuotaStateForKeys } from '../services/provider-quota.js';
@@ -54,6 +55,7 @@ healthRouter.get('/', (_req: Request, res: Response) => {
lastHealthError: k.last_health_error,
})),
quotaStates: getQuotaStateForKeys(),
+ degradation: getDegradationStatus(),
});
});
diff --git a/server/src/services/degradation.ts b/server/src/services/degradation.ts
new file mode 100644
index 00000000..59f2f3c2
--- /dev/null
+++ b/server/src/services/degradation.ts
@@ -0,0 +1,173 @@
+import { getDb } from '../db/index.js';
+
+/**
+ * Global degraded-mode state machine (#904).
+ *
+ * When a large share of enabled providers fails at once, per-request probing
+ * and bandit exploration just burn retry budget on dead routes. This module
+ * tracks the healthy-provider ratio (driven by the scheduled health pass) and
+ * flips the gateway into a degraded state once the ratio stays below a
+ * threshold for a sustained period. While degraded, the router skips
+ * exploration and sticks to the scored order of remaining healthy providers;
+ * the health endpoint reports the state so operators can see it.
+ *
+ * Recovery is the mirror image: the ratio must stay at or above the threshold
+ * for a sustained period before the gateway exits degraded mode, so a single
+ * flickering pass doesn't flap the state.
+ */
+
+export type DegradationState = 'normal' | 'degraded';
+
+export interface HealthSnapshot {
+ /** Enabled providers that still have at least one usable key. */
+ healthyProviders: number;
+ /** Enabled providers with at least one key, usable or not. */
+ totalProviders: number;
+ /** healthyProviders / totalProviders, 1 when there are no providers. */
+ ratio: number;
+}
+
+export interface DegradationStatus extends HealthSnapshot {
+ state: DegradationState;
+ /** ms since epoch when degraded mode was entered; null while normal. */
+ degradedAt: number | null;
+}
+
+// Healthy keys are the ones the router can actually use. 'unknown' counts as
+// healthy: an unprobed key is treated as usable until a probe says otherwise
+// (the router does the same for skip/fallback decisions).
+const HEALTHY_STATUSES = new Set(['healthy', 'unknown']);
+
+function positiveEnv(raw: string | undefined, fallback: number): number {
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isFinite(n) && n > 0) return n;
+ }
+ return fallback;
+}
+
+// Fraction of providers that must remain healthy to stay out of degraded mode.
+const DEGRADED_HEALTHY_RATIO = positiveEnv(process.env.DEGRADED_HEALTHY_RATIO, 0.5);
+// Only meaningful when there are at least this many enabled providers — a
+// single-provider deployment shouldn't flap degraded mode every outage.
+const DEGRADED_MIN_PROVIDERS = positiveEnv(process.env.DEGRADED_MIN_PROVIDERS, 3);
+// How long the ratio must stay below the threshold before entering degraded
+// mode, so a transient bad pass doesn't flip the gateway.
+const DEGRADED_ENTRY_GRACE_MS = positiveEnv(process.env.DEGRADED_ENTRY_GRACE_MS, 60_000);
+// How long the ratio must stay at/above the threshold before exiting degraded
+// mode. Longer than the entry grace: exiting prematurely re-enters quickly.
+const DEGRADED_EXIT_GRACE_MS = positiveEnv(process.env.DEGRADED_EXIT_GRACE_MS, 120_000);
+
+interface State {
+ state: DegradationState;
+ degradedAt: number | null;
+ /** ms since epoch the ratio first dropped below the threshold (streak start). */
+ belowSince: number | null;
+ /** ms since epoch the ratio first recovered to the threshold (streak start). */
+ recoveredSince: number | null;
+}
+
+const state: State = {
+ state: 'normal',
+ degradedAt: null,
+ belowSince: null,
+ recoveredSince: null,
+};
+
+let lastSnapshot: HealthSnapshot | null = null;
+
+/** Compute the current healthy-provider ratio from the key table. Exported for
+ * tests to inject a fake DB; callers normally use updateDegradationState. */
+export function computeHealthSnapshot(): HealthSnapshot {
+ const db = getDb();
+ const rows = db.prepare(`
+ SELECT platform, status
+ FROM api_keys
+ WHERE enabled = 1
+ `).all() as { platform: string; status: string }[];
+
+ const usable = new Set<string>();
+ const present = new Set<string>();
+ for (const row of rows) {
+ present.add(row.platform);
+ if (HEALTHY_STATUSES.has(row.status)) usable.add(row.platform);
+ }
+
+ const totalProviders = present.size;
+ const healthyProviders = usable.size;
+ const ratio = totalProviders === 0 ? 1 : healthyProviders / totalProviders;
+ const snapshot: HealthSnapshot = { healthyProviders, totalProviders, ratio };
+ lastSnapshot = snapshot;
+ return snapshot;
+}
+
+/** Re-evaluate the degraded state from the current key table. Call this after
+ * every health pass; also callable on demand (dashboard, router entry). */
+export function updateDegradationState(now = Date.now()): DegradationStatus {
+ const snapshot = computeHealthSnapshot();
+
+ // Degradation is only meaningful with enough providers to judge.
+ if (snapshot.totalProviders < DEGRADED_MIN_PROVIDERS) {
+ state.state = 'normal';
+ state.degradedAt = null;
+ state.belowSince = null;
+ state.recoveredSince = null;
+ return getDegradationStatus();
+ }
+
+ const belowThreshold = snapshot.ratio < DEGRADED_HEALTHY_RATIO;
+
+ if (belowThreshold) {
+ state.recoveredSince = null;
+ if (state.state === 'normal') {
+ state.belowSince ??= now;
+ if (now - state.belowSince >= DEGRADED_ENTRY_GRACE_MS) {
+ state.state = 'degraded';
+ state.degradedAt = now;
+ console.warn(
+ `[Degradation] Entering degraded mode: ${snapshot.healthyProviders}/${snapshot.totalProviders} providers healthy ` +
+ `(${(snapshot.ratio * 100).toFixed(0)}% < ${(DEGRADED_HEALTHY_RATIO * 100).toFixed(0)}%)`,
+ );
+ }
+ }
+ } else {
+ state.belowSince = null;
+ if (state.state === 'degraded') {
+ state.recoveredSince ??= now;
+ if (now - state.recoveredSince >= DEGRADED_EXIT_GRACE_MS) {
+ state.state = 'normal';
+ state.degradedAt = null;
+ console.log(
+ `[Degradation] Exiting degraded mode: ${snapshot.healthyProviders}/${snapshot.totalProviders} providers healthy ` +
+ `(${(snapshot.ratio * 100).toFixed(0)}% >= ${(DEGRADED_HEALTHY_RATIO * 100).toFixed(0)}%)`,
+ );
+ }
+ } else {
+ state.recoveredSince = null;
+ }
+ }
+
+ return getDegradationStatus();
+}
+
+export function isDegraded(): boolean {
+ return state.state === 'degraded';
+}
+
+export function getDegradationStatus(): DegradationStatus {
+ const snapshot = lastSnapshot ?? computeHealthSnapshot();
+ return {
+ ...snapshot,
+ state: state.state,
+ degradedAt: state.degradedAt,
+ };
+}
+
+/** Reset the machine (tests, and the post-start re-probe path). */
+export function resetDegradationState(): void {
+ state.state = 'normal';
+ state.degradedAt = null;
+ state.belowSince = null;
+ state.recoveredSince = null;
+ lastSnapshot = null;
+}
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index dc9927f1..6a171dfb 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -5,6 +5,7 @@ import { decryptProxyUrl } from '../lib/key-proxy.js';
import { withKeyProxy } from '../lib/proxy.js';
import type { Platform, KeyStatus } from '@freellmapi/shared/types.js';
import { inferQuotaPoolKey } from './provider-quota.js';
+import { updateDegradationState } from './degradation.js';
import type { Scheduler } from '../lib/scheduler.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
@@ -276,6 +277,10 @@ export function checkAllKeys(opts: HealthPassOptions = {}): Promise<HealthPassRe
checkAllInFlight = runHealthPass(opts).finally(() => {
checkAllInFlight = null;
});
+ // Keep the degraded-mode state machine in step with the latest verdicts
+ // (#904): a fleet-wide outage should flip the gateway into degraded mode
+ // shortly after the pass that observed it, not after the next request.
+ void checkAllInFlight.then(() => updateDegradationState());
return checkAllInFlight;
}
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 1063836d..66137eff 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -28,6 +28,7 @@ import { platformDropsResponseFormat } from '../lib/sampling-params.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from './model-groups.js';
import { getActiveProfileId } from './profile-models.js';
import { customEndpointKeyIds } from './custom-endpoint.js';
+import { isDegraded } from './degradation.js';
import { modelStatsKey, endpointScopeForBaseUrl } from '../lib/endpoint-scope.js';
import { parseModelScope, scopeAllows } from '../lib/model-scope.js';
import type { BaseProvider } from '../providers/base.js';
@@ -1563,7 +1564,11 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// that can't serve the request ahead of capable ones would just get it
// skipped a moment later (e.g. an image request must never randomly probe a
// text-only model).
- if (strategy !== 'priority' && getExploreEnabled() && Math.random() < EXPLORE_CHANCE) {
+ // While the gateway is in degraded mode (#904) exploration is skipped
+ // entirely: probing unmeasured models during a fleet-wide outage just burns
+ // retry budget on the same dead providers, and the scored order of known
+ // survivors is the only thing worth trying.
+ if (strategy !== 'priority' && getExploreEnabled() && !isDegraded() && Math.random() < EXPLORE_CHANCE) {
// A model the operator zeroed out via MODEL_ROUTING_OVERRIDES never wins a
// bandit draw, so it would stay under EXPLORE_MIN_SAMPLES forever and become
// a perpetual probe target — the explicit ban outranks exploration.