7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(server): auto-cooldown repeatedly failing models, extend penalty beyond 429 (#806)
suantea (asuan_suan)398ed976d74e
diff --git a/server/src/__tests__/lib/fallback-loop-model-bench.test.ts b/server/src/__tests__/lib/fallback-loop-model-bench.test.tsnew file mode 100644index 00000000..eaf90ffa--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-model-bench.test.ts@@ -0,0 +1,126 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// Model-level failure benching: a model that keeps failing upstream must sink+// out of routing on EVERY key that can serve it, not only on the key that+// happened to fail last — the failure window counts across keys, so a per-key+// bench would leave the siblings feeding the same sick model.++vi.mock('../../services/health.js', () => ({+ checkKeyHealth: vi.fn(),+ // recordUpstreamSuccess touches this on the streak-clearing path.+ markKeyHealthyFromRequest: vi.fn(),+}));++import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import {+ newFallbackState,+ recordRetryableFailure,+ recordUpstreamSuccess,+ MODEL_FAILURE_COOLDOWN_MS,+ MODEL_FAILURE_THRESHOLD,+} from '../../lib/fallback-loop.js';+import {+ getActiveCooldownsForKeys,+ isOnCooldown,+ resetKeyLocalityCache,+} from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++const PLATFORM = 'groq';+let keyA = 0;+let keyB = 0;+let modelA: { id: number; model_id: string };+let modelB: { id: number; model_id: string };++function insertKey(label: string): number {+ const { encrypted, iv, authTag } = encrypt(`test-${label}`);+ const info = getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)+ `).run(PLATFORM, label, encrypted, iv, authTag);+ return Number(info.lastInsertRowid);+}++function routeFor(model: { id: number; model_id: string }, keyId: number): RouteResult {+ return {+ provider: {} as any,+ modelId: model.model_id,+ modelDbId: model.id,+ apiKey: 'k',+ keyId,+ platform: PLATFORM,+ displayName: model.model_id,+ rpdLimit: null,+ tpdLimit: null,+ };+}++// A plain 5xx: retryable, no quota signal, so the per-key bench is the short+// transient one (90s) — far below the model bench, which is what makes the+// assertions below able to tell the two apart.+const upstream500 = () => Object.assign(new Error('Groq API error 500: upstream'), { status: 500 });++function failOnce(model: { id: number; model_id: string }, keyId: number) {+ // Fresh state per call: each failure stands for a separate request.+ recordRetryableFailure(routeFor(model, keyId), upstream500(), newFallbackState());+}++function cooldownFor(model: { id: number; model_id: string }, keyId: number) {+ return getActiveCooldownsForKeys([keyId])+ .get(keyId)+ ?.find(c => c.platform === PLATFORM && c.modelId === model.model_id);+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ keyA = insertKey('key-a');+ keyB = insertKey('key-b');+ resetKeyLocalityCache();+ const models = db.prepare(+ 'SELECT id, model_id FROM models WHERE platform = ? ORDER BY id LIMIT 2'+ ).all(PLATFORM) as { id: number; model_id: string }[];+ expect(models.length).toBe(2);+ [modelA, modelB] = models;+});++beforeEach(() => {+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+});++describe('model-level failure benching covers every key of the model', () => {+ it('benches BOTH keys once the window trips, even though only one key failed last', () => {+ // Three failures inside the window, spread across the two keys.+ failOnce(modelA, keyA);+ failOnce(modelA, keyB);+ expect(MODEL_FAILURE_THRESHOLD).toBe(3);+ failOnce(modelA, keyA);++ for (const keyId of [keyA, keyB]) {+ expect(isOnCooldown(PLATFORM, modelA.model_id, keyId)).toBe(true);+ const cd = cooldownFor(modelA, keyId);+ expect(cd).toBeDefined();+ // The per-key transient bench is 90s; only the model bench reaches here.+ expect(cd!.remainingMs).toBeGreaterThan(MODEL_FAILURE_COOLDOWN_MS / 2);+ }+ });++ it('starts the streak over after a served request, so the next failure benches nothing extra', () => {+ failOnce(modelB, keyA);+ failOnce(modelB, keyB);+ // A served request is counter-evidence: the two failures above no longer count.+ recordUpstreamSuccess(routeFor(modelB, keyA), 100);+ failOnce(modelB, keyA);++ // Only the short per-key benches, no model-wide one.+ const benchedB = cooldownFor(modelB, keyB);+ expect(benchedB).toBeDefined();+ expect(benchedB!.remainingMs).toBeLessThan(MODEL_FAILURE_COOLDOWN_MS / 2);+ const benchedA = cooldownFor(modelB, keyA);+ expect(benchedA).toBeDefined();+ expect(benchedA!.remainingMs).toBeLessThan(MODEL_FAILURE_COOLDOWN_MS / 2);+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 93b8bd84..0f8fa23c 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -16,12 +16,13 @@// the fire-and-forget key revalidation kicked off on an upstream 401 (below).import type { RouteResult } from '../services/router.js';-import { recordRateLimitHit, recordSuccess, hasOtherUsableKey, formatResetEta } from '../services/router.js';+import { recordRateLimitHit, recordModelFailure, recordSuccess, hasOtherUsableKey, routableKeyIdsForModel, formatResetEta } from '../services/router.js';import { safeHeaderValue } from './header-value.js';import {recordRequest,recordTokens,setCooldown,+ getActiveCooldownsForKeys,getCooldownDecisionForLimit,getSoonestCooldownExpiry,PAYMENT_REQUIRED_COOLDOWN_MS,@@ -54,6 +55,61 @@ import { logRequest, persistRequestAttempts } from './request-log.js';// Every surface caps failover hops at the same number.export const FALLBACK_MAX_RETRIES = 20;+// ── Model-level failure benching ─────────────────────────────────────────────+// A model that keeps failing upstream (401/429/5xx/empty-stream/timeout) must+// not keep being picked by auto-routing: every dead-end attempt wastes seconds+// of user-visible latency. The per-key cooldown above benches the *key*; this+// is the model-level counterpart — a sliding window of failures *across keys*+// that benches the whole MODEL once the streak is convincing. The window counts+// across keys, so the bench must span keys too: benching only the key that+// happened to fail last leaves every sibling key serving the same sick model+// until each one trips its own streak. Recovery stays automatic: the bench is+// 'heuristic', the only source the cooldown-probe job may clear early, and it+// re-validates each benched key once half the bench has been served (the probe+// exercises the credential, not the model, so a model still sick behind a good+// key simply re-trips the streak). Client-cancels never reach+// recordRetryableFailure (the loop returns on isClientAbortError before this+// bookkeeping), so they don't count.+const MODEL_FAILURE_WINDOW_MS = 15 * 60 * 1000; // sliding window: 15 min+export const MODEL_FAILURE_THRESHOLD = 3; // failures within window+export const MODEL_FAILURE_COOLDOWN_MS = 10 * 60 * 1000; // bench duration: 10 min+const modelFailureTimestamps = new Map<number, number[]>(); // model_db_id → times++/** Record one retryable upstream failure for a model. Once the sliding window+ * holds ≥ MODEL_FAILURE_THRESHOLD failures, bench the model on EVERY key that+ * can route to it so the model sinks out of routing until upstream heals, then+ * reset the counter (one bench per streak). */+function noteModelFailure(route: RouteResult): void {+ const now = Date.now();+ const window = (modelFailureTimestamps.get(route.modelDbId) ?? [])+ .filter(t => now - t < MODEL_FAILURE_WINDOW_MS);+ window.push(now);+ modelFailureTimestamps.set(route.modelDbId, window);+ if (window.length < MODEL_FAILURE_THRESHOLD) return;+ // Fall back to the failing key alone when the model's key set can't be read+ // (model row gone, DB unavailable) — a narrower bench beats none.+ const keyIds = routableKeyIdsForModel(route.modelDbId);+ const targets = keyIds.length > 0 ? keyIds : [route.keyId];+ const benchUntil = now + MODEL_FAILURE_COOLDOWN_MS;+ const active = getActiveCooldownsForKeys(targets, now);+ for (const keyId of targets) {+ // Never SHORTEN an existing bench: a provider-stated reset or an escalated+ // ladder step outlasting this window knows more than this heuristic does,+ // and overwriting it would also drop its non-probeable provenance.+ const current = active.get(keyId)+ ?.find(c => c.platform === route.platform && c.modelId === route.modelId);+ if (current && current.expiresAtMs >= benchUntil) continue;+ setCooldown(route.platform, route.modelId, keyId, MODEL_FAILURE_COOLDOWN_MS, 'heuristic');+ }+ modelFailureTimestamps.delete(route.modelDbId);+}++/** A served request is the strongest counter-evidence: clear the failure+ * window so a recovered model is not benched for a stale streak. */+function clearModelFailure(route: RouteResult): void {+ modelFailureTimestamps.delete(route.modelDbId);+}+// ── Wall-clock retry budget ──────────────────────────────────────────────────// Serial failover has no time bound of its own: the observed worst case was a// 38.8s TTFB over 11 attempts, and the theoretical worst is maxRetries x the@@ -241,9 +297,19 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (consumeSkipBenchExemption(route, err)) return true;const decision = cooldownDecisionForError(route, err);setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);+ // Model-level failure benching: a model failing across keys (or repeatedly on+ // one key) must sink out of routing instead of being re-picked every request.+ noteModelFailure(route);// Model-level penalty only when no sibling key can still serve (#454).if (!hasOtherUsableKey(route.modelDbId, route.keyId, state.skipKeys)) {- recordRateLimitHit(route.modelDbId);+ // Hard limit signals (429/402) carry the heavier demotion; ordinary+ // upstream failures (5xx/timeout/empty stream) get a lighter one so the+ // model sinks gradually instead of being banished on a single blip.+ if (isRateLimitSignal(err)) {+ recordRateLimitHit(route.modelDbId);+ } else {+ recordModelFailure(route.modelDbId);+ }}learnLimitFromError(route.modelDbId, err);return false;@@ -300,6 +366,9 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numbe// A served request proves the model+key can complete: the empty-completion// streak (#751) starts over.emptyCompletionStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);+ // A served request is the strongest possible evidence the model works, so+ // clear any model-level failure streak that could bench it later.+ clearModelFailure(route);// A served request is the strongest possible evidence the key works, so clear// any stale 'error' status left by an earlier transport blip instead of waiting// for the next health pass to make the key routable again.diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 1302cf8b..906c3a8a 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -225,14 +225,19 @@ const rateLimitPenalties = new Map<number, { count: number; lastHit: number; pen// Penalty decays over time so models recoverconst PENALTY_PER_429 = 3; // each 429 adds this many priority positions+const PENALTY_PER_FAIL = 1; // each non-limit upstream failure (5xx/timeout/empty stream)const MAX_PENALTY = 10; // cap so a model doesn't sink foreverconst DECAY_INTERVAL_MS = 2 * 60 * 1000; // penalty decays every 2 minutesconst DECAY_AMOUNT = 1; // remove this much penalty per decay interval/**- * Record a 429 for a model — increases its penalty so it sinks in priority.+ * Record an upstream failure for a model — increases its penalty so it sinks in+ * priority. Default weight is the LIGHT one for ordinary upstream failures+ * (5xx/timeout/empty stream, +1); callers that know they saw a hard limit+ * signal (429/402) pass the heavier weight — a quota limit is the stronger,+ * longer-lived health cue.*/-export function recordRateLimitHit(modelDbId: number) {+export function recordModelFailure(modelDbId: number, weight = PENALTY_PER_FAIL) {const existing = rateLimitPenalties.get(modelDbId);const now = Date.now();if (existing) {@@ -240,12 +245,20 @@ export function recordRateLimitHit(modelDbId: number) {existing.penalty = Math.max(0, existing.penalty - decaySteps * DECAY_AMOUNT);existing.count++;existing.lastHit = now;- existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY);+ existing.penalty = Math.min(existing.penalty + weight, MAX_PENALTY);} else {- rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: PENALTY_PER_429 });+ rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: weight });}}+/**+ * Record a 429 for a model — heavier penalty (priority demotion) than ordinary+ * failures, since a rate-limit signal is the strongest short-term health cue.+ */+export function recordRateLimitHit(modelDbId: number) {+ recordModelFailure(modelDbId, PENALTY_PER_429);+}+/*** Record a success for a model — reduces its penalty so it rises back up.*/@@ -1236,6 +1249,34 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, skireturn false;}+/**+ * Every key that can be ROUTED to this model: enabled + healthy/unknown, not+ * scoped away from the model (#657), and — for a custom model — belonging to+ * the model's own endpoint (#212, #619). Deliberately ignores the transient+ * gates hasOtherUsableKey applies (cooldown, quotas): the caller here is the+ * model-level bench, which needs the full key set to take a sick model out of+ * rotation, not "who could serve the next request".+ */+export function routableKeyIdsForModel(modelDbId: number): number[] {+ const db = getDb();+ const m = db.prepare('SELECT platform, model_id, key_id FROM models WHERE id = ?')+ .get(modelDbId) as { platform: string; model_id: string; key_id: number | null } | undefined;+ if (!m) return [];++ const keys = db.prepare(+ "SELECT id, model_scope_json FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"+ ).all(m.platform) as { id: number; model_scope_json: string | null }[];++ const endpointKeyIds = m.platform === 'custom' && m.key_id != null+ ? customEndpointKeyIds(db, m.key_id)+ : null;++ return keys+ .filter(k => !endpointKeyIds || endpointKeyIds.has(k.id))+ .filter(k => scopeAllows(parseModelScope(k.model_scope_json), m.model_id))+ .map(k => k.id);+}+/*** Fetch a single enabled model's chain row by its db id.*/