Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(server): auto-cooldown repeatedly failing models, extend penalty beyond 429 (#806)

suantea (asuan_suan)398ed976d74e
diff --git a/server/src/__tests__/lib/fallback-loop-model-bench.test.ts b/server/src/__tests__/lib/fallback-loop-model-bench.test.ts
new file mode 100644
index 00000000..eaf90ffa
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-model-bench.test.ts
@@ -0,0 +1,126 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+// Model-level failure benching: a model that keeps failing upstream must sink
+// out of routing on EVERY key that can serve it, not only on the key that
+// happened to fail last — the failure window counts across keys, so a per-key
+// bench would leave the siblings feeding the same sick model.
+
+vi.mock('../../services/health.js', () => ({
+ checkKeyHealth: vi.fn(),
+ // recordUpstreamSuccess touches this on the streak-clearing path.
+ markKeyHealthyFromRequest: vi.fn(),
+}));
+
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import {
+ newFallbackState,
+ recordRetryableFailure,
+ recordUpstreamSuccess,
+ MODEL_FAILURE_COOLDOWN_MS,
+ MODEL_FAILURE_THRESHOLD,
+} from '../../lib/fallback-loop.js';
+import {
+ getActiveCooldownsForKeys,
+ isOnCooldown,
+ resetKeyLocalityCache,
+} from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+const PLATFORM = 'groq';
+let keyA = 0;
+let keyB = 0;
+let modelA: { id: number; model_id: string };
+let modelB: { id: number; model_id: string };
+
+function insertKey(label: string): number {
+ const { encrypted, iv, authTag } = encrypt(`test-${label}`);
+ const info = getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)
+ `).run(PLATFORM, label, encrypted, iv, authTag);
+ return Number(info.lastInsertRowid);
+}
+
+function routeFor(model: { id: number; model_id: string }, keyId: number): RouteResult {
+ return {
+ provider: {} as any,
+ modelId: model.model_id,
+ modelDbId: model.id,
+ apiKey: 'k',
+ keyId,
+ platform: PLATFORM,
+ displayName: model.model_id,
+ rpdLimit: null,
+ tpdLimit: null,
+ };
+}
+
+// A plain 5xx: retryable, no quota signal, so the per-key bench is the short
+// transient one (90s) — far below the model bench, which is what makes the
+// assertions below able to tell the two apart.
+const upstream500 = () => Object.assign(new Error('Groq API error 500: upstream'), { status: 500 });
+
+function failOnce(model: { id: number; model_id: string }, keyId: number) {
+ // Fresh state per call: each failure stands for a separate request.
+ recordRetryableFailure(routeFor(model, keyId), upstream500(), newFallbackState());
+}
+
+function cooldownFor(model: { id: number; model_id: string }, keyId: number) {
+ return getActiveCooldownsForKeys([keyId])
+ .get(keyId)
+ ?.find(c => c.platform === PLATFORM && c.modelId === model.model_id);
+}
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ keyA = insertKey('key-a');
+ keyB = insertKey('key-b');
+ resetKeyLocalityCache();
+ const models = db.prepare(
+ 'SELECT id, model_id FROM models WHERE platform = ? ORDER BY id LIMIT 2'
+ ).all(PLATFORM) as { id: number; model_id: string }[];
+ expect(models.length).toBe(2);
+ [modelA, modelB] = models;
+});
+
+beforeEach(() => {
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+});
+
+describe('model-level failure benching covers every key of the model', () => {
+ it('benches BOTH keys once the window trips, even though only one key failed last', () => {
+ // Three failures inside the window, spread across the two keys.
+ failOnce(modelA, keyA);
+ failOnce(modelA, keyB);
+ expect(MODEL_FAILURE_THRESHOLD).toBe(3);
+ failOnce(modelA, keyA);
+
+ for (const keyId of [keyA, keyB]) {
+ expect(isOnCooldown(PLATFORM, modelA.model_id, keyId)).toBe(true);
+ const cd = cooldownFor(modelA, keyId);
+ expect(cd).toBeDefined();
+ // The per-key transient bench is 90s; only the model bench reaches here.
+ expect(cd!.remainingMs).toBeGreaterThan(MODEL_FAILURE_COOLDOWN_MS / 2);
+ }
+ });
+
+ it('starts the streak over after a served request, so the next failure benches nothing extra', () => {
+ failOnce(modelB, keyA);
+ failOnce(modelB, keyB);
+ // A served request is counter-evidence: the two failures above no longer count.
+ recordUpstreamSuccess(routeFor(modelB, keyA), 100);
+ failOnce(modelB, keyA);
+
+ // Only the short per-key benches, no model-wide one.
+ const benchedB = cooldownFor(modelB, keyB);
+ expect(benchedB).toBeDefined();
+ expect(benchedB!.remainingMs).toBeLessThan(MODEL_FAILURE_COOLDOWN_MS / 2);
+ const benchedA = cooldownFor(modelB, keyA);
+ expect(benchedA).toBeDefined();
+ expect(benchedA!.remainingMs).toBeLessThan(MODEL_FAILURE_COOLDOWN_MS / 2);
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 93b8bd84..0f8fa23c 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -16,12 +16,13 @@
// the fire-and-forget key revalidation kicked off on an upstream 401 (below).
import type { RouteResult } from '../services/router.js';
-import { recordRateLimitHit, recordSuccess, hasOtherUsableKey, formatResetEta } from '../services/router.js';
+import { recordRateLimitHit, recordModelFailure, recordSuccess, hasOtherUsableKey, routableKeyIdsForModel, formatResetEta } from '../services/router.js';
import { safeHeaderValue } from './header-value.js';
import {
recordRequest,
recordTokens,
setCooldown,
+ getActiveCooldownsForKeys,
getCooldownDecisionForLimit,
getSoonestCooldownExpiry,
PAYMENT_REQUIRED_COOLDOWN_MS,
@@ -54,6 +55,61 @@ import { logRequest, persistRequestAttempts } from './request-log.js';
// Every surface caps failover hops at the same number.
export const FALLBACK_MAX_RETRIES = 20;
+// ── Model-level failure benching ─────────────────────────────────────────────
+// A model that keeps failing upstream (401/429/5xx/empty-stream/timeout) must
+// not keep being picked by auto-routing: every dead-end attempt wastes seconds
+// of user-visible latency. The per-key cooldown above benches the *key*; this
+// is the model-level counterpart — a sliding window of failures *across keys*
+// that benches the whole MODEL once the streak is convincing. The window counts
+// across keys, so the bench must span keys too: benching only the key that
+// happened to fail last leaves every sibling key serving the same sick model
+// until each one trips its own streak. Recovery stays automatic: the bench is
+// 'heuristic', the only source the cooldown-probe job may clear early, and it
+// re-validates each benched key once half the bench has been served (the probe
+// exercises the credential, not the model, so a model still sick behind a good
+// key simply re-trips the streak). Client-cancels never reach
+// recordRetryableFailure (the loop returns on isClientAbortError before this
+// bookkeeping), so they don't count.
+const MODEL_FAILURE_WINDOW_MS = 15 * 60 * 1000; // sliding window: 15 min
+export const MODEL_FAILURE_THRESHOLD = 3; // failures within window
+export const MODEL_FAILURE_COOLDOWN_MS = 10 * 60 * 1000; // bench duration: 10 min
+const modelFailureTimestamps = new Map<number, number[]>(); // model_db_id → times
+
+/** Record one retryable upstream failure for a model. Once the sliding window
+ * holds ≥ MODEL_FAILURE_THRESHOLD failures, bench the model on EVERY key that
+ * can route to it so the model sinks out of routing until upstream heals, then
+ * reset the counter (one bench per streak). */
+function noteModelFailure(route: RouteResult): void {
+ const now = Date.now();
+ const window = (modelFailureTimestamps.get(route.modelDbId) ?? [])
+ .filter(t => now - t < MODEL_FAILURE_WINDOW_MS);
+ window.push(now);
+ modelFailureTimestamps.set(route.modelDbId, window);
+ if (window.length < MODEL_FAILURE_THRESHOLD) return;
+ // Fall back to the failing key alone when the model's key set can't be read
+ // (model row gone, DB unavailable) — a narrower bench beats none.
+ const keyIds = routableKeyIdsForModel(route.modelDbId);
+ const targets = keyIds.length > 0 ? keyIds : [route.keyId];
+ const benchUntil = now + MODEL_FAILURE_COOLDOWN_MS;
+ const active = getActiveCooldownsForKeys(targets, now);
+ for (const keyId of targets) {
+ // Never SHORTEN an existing bench: a provider-stated reset or an escalated
+ // ladder step outlasting this window knows more than this heuristic does,
+ // and overwriting it would also drop its non-probeable provenance.
+ const current = active.get(keyId)
+ ?.find(c => c.platform === route.platform && c.modelId === route.modelId);
+ if (current && current.expiresAtMs >= benchUntil) continue;
+ setCooldown(route.platform, route.modelId, keyId, MODEL_FAILURE_COOLDOWN_MS, 'heuristic');
+ }
+ modelFailureTimestamps.delete(route.modelDbId);
+}
+
+/** A served request is the strongest counter-evidence: clear the failure
+ * window so a recovered model is not benched for a stale streak. */
+function clearModelFailure(route: RouteResult): void {
+ modelFailureTimestamps.delete(route.modelDbId);
+}
+
// ── Wall-clock retry budget ──────────────────────────────────────────────────
// Serial failover has no time bound of its own: the observed worst case was a
// 38.8s TTFB over 11 attempts, and the theoretical worst is maxRetries x the
@@ -241,9 +297,19 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
+ // Model-level failure benching: a model failing across keys (or repeatedly on
+ // one key) must sink out of routing instead of being re-picked every request.
+ noteModelFailure(route);
// Model-level penalty only when no sibling key can still serve (#454).
if (!hasOtherUsableKey(route.modelDbId, route.keyId, state.skipKeys)) {
- recordRateLimitHit(route.modelDbId);
+ // Hard limit signals (429/402) carry the heavier demotion; ordinary
+ // upstream failures (5xx/timeout/empty stream) get a lighter one so the
+ // model sinks gradually instead of being banished on a single blip.
+ if (isRateLimitSignal(err)) {
+ recordRateLimitHit(route.modelDbId);
+ } else {
+ recordModelFailure(route.modelDbId);
+ }
}
learnLimitFromError(route.modelDbId, err);
return false;
@@ -300,6 +366,9 @@ export function recordUpstreamSuccess(route: RouteResult, rateLimitTokens: numbe
// A served request proves the model+key can complete: the empty-completion
// streak (#751) starts over.
emptyCompletionStreaks.delete(`${route.platform}:${route.modelId}:${route.keyId}`);
+ // A served request is the strongest possible evidence the model works, so
+ // clear any model-level failure streak that could bench it later.
+ clearModelFailure(route);
// A served request is the strongest possible evidence the key works, so clear
// any stale 'error' status left by an earlier transport blip instead of waiting
// for the next health pass to make the key routable again.
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 1302cf8b..906c3a8a 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -225,14 +225,19 @@ const rateLimitPenalties = new Map<number, { count: number; lastHit: number; pen
// Penalty decays over time so models recover
const PENALTY_PER_429 = 3; // each 429 adds this many priority positions
+const PENALTY_PER_FAIL = 1; // each non-limit upstream failure (5xx/timeout/empty stream)
const MAX_PENALTY = 10; // cap so a model doesn't sink forever
const DECAY_INTERVAL_MS = 2 * 60 * 1000; // penalty decays every 2 minutes
const DECAY_AMOUNT = 1; // remove this much penalty per decay interval
/**
- * Record a 429 for a model — increases its penalty so it sinks in priority.
+ * Record an upstream failure for a model — increases its penalty so it sinks in
+ * priority. Default weight is the LIGHT one for ordinary upstream failures
+ * (5xx/timeout/empty stream, +1); callers that know they saw a hard limit
+ * signal (429/402) pass the heavier weight — a quota limit is the stronger,
+ * longer-lived health cue.
*/
-export function recordRateLimitHit(modelDbId: number) {
+export function recordModelFailure(modelDbId: number, weight = PENALTY_PER_FAIL) {
const existing = rateLimitPenalties.get(modelDbId);
const now = Date.now();
if (existing) {
@@ -240,12 +245,20 @@ export function recordRateLimitHit(modelDbId: number) {
existing.penalty = Math.max(0, existing.penalty - decaySteps * DECAY_AMOUNT);
existing.count++;
existing.lastHit = now;
- existing.penalty = Math.min(existing.penalty + PENALTY_PER_429, MAX_PENALTY);
+ existing.penalty = Math.min(existing.penalty + weight, MAX_PENALTY);
} else {
- rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: PENALTY_PER_429 });
+ rateLimitPenalties.set(modelDbId, { count: 1, lastHit: now, penalty: weight });
}
}
+/**
+ * Record a 429 for a model — heavier penalty (priority demotion) than ordinary
+ * failures, since a rate-limit signal is the strongest short-term health cue.
+ */
+export function recordRateLimitHit(modelDbId: number) {
+ recordModelFailure(modelDbId, PENALTY_PER_429);
+}
+
/**
* Record a success for a model — reduces its penalty so it rises back up.
*/
@@ -1236,6 +1249,34 @@ export function hasOtherUsableKey(modelDbId: number, excludingKeyId: number, ski
return false;
}
+/**
+ * Every key that can be ROUTED to this model: enabled + healthy/unknown, not
+ * scoped away from the model (#657), and — for a custom model — belonging to
+ * the model's own endpoint (#212, #619). Deliberately ignores the transient
+ * gates hasOtherUsableKey applies (cooldown, quotas): the caller here is the
+ * model-level bench, which needs the full key set to take a sick model out of
+ * rotation, not "who could serve the next request".
+ */
+export function routableKeyIdsForModel(modelDbId: number): number[] {
+ const db = getDb();
+ const m = db.prepare('SELECT platform, model_id, key_id FROM models WHERE id = ?')
+ .get(modelDbId) as { platform: string; model_id: string; key_id: number | null } | undefined;
+ if (!m) return [];
+
+ const keys = db.prepare(
+ "SELECT id, model_scope_json FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown')"
+ ).all(m.platform) as { id: number; model_scope_json: string | null }[];
+
+ const endpointKeyIds = m.platform === 'custom' && m.key_id != null
+ ? customEndpointKeyIds(db, m.key_id)
+ : null;
+
+ return keys
+ .filter(k => !endpointKeyIds || endpointKeyIds.has(k.id))
+ .filter(k => scopeAllows(parseModelScope(k.model_scope_json), m.model_id))
+ .map(k => k.id);
+}
+
/**
* Fetch a single enabled model's chain row by its db id.
*/