7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fallback): bench the whole key when a provider reports the account suspended (#1201)
Chong Jia Zhen23baea01daf3
diff --git a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts b/server/src/__tests__/lib/fallback-loop-account-suspended.test.tsnew file mode 100644index 00000000..c851d98e--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts@@ -0,0 +1,168 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// Account-level suspension: a relay answers EVERY model behind one key with the+// same 403 (observed on NavyAI: "The Free plan is temporarily disabled due to+// abuse"). Classified as model-forbidden it benched a single model per attempt,+// so on a platform with a large catalog each request spent its whole failover+// budget re-discovering the same dead account. The key must sink out of routing+// on every model of the platform at once.++vi.mock('../../services/health.js', () => ({+ checkKeyHealth: vi.fn(),+ markKeyHealthyFromRequest: vi.fn(),+}));++import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import {+ newFallbackState,+ recordRetryableFailure,+ classifyAttemptError,+ cooldownDecisionForError,+ resetModelFailureWindows,+} from '../../lib/fallback-loop.js';+import { isAccountSuspendedError, isModelNotFoundError } from '../../lib/error-classify.js';+import {+ isOnCooldown,+ getActiveCooldownsForKeys,+ clearCooldownsForKey,+ resetKeyLocalityCache,+ setCooldownCeilingMs,+ PAYMENT_REQUIRED_COOLDOWN_MS,+} from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++const PLATFORM = 'groq';+let keyA = 0;+let keyB = 0;+let models: { id: number; model_id: string }[] = [];++function insertKey(label: string): number {+ const { encrypted, iv, authTag } = encrypt(`test-${label}`);+ const info = getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)+ `).run(PLATFORM, label, encrypted, iv, authTag);+ return Number(info.lastInsertRowid);+}++function routeFor(model: { id: number; model_id: string }, keyId: number): RouteResult {+ return {+ provider: {} as any,+ modelId: model.model_id,+ modelDbId: model.id,+ apiKey: 'k',+ keyId,+ platform: PLATFORM,+ displayName: model.model_id,+ rpdLimit: null,+ tpdLimit: null,+ };+}++// The exact NavyAI body observed on the gray tier (URL redacted by the proxy).+const SUSPENDED = () => Object.assign(+ new Error('NavyAI API error 403: The Free plan is temporarily disabled due to abuse. You can purchase a plan at [redacted-url] to continue using the API.'),+ { status: 403 },+);+// A model-tier 403 behind an otherwise-valid key (#256): stays per-model.+const TIER_403 = () => Object.assign(+ new Error('Groq API error 403: model llama-guard is not available on the free tier'),+ { status: 403 },+);++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ keyA = insertKey('key-a');+ keyB = insertKey('key-b');+ resetKeyLocalityCache();+ models = db.prepare(+ 'SELECT id, model_id FROM models WHERE platform = ? AND enabled = 1 ORDER BY id'+ ).all(PLATFORM) as { id: number; model_id: string }[];+ expect(models.length).toBeGreaterThanOrEqual(3);+});++beforeEach(() => {+ // Memory AND disk: isOnCooldown falls back to the in-process map when no+ // persisted row exists, so a table wipe alone leaks the last case's bench.+ clearCooldownsForKey(keyA);+ clearCooldownsForKey(keyB);+ resetModelFailureWindows();+});++describe('isAccountSuspendedError', () => {+ it('matches the NavyAI plan-disabled 403 and generic account-suspension wording', () => {+ expect(isAccountSuspendedError(SUSPENDED())).toBe(true);+ expect(isAccountSuspendedError(Object.assign(new Error('API error 403: Your account has been suspended'), { status: 403 }))).toBe(true);+ expect(isAccountSuspendedError(new Error('API error 403: account is banned'))).toBe(true);+ });++ it('is status-gated: the wording alone never condemns a key', () => {+ expect(isAccountSuspendedError(TIER_403())).toBe(false);+ expect(isAccountSuspendedError(Object.assign(new Error('Rate limited; note your plan is disabled for bursts'), { status: 429 }))).toBe(false);+ expect(isAccountSuspendedError(Object.assign(new Error('The Free plan is temporarily disabled due to abuse'), { status: 400 }))).toBe(false);+ });++ it('is not the model-not-found family', () => {+ expect(isModelNotFoundError(SUSPENDED())).toBe(false);+ });+});++describe('a suspended account benches the whole key, not one model', () => {+ it('cools every enabled model of the platform on the failing key for the credit window', () => {+ const state = newFallbackState();+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), state);++ for (const m of models) {+ expect(isOnCooldown(PLATFORM, m.model_id, keyA), m.model_id).toBe(true);+ }+ // Credit-length bench (a day), on the sibling models too, not the 90s+ // transient or the per-model tier window.+ expect(cooldownDecisionForError(routeFor(models[0], keyA), SUSPENDED()).source).toBe('credit');+ const cds = getActiveCooldownsForKeys([keyA]).get(keyA) ?? [];+ const sibling = cds.find(c => c.modelId === models[1].model_id);+ expect(sibling).toBeDefined();+ expect(sibling!.expiresAtMs - Date.now()).toBeGreaterThan(PAYMENT_REQUIRED_COOLDOWN_MS - 60_000);+ // The platform is out for the rest of THIS request.+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);+ });++ it('leaves a sibling key on the same platform alone', () => {+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), newFallbackState());+ for (const m of models) {+ expect(isOnCooldown(PLATFORM, m.model_id, keyB), m.model_id).toBe(false);+ }+ });++ it('keeps a model-tier 403 on the per-model path', () => {+ const state = newFallbackState();+ recordRetryableFailure(routeFor(models[0], keyA), TIER_403(), state);+ expect(isOnCooldown(PLATFORM, models[0].model_id, keyA)).toBe(true);+ expect(isOnCooldown(PLATFORM, models[1].model_id, keyA)).toBe(false);+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);+ expect(cooldownDecisionForError(routeFor(models[0], keyA), TIER_403()).source).toBe('tier');+ });+++ // #952: the bench is one of the router's OWN guesses, so an operator ceiling+ // caps it like the 402/403 benches it borrows its duration from.+ it('honours the operator cooldown ceiling', () => {+ setCooldownCeilingMs(10 * 60_000);+ try {+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), newFallbackState());+ const cds = getActiveCooldownsForKeys([keyA]).get(keyA) ?? [];+ expect(cds.length).toBeGreaterThan(1);+ for (const cd of cds) {+ expect(cd.expiresAtMs - Date.now()).toBeLessThanOrEqual(10 * 60_000 + 1_000);+ }+ } finally {+ setCooldownCeilingMs(null);+ }+ });+ it('classifies for the attempt trail as forbidden (a 403 is what the client sees)', () => {+ expect(classifyAttemptError(SUSPENDED())).toBe('forbidden');+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 81049125..ffbef44f 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -508,6 +508,24 @@ export function isModelNotFoundError(err: any): boolean {|| msg.includes('410') || msg.includes('gone');}+// A 403 that suspends the ACCOUNT, not one model: NavyAI answers every model+// behind a benched free key with "The Free plan is temporarily disabled due to+// abuse. You can purchase a plan ...". Every model of the platform fails the+// same way, so classifying it as model-forbidden benched ONE model per attempt:+// with 93 catalog rows on that platform, each request burned its whole failover+// budget re-discovering the same dead account. Not key-auth either: the+// credential itself is valid, the plan behind it is not, and validateKey's+// /models probe still passes, so the health checker never demotes the key.+// Status-gated to 403 (or a status-less message that names one) so provider+// wording alone can never condemn a healthy key.+export function isAccountSuspendedError(err: any): boolean {+ const status = typeof err?.status === 'number' ? err.status : 0;+ const msg = (err?.message ?? '').toLowerCase();+ if (status !== 403 && !(status === 0 && msg.includes('403'))) return false;+ return /\b(plan|account|subscription) (is|has been|was) (temporarily )?(disabled|suspended|banned|deactivated)\b/.test(msg)+ || msg.includes('due to abuse');+}+// A 403 Forbidden returned for a specific model behind an otherwise-valid key.// Drives the same whole-model skip as a 404: every key on this platform's tier// would be forbidden the same model, so rule it out for the rest of the requestdiff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex ece1c7e5..4625246b 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -29,6 +29,7 @@ import {getModelForbiddenCooldownMs,learnLimitFromError,type CooldownDecision,+ type CooldownSource,} from '../services/ratelimit.js';import {isRetryableError,@@ -40,6 +41,7 @@ import {isPaymentRequiredError,isModelNotFoundError,isModelAccessForbiddenError,+ isAccountSuspendedError,isProviderBadRequestError,isProviderDegradedError,isProviderLevelError,@@ -49,7 +51,7 @@ import {import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';import { noteModelRetirementSignal } from '../services/model-retirement.js';-import { getSetting } from '../db/index.js';+import { getDb, getSetting } from '../db/index.js';import { newBreaker, recordBreakerFailure } from './guardrails.js';import { getRequestTrace, newRequestTrace, runWithRequestTrace, type AttemptOutcome, type AttemptTraceRecord, type RequestTrace } from './attempt-trace.js';import { logRequest, persistRequestAttempts } from './request-log.js';@@ -112,6 +114,30 @@ function clearModelFailure(route: RouteResult): void {modelFailureTimestamps.delete(route.modelDbId);}+/** Bench one key on every enabled model of its platform it can route to (the+ * route's own model is benched by the caller). For KEY-level verdicts only —+ * an account suspension — never for a model-level one. Never throws: an+ * unreadable catalog leaves the narrower per-route bench in place.+ */+function benchKeyAcrossPlatform(route: RouteResult, durationMs: number, source: CooldownSource): void {+ let modelIds: string[] = [];+ try {+ modelIds = (getDb().prepare(+ 'SELECT model_id FROM models WHERE platform = ? AND enabled = 1 AND (key_id IS NULL OR key_id = ?)',+ ).all(route.platform, route.keyId) as { model_id: string }[]).map(r => r.model_id);+ } catch (dbErr: any) {+ console.warn(`[FallbackLoop] could not list ${route.platform} models to bench key ${route.keyId}: ${dbErr?.message ?? dbErr}`);+ return;+ }+ let benched = 0;+ for (const modelId of modelIds) {+ if (modelId === route.modelId) continue;+ setCooldown(route.platform, modelId, route.keyId, durationMs, source);+ benched++;+ }+ console.warn(`[FallbackLoop] ${route.platform} key ${route.keyId} reports the account suspended; benched it on ${benched + 1} model(s) for ${Math.round(durationMs / 60_000)}min`);+}+// ── Wall-clock retry budget ──────────────────────────────────────────────────// Serial failover has no time bound of its own: the observed worst case was a// 38.8s TTFB over 11 attempts, and the theoretical worst is maxRetries x the@@ -212,6 +238,10 @@ export function cooldownForError(route: RouteResult, err: any): number {*/export function cooldownDecisionForError(route: RouteResult, err: any): CooldownDecision {if (isPaymentRequiredError(err)) return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };+ // Before the model-forbidden check: a suspended account is a 403 too, but+ // it is the KEY that is out, for as long as an empty balance would be — and+ // under the same operator ceiling (#952).+ if (isAccountSuspendedError(err)) return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };if (isModelAccessForbiddenError(err)) return { durationMs: getModelForbiddenCooldownMs(), source: 'tier' };if (isDailyQuotaExhaustedError(err)) {return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };@@ -340,6 +370,19 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (consumeSkipBenchExemption(route, err)) return true;const decision = cooldownDecisionForError(route, err);setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);+ // A suspended ACCOUNT fails every model behind the key identically. The+ // per-route cooldown above benches only the model that happened to be+ // tried, so the next request picks the platform's next model and pays the+ // same round trip again — on a platform with a large catalog that is the+ // whole failover budget, every request, until each model has tripped+ // separately. Bench the key across the platform's catalog in one go, and+ // rule the platform out for the rest of this request: a sibling key would+ // be a different account, but with the budget already spent on this one+ // the next PROVIDER is the better hop.+ if (isAccountSuspendedError(err)) {+ state.skipPlatforms.add(route.platform);+ benchKeyAcrossPlatform(route, decision.durationMs, decision.source);+ }// Model-level failure benching: a model failing across keys (or repeatedly on// one key) must sink out of routing instead of being re-picked every request.noteModelFailure(route, now);