Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(fallback): bench the whole key when a provider reports the account suspended (#1201)

Chong Jia Zhen23baea01daf3
diff --git a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts
new file mode 100644
index 00000000..c851d98e
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts
@@ -0,0 +1,168 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+// Account-level suspension: a relay answers EVERY model behind one key with the
+// same 403 (observed on NavyAI: "The Free plan is temporarily disabled due to
+// abuse"). Classified as model-forbidden it benched a single model per attempt,
+// so on a platform with a large catalog each request spent its whole failover
+// budget re-discovering the same dead account. The key must sink out of routing
+// on every model of the platform at once.
+
+vi.mock('../../services/health.js', () => ({
+ checkKeyHealth: vi.fn(),
+ markKeyHealthyFromRequest: vi.fn(),
+}));
+
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import {
+ newFallbackState,
+ recordRetryableFailure,
+ classifyAttemptError,
+ cooldownDecisionForError,
+ resetModelFailureWindows,
+} from '../../lib/fallback-loop.js';
+import { isAccountSuspendedError, isModelNotFoundError } from '../../lib/error-classify.js';
+import {
+ isOnCooldown,
+ getActiveCooldownsForKeys,
+ clearCooldownsForKey,
+ resetKeyLocalityCache,
+ setCooldownCeilingMs,
+ PAYMENT_REQUIRED_COOLDOWN_MS,
+} from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+const PLATFORM = 'groq';
+let keyA = 0;
+let keyB = 0;
+let models: { id: number; model_id: string }[] = [];
+
+function insertKey(label: string): number {
+ const { encrypted, iv, authTag } = encrypt(`test-${label}`);
+ const info = getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)
+ `).run(PLATFORM, label, encrypted, iv, authTag);
+ return Number(info.lastInsertRowid);
+}
+
+function routeFor(model: { id: number; model_id: string }, keyId: number): RouteResult {
+ return {
+ provider: {} as any,
+ modelId: model.model_id,
+ modelDbId: model.id,
+ apiKey: 'k',
+ keyId,
+ platform: PLATFORM,
+ displayName: model.model_id,
+ rpdLimit: null,
+ tpdLimit: null,
+ };
+}
+
+// The exact NavyAI body observed on the gray tier (URL redacted by the proxy).
+const SUSPENDED = () => Object.assign(
+ new Error('NavyAI API error 403: The Free plan is temporarily disabled due to abuse. You can purchase a plan at [redacted-url] to continue using the API.'),
+ { status: 403 },
+);
+// A model-tier 403 behind an otherwise-valid key (#256): stays per-model.
+const TIER_403 = () => Object.assign(
+ new Error('Groq API error 403: model llama-guard is not available on the free tier'),
+ { status: 403 },
+);
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ keyA = insertKey('key-a');
+ keyB = insertKey('key-b');
+ resetKeyLocalityCache();
+ models = db.prepare(
+ 'SELECT id, model_id FROM models WHERE platform = ? AND enabled = 1 ORDER BY id'
+ ).all(PLATFORM) as { id: number; model_id: string }[];
+ expect(models.length).toBeGreaterThanOrEqual(3);
+});
+
+beforeEach(() => {
+ // Memory AND disk: isOnCooldown falls back to the in-process map when no
+ // persisted row exists, so a table wipe alone leaks the last case's bench.
+ clearCooldownsForKey(keyA);
+ clearCooldownsForKey(keyB);
+ resetModelFailureWindows();
+});
+
+describe('isAccountSuspendedError', () => {
+ it('matches the NavyAI plan-disabled 403 and generic account-suspension wording', () => {
+ expect(isAccountSuspendedError(SUSPENDED())).toBe(true);
+ expect(isAccountSuspendedError(Object.assign(new Error('API error 403: Your account has been suspended'), { status: 403 }))).toBe(true);
+ expect(isAccountSuspendedError(new Error('API error 403: account is banned'))).toBe(true);
+ });
+
+ it('is status-gated: the wording alone never condemns a key', () => {
+ expect(isAccountSuspendedError(TIER_403())).toBe(false);
+ expect(isAccountSuspendedError(Object.assign(new Error('Rate limited; note your plan is disabled for bursts'), { status: 429 }))).toBe(false);
+ expect(isAccountSuspendedError(Object.assign(new Error('The Free plan is temporarily disabled due to abuse'), { status: 400 }))).toBe(false);
+ });
+
+ it('is not the model-not-found family', () => {
+ expect(isModelNotFoundError(SUSPENDED())).toBe(false);
+ });
+});
+
+describe('a suspended account benches the whole key, not one model', () => {
+ it('cools every enabled model of the platform on the failing key for the credit window', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), state);
+
+ for (const m of models) {
+ expect(isOnCooldown(PLATFORM, m.model_id, keyA), m.model_id).toBe(true);
+ }
+ // Credit-length bench (a day), on the sibling models too, not the 90s
+ // transient or the per-model tier window.
+ expect(cooldownDecisionForError(routeFor(models[0], keyA), SUSPENDED()).source).toBe('credit');
+ const cds = getActiveCooldownsForKeys([keyA]).get(keyA) ?? [];
+ const sibling = cds.find(c => c.modelId === models[1].model_id);
+ expect(sibling).toBeDefined();
+ expect(sibling!.expiresAtMs - Date.now()).toBeGreaterThan(PAYMENT_REQUIRED_COOLDOWN_MS - 60_000);
+ // The platform is out for the rest of THIS request.
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);
+ });
+
+ it('leaves a sibling key on the same platform alone', () => {
+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), newFallbackState());
+ for (const m of models) {
+ expect(isOnCooldown(PLATFORM, m.model_id, keyB), m.model_id).toBe(false);
+ }
+ });
+
+ it('keeps a model-tier 403 on the per-model path', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(routeFor(models[0], keyA), TIER_403(), state);
+ expect(isOnCooldown(PLATFORM, models[0].model_id, keyA)).toBe(true);
+ expect(isOnCooldown(PLATFORM, models[1].model_id, keyA)).toBe(false);
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);
+ expect(cooldownDecisionForError(routeFor(models[0], keyA), TIER_403()).source).toBe('tier');
+ });
+
+
+ // #952: the bench is one of the router's OWN guesses, so an operator ceiling
+ // caps it like the 402/403 benches it borrows its duration from.
+ it('honours the operator cooldown ceiling', () => {
+ setCooldownCeilingMs(10 * 60_000);
+ try {
+ recordRetryableFailure(routeFor(models[0], keyA), SUSPENDED(), newFallbackState());
+ const cds = getActiveCooldownsForKeys([keyA]).get(keyA) ?? [];
+ expect(cds.length).toBeGreaterThan(1);
+ for (const cd of cds) {
+ expect(cd.expiresAtMs - Date.now()).toBeLessThanOrEqual(10 * 60_000 + 1_000);
+ }
+ } finally {
+ setCooldownCeilingMs(null);
+ }
+ });
+ it('classifies for the attempt trail as forbidden (a 403 is what the client sees)', () => {
+ expect(classifyAttemptError(SUSPENDED())).toBe('forbidden');
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 81049125..ffbef44f 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -508,6 +508,24 @@ export function isModelNotFoundError(err: any): boolean {
|| msg.includes('410') || msg.includes('gone');
}
+// A 403 that suspends the ACCOUNT, not one model: NavyAI answers every model
+// behind a benched free key with "The Free plan is temporarily disabled due to
+// abuse. You can purchase a plan ...". Every model of the platform fails the
+// same way, so classifying it as model-forbidden benched ONE model per attempt:
+// with 93 catalog rows on that platform, each request burned its whole failover
+// budget re-discovering the same dead account. Not key-auth either: the
+// credential itself is valid, the plan behind it is not, and validateKey's
+// /models probe still passes, so the health checker never demotes the key.
+// Status-gated to 403 (or a status-less message that names one) so provider
+// wording alone can never condemn a healthy key.
+export function isAccountSuspendedError(err: any): boolean {
+ const status = typeof err?.status === 'number' ? err.status : 0;
+ const msg = (err?.message ?? '').toLowerCase();
+ if (status !== 403 && !(status === 0 && msg.includes('403'))) return false;
+ return /\b(plan|account|subscription) (is|has been|was) (temporarily )?(disabled|suspended|banned|deactivated)\b/.test(msg)
+ || msg.includes('due to abuse');
+}
+
// A 403 Forbidden returned for a specific model behind an otherwise-valid key.
// Drives the same whole-model skip as a 404: every key on this platform's tier
// would be forbidden the same model, so rule it out for the rest of the request
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index ece1c7e5..4625246b 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -29,6 +29,7 @@ import {
getModelForbiddenCooldownMs,
learnLimitFromError,
type CooldownDecision,
+ type CooldownSource,
} from '../services/ratelimit.js';
import {
isRetryableError,
@@ -40,6 +41,7 @@ import {
isPaymentRequiredError,
isModelNotFoundError,
isModelAccessForbiddenError,
+ isAccountSuspendedError,
isProviderBadRequestError,
isProviderDegradedError,
isProviderLevelError,
@@ -49,7 +51,7 @@ import {
import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';
import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
import { noteModelRetirementSignal } from '../services/model-retirement.js';
-import { getSetting } from '../db/index.js';
+import { getDb, getSetting } from '../db/index.js';
import { newBreaker, recordBreakerFailure } from './guardrails.js';
import { getRequestTrace, newRequestTrace, runWithRequestTrace, type AttemptOutcome, type AttemptTraceRecord, type RequestTrace } from './attempt-trace.js';
import { logRequest, persistRequestAttempts } from './request-log.js';
@@ -112,6 +114,30 @@ function clearModelFailure(route: RouteResult): void {
modelFailureTimestamps.delete(route.modelDbId);
}
+/** Bench one key on every enabled model of its platform it can route to (the
+ * route's own model is benched by the caller). For KEY-level verdicts only —
+ * an account suspension — never for a model-level one. Never throws: an
+ * unreadable catalog leaves the narrower per-route bench in place.
+ */
+function benchKeyAcrossPlatform(route: RouteResult, durationMs: number, source: CooldownSource): void {
+ let modelIds: string[] = [];
+ try {
+ modelIds = (getDb().prepare(
+ 'SELECT model_id FROM models WHERE platform = ? AND enabled = 1 AND (key_id IS NULL OR key_id = ?)',
+ ).all(route.platform, route.keyId) as { model_id: string }[]).map(r => r.model_id);
+ } catch (dbErr: any) {
+ console.warn(`[FallbackLoop] could not list ${route.platform} models to bench key ${route.keyId}: ${dbErr?.message ?? dbErr}`);
+ return;
+ }
+ let benched = 0;
+ for (const modelId of modelIds) {
+ if (modelId === route.modelId) continue;
+ setCooldown(route.platform, modelId, route.keyId, durationMs, source);
+ benched++;
+ }
+ console.warn(`[FallbackLoop] ${route.platform} key ${route.keyId} reports the account suspended; benched it on ${benched + 1} model(s) for ${Math.round(durationMs / 60_000)}min`);
+}
+
// ── Wall-clock retry budget ──────────────────────────────────────────────────
// Serial failover has no time bound of its own: the observed worst case was a
// 38.8s TTFB over 11 attempts, and the theoretical worst is maxRetries x the
@@ -212,6 +238,10 @@ export function cooldownForError(route: RouteResult, err: any): number {
*/
export function cooldownDecisionForError(route: RouteResult, err: any): CooldownDecision {
if (isPaymentRequiredError(err)) return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };
+ // Before the model-forbidden check: a suspended account is a 403 too, but
+ // it is the KEY that is out, for as long as an empty balance would be — and
+ // under the same operator ceiling (#952).
+ if (isAccountSuspendedError(err)) return { durationMs: getPaymentRequiredCooldownMs(), source: 'credit' };
if (isModelAccessForbiddenError(err)) return { durationMs: getModelForbiddenCooldownMs(), source: 'tier' };
if (isDailyQuotaExhaustedError(err)) {
return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };
@@ -340,6 +370,19 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
+ // A suspended ACCOUNT fails every model behind the key identically. The
+ // per-route cooldown above benches only the model that happened to be
+ // tried, so the next request picks the platform's next model and pays the
+ // same round trip again — on a platform with a large catalog that is the
+ // whole failover budget, every request, until each model has tripped
+ // separately. Bench the key across the platform's catalog in one go, and
+ // rule the platform out for the rest of this request: a sibling key would
+ // be a different account, but with the budget already spent on this one
+ // the next PROVIDER is the better hop.
+ if (isAccountSuspendedError(err)) {
+ state.skipPlatforms.add(route.platform);
+ benchKeyAcrossPlatform(route, decision.durationMs, decision.source);
+ }
// Model-level failure benching: a model failing across keys (or repeatedly on
// one key) must sink out of routing instead of being re-picked every request.
noteModelFailure(route, now);