7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): fail over on model-level 403 instead of 502 (#256)
Tashfeenfb69fbcef193
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.tsindex c75f7584..cbda67ea 100644--- a/server/src/__tests__/routes/proxy-retry.test.ts+++ b/server/src/__tests__/routes/proxy-retry.test.ts@@ -1,5 +1,23 @@import { describe, it, expect } from 'vitest';-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError } from '../../routes/proxy.js';+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../../routes/proxy.js';++describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model skip — issue #256)', () => {+ it('flags a 403 reaching the proxy by message or attached status', () => {+ // GitHub Models / Cloudflare 403 a model the key's free tier can't reach.+ expect(isModelAccessForbiddenError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);+ expect(isModelAccessForbiddenError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);+ expect(isModelAccessForbiddenError(new Error('Forbidden'))).toBe(true);+ // #261 attaches the upstream status to the thrown error; honor it even if+ // the message phrasing omits the code.+ expect(isModelAccessForbiddenError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);+ });++ it('does not flag rate limits, 404s, or payment errors', () => {+ expect(isModelAccessForbiddenError(new Error('429 Too Many Requests'))).toBe(false);+ expect(isModelAccessForbiddenError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(false);+ expect(isModelAccessForbiddenError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);+ });+});describe('isModelNotFoundError (drives whole-model skip within a request)', () => {it('flags 404 / not-found / no-endpoints phrasings', () => {@@ -67,6 +85,22 @@ describe('isRetryableError', () => {});});+ describe('403 model not on this key\'s tier fails over instead of 502 (issue #256)', () => {+ it('treats a 403 from GitHub Models / Cloudflare as retryable', () => {+ expect(isRetryableError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);+ expect(isRetryableError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);+ });++ it('treats a bare "Forbidden" / attached 403 status as retryable', () => {+ expect(isRetryableError(new Error('Forbidden'))).toBe(true);+ expect(isRetryableError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);+ });++ it('still treats a bare 400 validation error as non-retryable', () => {+ expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);+ });+ });+describe('402 Payment Required out-of-credits fails over (graceful degradation)', () => {it('treats a HuggingFace Router 402 as retryable (same model lives on other providers)', () => {expect(isRetryableError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(true);@@ -100,11 +134,14 @@ describe('isRetryableError', () => {expect(isRetryableError(new Error('ECONNREFUSED'))).toBe(true);});- it('4xx auth/validation errors are NOT retryable', () => {+ it('401 / bare-400 auth & validation errors are NOT retryable', () => {expect(isRetryableError(new Error('401 Unauthorized'))).toBe(false);- expect(isRetryableError(new Error('403 Forbidden'))).toBe(false);expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);expect(isRetryableError(new Error('Invalid API key'))).toBe(false);+ // 403 is deliberately NOT here anymore: a request-time 403 on a key that+ // passed validateKey is a model-not-on-tier gate, so it fails over to the+ // next model rather than 502-ing the request (issue #256). The 403 cases+ // are covered in the dedicated describe block above.});});});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 8b6a0b84..1fb5e283 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -4,7 +4,7 @@ import type { Request, Response } from 'express';import { z } from 'zod';import type { ChatMessage, ModelListRow } from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult } from '../services/router.js';-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS } from '../services/ratelimit.js';+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS } from '../services/ratelimit.js';import { pruneRequestAnalytics } from '../services/request-retention.js';import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';import { getDb, getUnifiedApiKey } from '../db/index.js';@@ -322,6 +322,13 @@ export function isRetryableError(err: any): boolean {// for a model that's been pulled). Rotate to the next model in the chain —// setCooldown + the health checker will avoid this model on subsequent requests.|| msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found')+ // 403: the key is valid (it passed validateKey, and the health checker+ // disables truly-forbidden keys) but this specific model is off-limits to+ // the key's tier — e.g. gpt-4o on GitHub Models' free tier, subscription-only+ // models on Cloudflare. Another model in the chain is reachable, so fail over+ // instead of 502-ing the whole request. Paired with isModelAccessForbiddenError+ // to rule the model out for this request and a day-long bench. See issue #256.+ || isModelAccessForbiddenError(err)// 400: one provider may reject parameters another accepts (e.g. max_tokens// limits, unsupported params). The matching pattern is "api error 400"// which comes from the OpenAI-compat provider's error formatting, not@@ -363,6 +370,18 @@ export function isModelNotFoundError(err: any): boolean {return msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found');}+// A 403 Forbidden returned for a specific model behind an otherwise-valid key.+// Drives the same whole-model skip as a 404: every key on this platform's tier+// would be forbidden the same model, so rule it out for the rest of the request+// rather than trying it again with a sibling key. Distinct from a dead key —+// validateKey returns false on 401/403, so the health checker disables genuinely+// forbidden keys; a 403 reaching here is model-not-on-this-tier. See issue #256.+export function isModelAccessForbiddenError(err: any): boolean {+ if (err?.status === 403) return true;+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('403') || msg.includes('forbidden');+}+// Pull the incremental text out of a streaming chunk for token counting.// Must tolerate chunks that carry no `choices` array at all: some providers// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no@@ -959,7 +978,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// out for the rest of this request — its other keys would 404 the same// way. The per-key cooldown below still applies, so cross-request// behavior (#66/#76) is unchanged. (PR #111, credits @barbotkonv.)- if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);+ // 404 (removed upstream) and 403 (model off-limits to this key's tier)+ // both rule the model out: a sibling key on the same platform would+ // fail it identically, so skip it for the rest of this request.+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);// Put this model+key on cooldown and try the next oneconst skipId = `${route.platform}:${route.modelId}:${route.keyId}`;@@ -970,6 +992,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {route.keyId,isPaymentRequiredError(err)? PAYMENT_REQUIRED_COOLDOWN_MS+ // A 403 won't clear on the next window (it's a tier/subscription gate,+ // not a transient limit), so bench this model+key for a day like a 402+ // instead of re-trying it every request. See issue #256.+ : isModelAccessForbiddenError(err)+ ? MODEL_FORBIDDEN_COOLDOWN_MS: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {rpd: route.rpdLimit,tpd: route.tpdLimit,diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 252ed09c..2f9540d6 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -308,6 +308,14 @@ const TRANSIENT_COOLDOWN_MS = 90 * 1000;// on the next 402 after expiry if still unpaid; a restart re-benches on first hit.export const PAYMENT_REQUIRED_COOLDOWN_MS = DAY;+// Long cooldown for a 403 Forbidden on a key that already passed validateKey+// (so it is not a dead key — the health checker disables those). A request-time+// 403 means this key's tier can't reach this specific model (e.g. gpt-4o on+// GitHub Models' free tier, subscription-only models on Cloudflare). That won't+// change within a minute window, so bench this model+key for a full day and let+// the router fail over to a model the key can actually serve. See issue #256.+export const MODEL_FORBIDDEN_COOLDOWN_MS = DAY;+// Decide how long to bench a model+key after an upstream 429. Escalate to the// long quarantine (getNextCooldownDuration, up to 24h) ONLY when the model is// genuinely at its DAILY limit (RPD or TPD) — that won't recover until the